From 0443bb29c41a9aaee8c72e507b92425bd0731f60 Mon Sep 17 00:00:00 2001 From: Alman One Date: Tue, 6 Aug 2013 14:48:42 +0200 Subject: [PATCH 0001/1103] Initial riak support --- celery/backends/__init__.py | 1 + celery/backends/riak.py | 139 +++++++++++++++++++++++ celery/tests/backends/test_riak.py | 174 +++++++++++++++++++++++++++++ requirements/test-ci.txt | 1 + 4 files changed, 315 insertions(+) create mode 100644 celery/backends/riak.py create mode 100644 celery/tests/backends/test_riak.py diff --git a/celery/backends/__init__.py b/celery/backends/__init__.py index 421f7f480..4c5d87c80 100644 --- a/celery/backends/__init__.py +++ b/celery/backends/__init__.py @@ -34,6 +34,7 @@ 'database': 'celery.backends.database:DatabaseBackend', 'cassandra': 'celery.backends.cassandra:CassandraBackend', 'couchbase': 'celery.backends.couchbase:CouchBaseBackend', + 'riak': 'celery.backends.riak:RiakBackend', 'disabled': 'celery.backends.base:DisabledBackend', } diff --git a/celery/backends/riak.py b/celery/backends/riak.py new file mode 100644 index 000000000..67cff59b7 --- /dev/null +++ b/celery/backends/riak.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +""" + celery.backends.riak + ~~~~~~~~~~~~~~~~~~~~~~~ + + Riak result store backend. + +""" +from __future__ import absolute_import, print_function + +from datetime import datetime + +try: + import riak + from riak import RiakClient, RiakNode + from riak.resolver import last_written_resolver +except ImportError: # pragma: no cover + riak = None # noqa + +from kombu.utils.url import _parse_url + +from celery import states +from celery.exceptions import ImproperlyConfigured +from celery.utils.timeutils import maybe_timedelta + +from .base import KeyValueStoreBackend + + +class NonAsciiBucket(Exception): + """ Bucket must ne ascii charchters only. """ + + +class Validators(object): + + @classmethod + def validate_riak_bucket_name(cls, bucket_name): + try: + bucket_name.decode('ascii') + except UnicodeDecodeError as ude: + return False + return True + + +class RiakBackend(KeyValueStoreBackend): + # use protobuf by default? + bucket_name = "default" + host = 'localhost' + port = 8087 + + # supports_autoexpire = False + + def __init__(self, host=None, port=None, bucket_name=None, protocol=None, + url=None, *args, **kwargs): + """Initialize Riak backend instance. + + :raises celery.exceptions.ImproperlyConfigured: if + module :mod:`riak` is not available. + """ + super(RiakBackend, self).__init__(*args, **kwargs) + + self.expires = kwargs.get('expires') or maybe_timedelta( + self.app.conf.CELERY_TASK_RESULT_EXPIRES) + + if not riak: + raise ImproperlyConfigured( + 'You need to install the riak library to use the ' + 'Riak backend.') + + uhost = uport = uname = upass = ubucket = None + if url: + _, uhost, uport, uname, upass, ubucket, _ = _parse_url(url) + if ubucket: + ubucket = ubucket.strip('/') + + config = self.app.conf.get('CELERY_RIAK_BACKEND_SETTINGS', None) + if config is not None: + if not isinstance(config, dict): + raise ImproperlyConfigured( + 'Riak backend settings should be grouped in a dict') + else: + config = {} + + self.host = uhost or config.get('host', self.host) + self.port = int(uport or config.get('port', self.port)) + self.bucket_name = ubucket or config.get('bucket', self.bucket_name) + + # riak bucket must be ascii letters or numbers only + if not Validators.validate_riak_bucket_name(self.bucket_name): + raise NonAsciiBucket("Riak bucket names must be ASCII characters") + + self._client = None + + def _get_client(self): + """Get client connection""" + if self._client is None or not self._client.is_alive(): + kwargs = { + 'host': self.host, + 'port': self.port + } + if self.port: + kwargs.update({'port': self.port}) + logging.debug("riak settings %s" % kwargs) + self._client = RiakClient(protocol='pbc', + host=kwargs.get('host'), + pb_port=kwargs.get('port')) + self._client.resolver = last_written_resolver + return self._client + + def _get_bucket(self): + """Connect to our bucket""" + if ( + self._client is None or not self._client.is_alive() + or not self._bucket + ): + self._bucket = self.client.bucket(self.bucket_name) + return self._bucket + + @property + def client(self): + return self._get_client() + + @property + def bucket(self): + return self._get_bucket() + + def get(self, key): + return self.bucket.get(key).data + + def set(self, key, value): + # RiakBucket.new(key=None, data=None, content_type='application/json', + # encoded_data=None) + _key = self.bucket.new(key, data=value) + _key.store() + + def mget(self, keys): + return [self.get(key).data for key in keys] + + def delete(self, key): + self.bucket.delete(key) diff --git a/celery/tests/backends/test_riak.py b/celery/tests/backends/test_riak.py new file mode 100644 index 000000000..fd2a3728f --- /dev/null +++ b/celery/tests/backends/test_riak.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- + +from __future__ import absolute_import, with_statement + +from mock import MagicMock, Mock, patch, sentinel +from nose import SkipTest + +from celery import Celery +from celery.backends import riak as module +from celery.backends.riak import RiakBackend, riak, NonAsciiBucket +from celery.exceptions import ImproperlyConfigured +from celery.tests.utils import AppCase + + +RIAK_BUCKET = 'riak_bucket' + + +class test_RiakBackend(AppCase): + + def setUp(self): + if riak is None: + raise SkipTest('riak is not installed.') + from celery.app import current_app + self.app = self._current_app = current_app() + self.backend = RiakBackend(app=self.app) + + def test_init_no_riak(self): + """ + test init no riak raises + """ + prev, module.riak = module.riak, None + try: + with self.assertRaises(ImproperlyConfigured): + RiakBackend(app=self.app) + finally: + module.riak = prev + + def test_init_no_settings(self): + """ + test init no settings + """ + celery = Celery(set_as_current=False) + celery.conf.CELERY_RIAK_BACKEND_SETTINGS = [] + with self.assertRaises(ImproperlyConfigured): + RiakBackend(app=celery) + + def test_init_settings_is_None(self): + """ + Test init settings is None + """ + celery = Celery(set_as_current=False) + celery.conf.CELERY_RIAK_BACKEND_SETTINGS = None + RiakBackend(app=celery) + + def test_get_client_client_exists(self): + """ + Test get existing client + """ + with patch('riak.client.RiakClient') as mock_connection: + self.backend._client = sentinel._client + + mocked_is_alive = self.backend._client.is_alive = Mock() + mocked_is_alive.return_value.value = True + client = self.backend._get_client() + self.assertEquals(sentinel._client, client) + self.assertFalse(mock_connection.called) + + def test_get(self): + """ + Test get + RiakBackend.get + should return and take two params + db conn to riak is mocked + TODO Should test on key not exists + """ + celery = Celery(set_as_current=False) + + celery.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = {} + + backend = RiakBackend(app=celery) + backend._client = Mock() + backend._bucket = Mock() + mocked_get = backend._bucket.get = Mock() + mocked_get.return_value.data = sentinel.retval + # should return None + self.assertEqual(backend.get('1f3fab'), sentinel.retval) + backend._bucket.get.assert_called_once_with('1f3fab') + + def test_set(self): + """ + Test set + RiakBackend.set + should return None and take two params + db conn to couchbase is mocked + """ + celery = Celery(set_as_current=False) + celery.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = None + backend = RiakBackend(app=celery) + backend._client = MagicMock() + backend._bucket = MagicMock() + backend._bucket.set = MagicMock() + # should return None + self.assertIsNone(backend.set(sentinel.key, sentinel.value)) + + def test_delete(self): + """ + Test get + RiakBackend.get + should return and take two params + db conn to couchbase is mocked + TODO Should test on key not exists + """ + celery = Celery(set_as_current=False) + + celery.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = {} + + backend = RiakBackend(app=celery) + backend._client = Mock() + backend._bucket = Mock() + mocked_delete = backend._client.delete = Mock() + mocked_delete.return_value = None + # should return None + self.assertIsNone(backend.delete('1f3fab')) + backend._bucket.delete.assert_called_once_with('1f3fab') + + def test_config_params(self): + """ + test celery.conf.CELERY_RIAK_BACKEND_SETTINGS + celery.conf.CELERY_RIAK_BACKEND_SETTINGS + is properly set + """ + celery = Celery(set_as_current=False) + celery.conf.CELERY_RIAK_BACKEND_SETTINGS = {'bucket': 'mycoolbucket', + 'host': 'there.host.com', + 'port': '1234'} + backend = RiakBackend(app=celery) + self.assertEqual(backend.bucket_name, "mycoolbucket") + self.assertEqual(backend.host, 'there.host.com') + self.assertEqual(backend.port, 1234) + + def test_backend_by_url(self, url='riak://myhost/mycoolbucket'): + """ + test get backend by url + """ + from celery.backends.riak import RiakBackend + backend, url_ = backends.get_backend_by_url(url) + self.assertIs(backend, RiakBackend) + self.assertEqual(url_, url) + + def test_backend_params_by_url(self): + """ + test get backend params by url + """ + celery = Celery(set_as_current=False, + backend='riak://myhost:123/mycoolbucket') + backend = celery.backend + self.assertEqual(backend.bucket_name, "mycoolbucket") + self.assertEqual(backend.host, "myhost") + self.assertEqual(backend.port, 123) + + def test_non_ASCII_bucket_raises(self): + """ + test celery.conf.CELERY_RIAK_BACKEND_SETTINGS + celery.conf.CELERY_RIAK_BACKEND_SETTINGS + is properly set + """ + with self.assertRaises(NonAsciiBucket): + celery = Celery(set_as_current=False) + celery.conf.CELERY_RIAK_BACKEND_SETTINGS = { + 'bucket': 'héhé', + 'host': 'there.host.com', + 'port': '1234', + } + RiakBackend(app=celery) diff --git a/requirements/test-ci.txt b/requirements/test-ci.txt index a220100d7..275494e1a 100644 --- a/requirements/test-ci.txt +++ b/requirements/test-ci.txt @@ -1,5 +1,6 @@ coverage>=3.0 redis +#riak >=2.0 #pymongo #SQLAlchemy #PyOpenSSL From 6511e70585ad2b3440eafc9af17906457a48a6b9 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 14 Oct 2013 11:15:50 +0200 Subject: [PATCH 0002/1103] Add some documentation on class attributes Also add handling of protocol. --- celery/backends/riak.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/celery/backends/riak.py b/celery/backends/riak.py index 67cff59b7..4beb9c550 100644 --- a/celery/backends/riak.py +++ b/celery/backends/riak.py @@ -42,9 +42,17 @@ def validate_riak_bucket_name(cls, bucket_name): class RiakBackend(KeyValueStoreBackend): - # use protobuf by default? + # TODO: allow using other protocols than protobuf ? + #: default protocol used to connect to Riak, might be `http` or `pbc` + protocol = 'pbc' + + #: default Riak bucket name (`default`) bucket_name = "default" + + #: default Riak server hostname (`localhost`) host = 'localhost' + + #: default Riak server port (8087) port = 8087 # supports_autoexpire = False @@ -68,7 +76,7 @@ def __init__(self, host=None, port=None, bucket_name=None, protocol=None, uhost = uport = uname = upass = ubucket = None if url: - _, uhost, uport, uname, upass, ubucket, _ = _parse_url(url) + uprot, uhost, uport, uname, upass, ubucket, _ = _parse_url(url) if ubucket: ubucket = ubucket.strip('/') @@ -83,6 +91,7 @@ def __init__(self, host=None, port=None, bucket_name=None, protocol=None, self.host = uhost or config.get('host', self.host) self.port = int(uport or config.get('port', self.port)) self.bucket_name = ubucket or config.get('bucket', self.bucket_name) + self.protocol = uprot or config.get('protocol', self.protocol) # riak bucket must be ascii letters or numbers only if not Validators.validate_riak_bucket_name(self.bucket_name): @@ -100,7 +109,7 @@ def _get_client(self): if self.port: kwargs.update({'port': self.port}) logging.debug("riak settings %s" % kwargs) - self._client = RiakClient(protocol='pbc', + self._client = RiakClient(protocol=self.protocol, host=kwargs.get('host'), pb_port=kwargs.get('port')) self._client.resolver = last_written_resolver From 844d9a6ef498ec4031460654398e16485085beca Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 14 Oct 2013 14:23:07 +0200 Subject: [PATCH 0003/1103] Cleanup code related to connection to Riak This is a leftover of previous work on this backend. --- celery/backends/riak.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/celery/backends/riak.py b/celery/backends/riak.py index 4beb9c550..b1c1d40af 100644 --- a/celery/backends/riak.py +++ b/celery/backends/riak.py @@ -102,16 +102,9 @@ def __init__(self, host=None, port=None, bucket_name=None, protocol=None, def _get_client(self): """Get client connection""" if self._client is None or not self._client.is_alive(): - kwargs = { - 'host': self.host, - 'port': self.port - } - if self.port: - kwargs.update({'port': self.port}) - logging.debug("riak settings %s" % kwargs) self._client = RiakClient(protocol=self.protocol, - host=kwargs.get('host'), - pb_port=kwargs.get('port')) + host=self.host, + pb_port=self.port) self._client.resolver = last_written_resolver return self._client From c4a372a15bd16cf4ca39751429ab9b1ba470f164 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 14 Oct 2013 15:06:10 +0200 Subject: [PATCH 0004/1103] Switch default bucket name to celery --- celery/backends/riak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/riak.py b/celery/backends/riak.py index b1c1d40af..ebdcca813 100644 --- a/celery/backends/riak.py +++ b/celery/backends/riak.py @@ -47,7 +47,7 @@ class RiakBackend(KeyValueStoreBackend): protocol = 'pbc' #: default Riak bucket name (`default`) - bucket_name = "default" + bucket_name = "celery" #: default Riak server hostname (`localhost`) host = 'localhost' From 4859492883757340eb16ad9858d57263a1468281 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 14 Oct 2013 15:36:38 +0200 Subject: [PATCH 0005/1103] Add riak to extra requirements --- requirements/extras/riak.txt | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 requirements/extras/riak.txt diff --git a/requirements/extras/riak.txt b/requirements/extras/riak.txt new file mode 100644 index 000000000..b6bfed133 --- /dev/null +++ b/requirements/extras/riak.txt @@ -0,0 +1 @@ +riak >=2.0 diff --git a/setup.py b/setup.py index a787d647d..d5dac4c07 100644 --- a/setup.py +++ b/setup.py @@ -199,6 +199,7 @@ def reqs(*f): 'mongodb': extras('mongodb.txt'), 'sqs': extras('sqs.txt'), 'couchdb': extras('couchdb.txt'), + 'riak': extras('riak.txt'), 'beanstalk': extras('beanstalk.txt'), 'zookeeper': extras('zookeeper.txt'), 'zeromq': extras('zeromq.txt'), From 2b79d6d5d69e22234e119e77d25020c6e1c466d4 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 14 Oct 2013 15:06:27 +0200 Subject: [PATCH 0006/1103] Add Riak backend documentation --- docs/configuration.rst | 65 ++++++++++++++++++++++++++++++++++ docs/includes/installation.txt | 3 ++ 2 files changed, 68 insertions(+) diff --git a/docs/configuration.rst b/docs/configuration.rst index ac113dbde..a44b06903 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -628,6 +628,71 @@ Example configuration 'max_retries': 10 } +.. _conf-riak-result-backend: + +Riak backend settings +--------------------- + +.. note:: + + The Riak backend requires the :mod:`riak` library: + http://pypi.python.org/pypi/riak/ + + To install the riak package use `pip` or `easy_install`: + + .. code-block:: bash + + $ pip install riak + +This backend requires the :setting:`CELERY_RESULT_BACKEND` +setting to be set to a Riak URL:: + + CELERY_RESULT_BACKEND = "riak://host:port/bucket" + +For example:: + + CELERY_RESULT_BACKEND = "riak://localhost/celery + +which is the same as:: + + CELERY_RESULT_BACKEND = "riak://" + +The fields of the URL is defined as folows: + +- *host* + +Host name or IP address of the Riak server. e.g. `"localhost"`. + +- *port* + +Port to the Riak server using the protobuf protocol. Default is 8087. + +- *bucket* + +Bucket name to use. Default is `celery`. +The bucket needs to be a string with ascii characters only. + +Altenatively, this backend can be configured with the following configuration directives. + +.. setting:: CELERY_RIAK_BACKEND_SETTINGS + +CELERY_RIAK_BACKEND_SETTINGS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This is a dict supporting the following keys: + +* host + The host name of the Riak server. Defaults to "localhost". + +* port + The port the Riak server is listening to. Defaults to 8087. + +* bucket + The bucket name to connect to. Defaults to "celery". + +* protocol + The protocol to use to connect to the Riak server. This is not configurable + via :setting:`CELERY_RESULT_BACKEND` .. _conf-ironcache-result-backend: diff --git a/docs/includes/installation.txt b/docs/includes/installation.txt index fda618d08..e568ebc52 100644 --- a/docs/includes/installation.txt +++ b/docs/includes/installation.txt @@ -85,6 +85,9 @@ Transports and Backends :celery[couchbase]: for using CouchBase as a result backend. +:celery[riak]: + for using Riak as a result backend. + :celery[beanstalk]: for using Beanstalk as a message transport. From 1ac10f3b8f2bc0a21b7e418ee6c967df614bd106 Mon Sep 17 00:00:00 2001 From: NoKriK Date: Tue, 15 Oct 2013 11:42:26 +0200 Subject: [PATCH 0007/1103] Fixed error in protocol handling for the riak backend --- celery/backends/riak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/riak.py b/celery/backends/riak.py index ebdcca813..725e396a5 100644 --- a/celery/backends/riak.py +++ b/celery/backends/riak.py @@ -91,7 +91,7 @@ def __init__(self, host=None, port=None, bucket_name=None, protocol=None, self.host = uhost or config.get('host', self.host) self.port = int(uport or config.get('port', self.port)) self.bucket_name = ubucket or config.get('bucket', self.bucket_name) - self.protocol = uprot or config.get('protocol', self.protocol) + self.protocol = protocol or config.get('protocol', self.protocol) # riak bucket must be ascii letters or numbers only if not Validators.validate_riak_bucket_name(self.bucket_name): From 7ad7a57fa12da3cde49c0f043c40b2bc5789c459 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 22 Nov 2013 17:06:43 +0000 Subject: [PATCH 0008/1103] Implements steeves dynamic tasks as task.replace. Issue #817 --- celery/app/task.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/celery/app/task.py b/celery/app/task.py index ee96c5c6b..d92e1a2e1 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -661,6 +661,17 @@ def retry(self, args=None, kwargs=None, exc=None, throw=True, raise ret return ret + def replace(self, sig): + request = self.request + sig.set_immutable(True) + chord_id, request.chord = request.chord, None + group_id, request.group = request.group, None + callbacks, request.callbacks = request.callbacks, [sig] + if group_id or chord_id: + sig.set(group=group_id, chord=chord_id) + sig |= callbacks[0] + return sig + def apply(self, args=None, kwargs=None, link=None, link_error=None, **options): """Execute this task locally, by blocking until the task returns. From dc4634298f557a2f898a9bc4772fffb79c45eede Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 23 Jan 2014 15:28:25 +0000 Subject: [PATCH 0009/1103] Use a heapq to avoid calling all entries for every iteration --- celery/beat.py | 73 +++++++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/celery/beat.py b/celery/beat.py index f93e3bbf9..8473017d7 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -9,12 +9,14 @@ from __future__ import absolute_import import errno +import heapq import os import time import shelve import sys import traceback +from collections import namedtuple from threading import Event, Thread from billiard import Process, ensure_multiprocessing @@ -34,6 +36,8 @@ __all__ = ['SchedulingError', 'ScheduleEntry', 'Scheduler', 'PersistentScheduler', 'Service', 'EmbeddedService'] +event_t = namedtuple('event_t', ('time', 'priority', 'entry')) + logger = get_logger(__name__) debug, info, error, warning = (logger.debug, logger.info, logger.error, logger.warning) @@ -173,6 +177,7 @@ def __init__(self, app, schedule=None, max_interval=None, or app.conf.CELERYBEAT_MAX_LOOP_INTERVAL or self.max_interval) self.Publisher = Publisher or app.amqp.TaskProducer + self._heap = None if not lazy: self.setup_schedule() @@ -191,32 +196,45 @@ def maybe_due(self, entry, publisher=None): is_due, next_time_to_run = entry.is_due() if is_due: - info('Scheduler: Sending due task %s (%s)', entry.name, entry.task) - try: - result = self.apply_async(entry, publisher=publisher) - except Exception as exc: - error('Message Error: %s\n%s', - exc, traceback.format_stack(), exc_info=True) - else: - debug('%s sent. id->%s', entry.task, result.id) + self.apply_entry(entry, producer=publisher, advance=True) return next_time_to_run - def tick(self): + def apply_entry(self, entry, producer=None): + info('Scheduler: Sending due task %s (%s)', entry.name, entry.task) + try: + result = self.apply_async(entry, producer=producer, advance=False) + except Exception as exc: + error('Message Error: %s\n%s', + exc, traceback.format_stack(), exc_info=True) + else: + debug('%s sent. id->%s', entry.task, result.id) + + def tick(self, event_t=event_t, min=min, + heappop=heapq.heappop, heappush=heapq.heappush): """Run a tick, that is one iteration of the scheduler. Executes all due tasks. """ - remaining_times = [] - try: - for entry in values(self.schedule): - next_time_to_run = self.maybe_due(entry, self.publisher) - if next_time_to_run: - remaining_times.append(next_time_to_run) - except RuntimeError: - pass - - return min(remaining_times + [self.max_interval]) + H = self._heap + if H is None: + H = self._heap = [event_t(e.is_due()[1], 5, e) + for e in values(self.schedule)] + print('HEAP: %r' % (H, )) + event = H[0] + entry = event[2] + is_due, next_time_to_run = entry.is_due() + if is_due: + verify = heappop(H) + if verify is event: + next_entry = self.reserve(entry) + self.apply_entry(entry, producer=self.publisher) + heappush(H, event_t(next_time_to_run, event[1], next_entry)) + return 0 + else: + heappush(H, verify) + return min(verify[0], self.max_interval) + return min(next_time_to_run, self.max_interval) def should_sync(self): return (not self._last_sync or @@ -226,22 +244,22 @@ def reserve(self, entry): new_entry = self.schedule[entry.name] = next(entry) return new_entry - def apply_async(self, entry, publisher=None, **kwargs): + def apply_async(self, entry, producer=None, advance=True, **kwargs): # Update timestamps and run counts before we actually execute, # so we have that done if an exception is raised (doesn't schedule # forever.) - entry = self.reserve(entry) + entry = self.reserve(entry) if advance else entry task = self.app.tasks.get(entry.task) try: if task: - result = task.apply_async(entry.args, entry.kwargs, - publisher=publisher, - **entry.options) - else: - result = self.send_task(entry.task, entry.args, entry.kwargs, - publisher=publisher, + return task.apply_async(entry.args, entry.kwargs, + producer=producer, **entry.options) + else: + return self.send_task(entry.task, entry.args, entry.kwargs, + producer=producer, + **entry.options) except Exception as exc: reraise(SchedulingError, SchedulingError( "Couldn't apply scheduled task {0.name}: {exc}".format( @@ -249,7 +267,6 @@ def apply_async(self, entry, publisher=None, **kwargs): finally: if self.should_sync(): self._do_sync() - return result def send_task(self, *args, **kwargs): return self.app.send_task(*args, **kwargs) From 044c8a462117c27cb27133f5ea484bb804f16a6e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 23 Jan 2014 15:30:55 +0000 Subject: [PATCH 0010/1103] Remove print --- celery/beat.py | 1 - 1 file changed, 1 deletion(-) diff --git a/celery/beat.py b/celery/beat.py index 8473017d7..48aad7fb4 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -220,7 +220,6 @@ def tick(self, event_t=event_t, min=min, if H is None: H = self._heap = [event_t(e.is_due()[1], 5, e) for e in values(self.schedule)] - print('HEAP: %r' % (H, )) event = H[0] entry = event[2] is_due, next_time_to_run = entry.is_due() From 95cdbac9683a89fd35429ce43129d191c6165bae Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Sat, 22 Mar 2014 16:31:18 +0000 Subject: [PATCH 0011/1103] Tox: Add debug test output --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index e532ca2d4..d2a0ac4a3 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt -r{toxinidir}/requirements/test-ci.txt commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + C_DEBUG_TEST=1 nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:3.3] basepython = python3.3 @@ -24,7 +24,7 @@ deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt -r{toxinidir}/requirements/test-ci.txt commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + C_DEBUG_TEST=1 nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:2.7] basepython = python2.7 @@ -32,7 +32,7 @@ deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/test-ci.txt commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + C_DEBUG_TEST=1 nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:2.6] basepython = python2.6 @@ -40,7 +40,7 @@ deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/test-ci.txt commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + C_DEBUG_TEST=1 nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:pypy] basepython = pypy @@ -48,4 +48,4 @@ deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/test-ci.txt commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + C_DEBUG_TEST=1 nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] From 5254f9a75b3a2c677fd7165de10fd09bbfd48232 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Sat, 22 Mar 2014 16:37:32 +0000 Subject: [PATCH 0012/1103] Tox: Set environment variables properly --- .travis.yml | 10 +++++----- tox.ini | 15 ++++++++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index a30e3602f..c8341f045 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,10 +4,10 @@ env: global: PYTHONUNBUFFERED=yes matrix: - - TOXENV=2.6 - - TOXENV=2.7 - - TOXENV=3.3 - - TOXENV=3.4 + - TOXENV=2.6 + - TOXENV=2.7 + - TOXENV=3.3 + - TOXENV=3.4 - TOXENV=pypy before_install: - | @@ -39,4 +39,4 @@ notifications: channels: - "chat.freenode.net#celery" on_success: always - on_failure: always \ No newline at end of file + on_failure: always diff --git a/tox.ini b/tox.ini index d2a0ac4a3..d8605c74d 100644 --- a/tox.ini +++ b/tox.ini @@ -15,37 +15,42 @@ basepython = python3.4 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt -r{toxinidir}/requirements/test-ci.txt +setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - C_DEBUG_TEST=1 nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:3.3] basepython = python3.3 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt -r{toxinidir}/requirements/test-ci.txt +setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - C_DEBUG_TEST=1 nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:2.7] basepython = python2.7 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/test-ci.txt +setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - C_DEBUG_TEST=1 nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:2.6] basepython = python2.6 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/test-ci.txt +setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - C_DEBUG_TEST=1 nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:pypy] basepython = pypy deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/test-ci.txt +setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - C_DEBUG_TEST=1 nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] From f333abc09168e5fd2ec9fd30a3ee80cc510e6513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ionel=20Cristian=20M=C4=83rie=C8=99?= Date: Sat, 1 Mar 2014 14:00:57 +0200 Subject: [PATCH 0013/1103] Make the database backend retry operations on ResourceClosedError and StaleDataError too. Make the operations close the connection if failure occurs (can't retry on broken connection). Fixes #1786. --- celery/backends/database/__init__.py | 69 ++++++++++++++++------------ 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/celery/backends/database/__init__.py b/celery/backends/database/__init__.py index 58109e782..dea6c833a 100644 --- a/celery/backends/database/__init__.py +++ b/celery/backends/database/__init__.py @@ -8,18 +8,22 @@ """ from __future__ import absolute_import +import logging +from contextlib import contextmanager from functools import wraps from celery import states +from celery.backends.base import BaseBackend from celery.exceptions import ImproperlyConfigured from celery.five import range from celery.utils.timeutils import maybe_timedelta -from celery.backends.base import BaseBackend - -from .models import Task, TaskSet +from .models import Task +from .models import TaskSet from .session import ResultSession +logger = logging.getLogger(__name__) + __all__ = ['DatabaseBackend'] @@ -33,7 +37,21 @@ def _sqlalchemy_installed(): return sqlalchemy _sqlalchemy_installed() -from sqlalchemy.exc import DatabaseError, OperationalError +from sqlalchemy.exc import DatabaseError, OperationalError, ResourceClosedError +from sqlalchemy.orm.exc import StaleDataError + + +@contextmanager +def session_cleanup(session): + try: + yield + except (DatabaseError, OperationalError, ResourceClosedError, StaleDataError): + session.rollback() + session.connection().invalidate() + session.close() + raise + else: + session.close() def retry(fun): @@ -45,7 +63,12 @@ def _inner(*args, **kwargs): for retries in range(max_retries): try: return fun(*args, **kwargs) - except (DatabaseError, OperationalError): + except (DatabaseError, OperationalError, ResourceClosedError, StaleDataError): + logger.critical( + "Failed operation %s. Retrying %s more times.", + fun.__name__, max_retries - retries - 1, + exc_info=True, + ) if retries + 1 >= max_retries: raise @@ -95,8 +118,9 @@ def _store_result(self, task_id, result, status, traceback=None, max_retries=3, **kwargs): """Store return value and status of an executed task.""" session = self.ResultSession() - try: - task = session.query(Task).filter(Task.task_id == task_id).first() + with session_cleanup(session): + task = list(session.query(Task).filter(Task.task_id == task_id)) + task = task and task[0] if not task: task = Task(task_id) session.add(task) @@ -106,83 +130,70 @@ def _store_result(self, task_id, result, status, task.traceback = traceback session.commit() return result - finally: - session.close() @retry def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" session = self.ResultSession() - try: - task = session.query(Task).filter(Task.task_id == task_id).first() - if task is None: + with session_cleanup(session): + task = list(session.query(Task).filter(Task.task_id == task_id)) + task = task and task[0] + if not task: task = Task(task_id) task.status = states.PENDING task.result = None return task.to_dict() - finally: - session.close() @retry def _save_group(self, group_id, result): """Store the result of an executed group.""" session = self.ResultSession() - try: + with session_cleanup(session): group = TaskSet(group_id, result) session.add(group) session.flush() session.commit() return result - finally: - session.close() @retry def _restore_group(self, group_id): """Get metadata for group by id.""" session = self.ResultSession() - try: + with session_cleanup(session): group = session.query(TaskSet).filter( TaskSet.taskset_id == group_id).first() if group: return group.to_dict() - finally: - session.close() @retry def _delete_group(self, group_id): """Delete metadata for group by id.""" session = self.ResultSession() - try: + with session_cleanup(session): session.query(TaskSet).filter( TaskSet.taskset_id == group_id).delete() session.flush() session.commit() - finally: - session.close() @retry def _forget(self, task_id): """Forget about result.""" session = self.ResultSession() - try: + with session_cleanup(session): session.query(Task).filter(Task.task_id == task_id).delete() session.commit() - finally: - session.close() def cleanup(self): """Delete expired metadata.""" session = self.ResultSession() expires = self.expires now = self.app.now() - try: + with session_cleanup(session): session.query(Task).filter( Task.date_done < (now - expires)).delete() session.query(TaskSet).filter( TaskSet.date_done < (now - expires)).delete() session.commit() - finally: - session.close() def __reduce__(self, args=(), kwargs={}): kwargs.update( From 688c3fe2eeacd4c3e9e6bce765f306bd44ea7786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ionel=20Cristian=20M=C4=83rie=C8=99?= Date: Sun, 2 Mar 2014 18:18:00 +0200 Subject: [PATCH 0014/1103] Don't close the session. Just rollback and/or close. --- celery/backends/database/__init__.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/celery/backends/database/__init__.py b/celery/backends/database/__init__.py index dea6c833a..ffcb6113b 100644 --- a/celery/backends/database/__init__.py +++ b/celery/backends/database/__init__.py @@ -37,7 +37,7 @@ def _sqlalchemy_installed(): return sqlalchemy _sqlalchemy_installed() -from sqlalchemy.exc import DatabaseError, OperationalError, ResourceClosedError +from sqlalchemy.exc import DatabaseError, OperationalError, ResourceClosedError, InvalidRequestError from sqlalchemy.orm.exc import StaleDataError @@ -45,12 +45,10 @@ def _sqlalchemy_installed(): def session_cleanup(session): try: yield - except (DatabaseError, OperationalError, ResourceClosedError, StaleDataError): + except Exception: session.rollback() - session.connection().invalidate() - session.close() raise - else: + finally: session.close() @@ -63,8 +61,8 @@ def _inner(*args, **kwargs): for retries in range(max_retries): try: return fun(*args, **kwargs) - except (DatabaseError, OperationalError, ResourceClosedError, StaleDataError): - logger.critical( + except (DatabaseError, OperationalError, ResourceClosedError, StaleDataError, InvalidRequestError): + logger.warning( "Failed operation %s. Retrying %s more times.", fun.__name__, max_retries - retries - 1, exc_info=True, From 7be5028d9abc94b70f16a4ee29beb78876718f66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ionel=20Cristian=20M=C4=83rie=C8=99?= Date: Mon, 3 Mar 2014 22:02:57 +0200 Subject: [PATCH 0015/1103] Close the sessions before disposing the engines as the engines won't close connections held up in sessions. --- celery/backends/database/session.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/celery/backends/database/session.py b/celery/backends/database/session.py index fef3843e4..cb25b5c95 100644 --- a/celery/backends/database/session.py +++ b/celery/backends/database/session.py @@ -29,10 +29,12 @@ class _after_fork(object): def __call__(self): self.registered = False # child must reregister + for session in _SESSIONS: + session.close() + _SESSIONS.clear() for engine in list(_ENGINES.values()): engine.dispose() _ENGINES.clear() - _SESSIONS.clear() after_fork = _after_fork() From 6e8ab99dc94c9461fc433763c9a693dac19b188a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ionel=20Cristian=20M=C4=83rie=C8=99?= Date: Tue, 4 Mar 2014 00:03:01 +0200 Subject: [PATCH 0016/1103] Use a session manager that has different behavior before the fork (effectivelly hardcodes NullPool - everything else is unreliable). --- celery/backends/database/__init__.py | 13 ++-- celery/backends/database/session.py | 97 +++++++++++++--------------- 2 files changed, 54 insertions(+), 56 deletions(-) diff --git a/celery/backends/database/__init__.py b/celery/backends/database/__init__.py index ffcb6113b..1dbcb5e32 100644 --- a/celery/backends/database/__init__.py +++ b/celery/backends/database/__init__.py @@ -20,7 +20,7 @@ from .models import Task from .models import TaskSet -from .session import ResultSession +from .session import SessionManager logger = logging.getLogger(__name__) @@ -37,7 +37,7 @@ def _sqlalchemy_installed(): return sqlalchemy _sqlalchemy_installed() -from sqlalchemy.exc import DatabaseError, OperationalError, ResourceClosedError, InvalidRequestError +from sqlalchemy.exc import DatabaseError, OperationalError, ResourceClosedError, InvalidRequestError, IntegrityError from sqlalchemy.orm.exc import StaleDataError @@ -61,7 +61,10 @@ def _inner(*args, **kwargs): for retries in range(max_retries): try: return fun(*args, **kwargs) - except (DatabaseError, OperationalError, ResourceClosedError, StaleDataError, InvalidRequestError): + except ( + DatabaseError, OperationalError, ResourceClosedError, StaleDataError, InvalidRequestError, + IntegrityError + ): logger.warning( "Failed operation %s. Retrying %s more times.", fun.__name__, max_retries - retries - 1, @@ -104,8 +107,8 @@ def __init__(self, dburi=None, expires=None, 'Missing connection string! Do you have ' 'CELERY_RESULT_DBURI set to a real value?') - def ResultSession(self): - return ResultSession( + def ResultSession(self, session_manager=SessionManager()): + return session_manager.session_factory( dburi=self.dburi, short_lived_sessions=self.short_lived_sessions, **self.engine_options diff --git a/celery/backends/database/session.py b/celery/backends/database/session.py index cb25b5c95..1575d7f32 100644 --- a/celery/backends/database/session.py +++ b/celery/backends/database/session.py @@ -8,60 +8,55 @@ """ from __future__ import absolute_import -from collections import defaultdict -from multiprocessing.util import register_after_fork +from billiard.util import register_after_fork from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import NullPool ResultModelBase = declarative_base() -_SETUP = defaultdict(lambda: False) -_ENGINES = {} -_SESSIONS = {} - -__all__ = ['ResultSession', 'get_engine', 'create_session'] - - -class _after_fork(object): - registered = False - - def __call__(self): - self.registered = False # child must reregister - for session in _SESSIONS: - session.close() - _SESSIONS.clear() - for engine in list(_ENGINES.values()): - engine.dispose() - _ENGINES.clear() -after_fork = _after_fork() - - -def get_engine(dburi, **kwargs): - try: - return _ENGINES[dburi] - except KeyError: - engine = _ENGINES[dburi] = create_engine(dburi, **kwargs) - after_fork.registered = True - register_after_fork(after_fork, after_fork) - return engine - - -def create_session(dburi, short_lived_sessions=False, **kwargs): - engine = get_engine(dburi, **kwargs) - if short_lived_sessions or dburi not in _SESSIONS: - _SESSIONS[dburi] = sessionmaker(bind=engine) - return engine, _SESSIONS[dburi] - - -def setup_results(engine): - if not _SETUP['results']: - ResultModelBase.metadata.create_all(engine) - _SETUP['results'] = True - - -def ResultSession(dburi, **kwargs): - engine, session = create_session(dburi, **kwargs) - setup_results(engine) - return session() +__all__ = ['SessionManager'] + + +class SessionManager(object): + def __init__(self): + self._engines = {} + self._sessions = {} + self.forked = False + self.prepared = False + register_after_fork(self, self._after_fork) + + def _after_fork(self,): + self.forked = True + + def get_engine(self, dburi, **kwargs): + if self.forked: + try: + return self._engines[dburi] + except KeyError: + engine = self._engines[dburi] = create_engine(dburi, **kwargs) + return engine + else: + kwargs['poolclass'] = NullPool + return create_engine(dburi, **kwargs) + + def create_session(self, dburi, short_lived_sessions=False, **kwargs): + engine = self.get_engine(dburi, **kwargs) + if self.forked: + if short_lived_sessions or dburi not in self._sessions: + self._sessions[dburi] = sessionmaker(bind=engine) + return engine, self._sessions[dburi] + else: + return engine, sessionmaker(bind=engine) + + def prepare_models(self, engine): + if not self.prepared: + ResultModelBase.metadata.create_all(engine) + self.prepared = True + + def session_factory(self, dburi, **kwargs): + engine, session = self.create_session(dburi, **kwargs) + self.prepare_models(engine) + return session() From df32c477e667e980a24e0c84f74dfb8546771e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ionel=20Cristian=20M=C4=83rie=C8=99?= Date: Sat, 22 Mar 2014 23:09:26 +0200 Subject: [PATCH 0017/1103] Update changelog. --- Changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Changelog b/Changelog index 9ecc78129..aa55bdc7f 100644 --- a/Changelog +++ b/Changelog @@ -19,6 +19,12 @@ new in Celery 3.1. - Now depends on :ref:`Kombu 3.0.14 `. +- **Results**: + + Reliability improvements to the SQLAlchemy database backend. Previously the + connection from the MainProcess was improperly shared with the workers. + (Issue #1786) + - **Redis:** Important note about events (Issue #1882). There is a new transport option for Redis that enables monitors From 59fa886071eedb4991c892784c6fb26898efdfd3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Sat, 22 Mar 2014 21:33:16 +0000 Subject: [PATCH 0018/1103] Cosmetics for #1736 --- celery/backends/database/__init__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/celery/backends/database/__init__.py b/celery/backends/database/__init__.py index 1dbcb5e32..c52e75879 100644 --- a/celery/backends/database/__init__.py +++ b/celery/backends/database/__init__.py @@ -37,7 +37,7 @@ def _sqlalchemy_installed(): return sqlalchemy _sqlalchemy_installed() -from sqlalchemy.exc import DatabaseError, OperationalError, ResourceClosedError, InvalidRequestError, IntegrityError +from sqlalchemy.exc import DatabaseError, InvalidRequestError from sqlalchemy.orm.exc import StaleDataError @@ -61,10 +61,7 @@ def _inner(*args, **kwargs): for retries in range(max_retries): try: return fun(*args, **kwargs) - except ( - DatabaseError, OperationalError, ResourceClosedError, StaleDataError, InvalidRequestError, - IntegrityError - ): + except (DatabaseError, InvalidRequestError, StaleDataError): logger.warning( "Failed operation %s. Retrying %s more times.", fun.__name__, max_retries - retries - 1, From 5ff19addac2992de2ed6a46f307cfcfb7543a384 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Sat, 22 Mar 2014 21:39:20 +0000 Subject: [PATCH 0019/1103] flakes --- celery/tests/backends/test_database.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/tests/backends/test_database.py b/celery/tests/backends/test_database.py index fac02215e..6b5bf9420 100644 --- a/celery/tests/backends/test_database.py +++ b/celery/tests/backends/test_database.py @@ -42,16 +42,16 @@ def setup(self): self.uri = 'sqlite:///test.db' def test_retry_helper(self): - from celery.backends.database import OperationalError + from celery.backends.database import DatabaseError calls = [0] @retry def raises(): calls[0] += 1 - raise OperationalError(1, 2, 3) + raise DatabaseError(1, 2, 3) - with self.assertRaises(OperationalError): + with self.assertRaises(DatabaseError): raises(max_retries=5) self.assertEqual(calls[0], 5) From 15b531a9dcc18a85aa8e6488024f3be9330285e1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Sat, 22 Mar 2014 21:39:33 +0000 Subject: [PATCH 0020/1103] Stress test app now loads on Windows --- funtests/stress/stress/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funtests/stress/stress/app.py b/funtests/stress/stress/app.py index e3d72da5e..0a9690cfb 100644 --- a/funtests/stress/stress/app.py +++ b/funtests/stress/stress/app.py @@ -91,7 +91,7 @@ def exiting(status=0): @app.task -def kill(sig=signal.SIGKILL): +def kill(sig=getattr(signal, 'SIGKILL', None) or signal.SIGTERM): os.kill(os.getpid(), sig) From 620828c26b874fd2c357cd7f8ba4d4ec1dadd0a4 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Sat, 22 Mar 2014 21:43:16 +0000 Subject: [PATCH 0021/1103] Sets release date for 3.1.10 --- Changelog | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Changelog b/Changelog index aa55bdc7f..62abdf694 100644 --- a/Changelog +++ b/Changelog @@ -12,19 +12,13 @@ new in Celery 3.1. 3.1.10 ====== -:release-date: 2014-XX-XX XX:XX X.X UTC -:release-by: XX +:release-date: 2014-03-22 09:40 P.M UTC +:release-by: Ask Solem - **Requirements**: - Now depends on :ref:`Kombu 3.0.14 `. -- **Results**: - - Reliability improvements to the SQLAlchemy database backend. Previously the - connection from the MainProcess was improperly shared with the workers. - (Issue #1786) - - **Redis:** Important note about events (Issue #1882). There is a new transport option for Redis that enables monitors @@ -50,7 +44,7 @@ new in Celery 3.1. This means that the global result cache can finally be disabled, and you can do so by setting :setting:`CELERY_MAX_CACHED_RESULTS` to :const:`-1`. The lifetime of the cache will then be bound to the - lifetime of the result object, and this will be the default behavior + lifetime of the result object, which will be the default behavior in Celery 3.2. - **Events**: The "Substantial drift" warning message is now logged once @@ -76,7 +70,7 @@ new in Celery 3.1. with workers and clients not using it, so be sure to enable the option in all clients and workers if you decide to use it. -- **Multi**: With ``-opt:index`` (e.g. ``-c:1``) the index now always refers +- **Multi**: With ``-opt:index`` (e.g. :option:`-c:1`) the index now always refers to the position of a node in the argument list. This means that referring to a number will work when specifying a list @@ -114,6 +108,14 @@ new in Celery 3.1. - **Canvas**: A chord task raising an exception will now result in any errbacks (``link_error``) to the chord callback to also be called. +- **Results**: Reliability improvements to the SQLAlchemy database backend + (Issue #1786). + + Previously the connection from the ``MainProcess`` was improperly + inherited by child processes. + + Fix contributed by Ionel Cristian Mărieș. + - **Task**: Task callbacks and errbacks are now called using the group primitive. From c353c804f12efa3bd861bbc96e3972bc26be5e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ionel=20Cristian=20M=C4=83rie=C8=99?= Date: Sun, 23 Mar 2014 00:39:34 +0200 Subject: [PATCH 0022/1103] Fix issue with timer starting after stop() is called. Now it doesn't start in this situation. --- celery/utils/timer2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/utils/timer2.py b/celery/utils/timer2.py index d462c6574..e42660c23 100644 --- a/celery/utils/timer2.py +++ b/celery/utils/timer2.py @@ -86,8 +86,8 @@ def run(self): os._exit(1) def stop(self): + self._is_shutdown.set() if self.running: - self._is_shutdown.set() self._is_stopped.wait() self.join(THREAD_TIMEOUT_MAX) self.running = False From e58d4bd6c9614005651c0d3ee69b298be17dbb03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ionel=20Cristian=20M=C4=83rie=C8=99?= Date: Mon, 24 Mar 2014 14:08:54 +0200 Subject: [PATCH 0023/1103] Match the signature used in the rest of the code. --- celery/concurrency/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/concurrency/base.py b/celery/concurrency/base.py index 6b3594a96..29c348d6a 100644 --- a/celery/concurrency/base.py +++ b/celery/concurrency/base.py @@ -111,7 +111,7 @@ def on_hard_timeout(self, job): def maintain_pool(self, *args, **kwargs): pass - def terminate_job(self, pid): + def terminate_job(self, pid, signal=None): raise NotImplementedError( '{0} does not implement kill_job'.format(type(self))) From 252a31939acff6196cc983b1b1244074879845e1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 24 Mar 2014 15:46:53 +0000 Subject: [PATCH 0024/1103] Result cache populated by join_native even if cache disabled --- celery/backends/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 2ca4cc001..2a40f0dc9 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -57,8 +57,9 @@ def unpickle_backend(cls, args, kwargs): class _nulldict(dict): - def __setitem__(self, k, v): + def ignore(self, *a, **kw): pass + __setitem__ = update = setdefault = ignore class BaseBackend(object): From 0f23e858bdc61f8864720a780ac992b34b1dfda9 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 24 Mar 2014 16:00:20 +0000 Subject: [PATCH 0025/1103] Stresstests: Delete results after bigtasksbigvalue test --- funtests/stress/stress/suite.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/funtests/stress/stress/suite.py b/funtests/stress/stress/suite.py index f2fb02120..c88ec095e 100755 --- a/funtests/stress/stress/suite.py +++ b/funtests/stress/stress/suite.py @@ -246,7 +246,11 @@ def _evil_groupmember(self, evil_t, *eargs, **opts): def bigtasksbigvalue(self): g = group(any_returning.s(BIG, sleep=0.3) for i in range(8)) r = g() - self.join(r, timeout=10) + try: + self.join(r, timeout=10) + finally: + # very big values so remove results from backend + r.forget() def bigtasks(self, wait=None): self._revoketerm(wait, False, False, BIG) From 8d60d4039760084af2c711726ca81a7a31464dfe Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 25 Mar 2014 11:49:12 +0000 Subject: [PATCH 0026/1103] Task: Do not send error emails for expected errors (@task(throws=...)) --- celery/worker/job.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/celery/worker/job.py b/celery/worker/job.py index b277520e3..72946d0d3 100644 --- a/celery/worker/job.py +++ b/celery/worker/job.py @@ -454,11 +454,13 @@ def _log_error(self, einfo, send_failed_event=True): ) task = self.task if task.throws and isinstance(eobj, task.throws): - severity, exc_info = logging.INFO, None - description = 'raised expected' + do_send_mail, severity, exc_info, description = ( + False, logging.INFO, None, 'raised expected', + ) else: - severity = logging.ERROR - description = 'raised unexpected' + do_send_mail, severity, description = ( + True, logging.ERROR, 'raised unexpected', + ) format = self.error_msg if send_failed_event: self.send_event( @@ -505,7 +507,8 @@ def _log_error(self, einfo, send_failed_event=True): 'hostname': self.hostname, 'internal': internal}}) - task.send_error_email(context, einfo.exception) + if do_send_mail: + task.send_error_email(context, einfo.exception) def acknowledge(self): """Acknowledge task.""" From 84006b2405f48bf61785a608e938be13204fc204 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 26 Mar 2014 13:51:04 +0000 Subject: [PATCH 0027/1103] Removes duplicate _set_cache. Closes #1940 --- celery/result.py | 1 - 1 file changed, 1 deletion(-) diff --git a/celery/result.py b/celery/result.py index 069d8fde8..3ea837c05 100644 --- a/celery/result.py +++ b/celery/result.py @@ -328,7 +328,6 @@ def _get_task_meta(self): if meta: state = meta['status'] if state == states.SUCCESS or state in states.PROPAGATE_STATES: - self._set_cache(meta) return self._set_cache(meta) return meta return self._cache From 36bdbde7c6af6845918fb8ed7ad7c911ba628772 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 27 Mar 2014 14:01:18 +0000 Subject: [PATCH 0028/1103] Stresstests: Backend may not implement .forget --- funtests/stress/stress/suite.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/funtests/stress/stress/suite.py b/funtests/stress/stress/suite.py index c88ec095e..3d2f91cb8 100755 --- a/funtests/stress/stress/suite.py +++ b/funtests/stress/stress/suite.py @@ -250,7 +250,10 @@ def bigtasksbigvalue(self): self.join(r, timeout=10) finally: # very big values so remove results from backend - r.forget() + try: + r.forget() + except NotImplementedError: + pass def bigtasks(self, wait=None): self._revoketerm(wait, False, False, BIG) From 61906fdde36b9014917c3b0f5e7cf55e3822b24a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 27 Mar 2014 14:08:26 +0000 Subject: [PATCH 0029/1103] Master branch is now 3.2a1 --- README.rst | 2 +- celery/__init__.py | 4 ++-- docs/includes/introduction.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 2ec057fce..8e349b866 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ .. image:: http://cloud.github.com/downloads/celery/celery/celery_128.png -:Version: 3.1.10 (Cipater) +:Version: 3.2.0a1 (Cipater) :Web: http://celeryproject.org/ :Download: http://pypi.python.org/pypi/celery/ :Source: http://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index 489b6b4e1..848907cf3 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -13,8 +13,8 @@ 'version_info_t', ('major', 'minor', 'micro', 'releaselevel', 'serial'), ) -SERIES = 'Cipater' -VERSION = version_info_t(3, 1, 10, '', '') +SERIES = 'DEV' +VERSION = version_info_t(3, 2, 0, 'a1', '') __version__ = '{0.major}.{0.minor}.{0.micro}{0.releaselevel}'.format(VERSION) __author__ = 'Ask Solem' __contact__ = 'ask@celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index c96304ff1..4cbc2627f 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 3.1.10 (Cipater) +:Version: 3.2.0a1 (Cipater) :Web: http://celeryproject.org/ :Download: http://pypi.python.org/pypi/celery/ :Source: http://github.com/celery/celery/ From 59ece7d5f21f1ccc67263053c641c03648baebad Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 27 Mar 2014 14:58:47 +0000 Subject: [PATCH 0030/1103] Stresstests: Refactors Suite into BaseSuite and Suite --- funtests/stress/stress/suite.py | 164 +++++++++++++++++++------------- 1 file changed, 98 insertions(+), 66 deletions(-) diff --git a/funtests/stress/stress/suite.py b/funtests/stress/stress/suite.py index 3d2f91cb8..10d56f4cf 100755 --- a/funtests/stress/stress/suite.py +++ b/funtests/stress/stress/suite.py @@ -1,11 +1,12 @@ from __future__ import absolute_import, print_function, unicode_literals +import inspect import platform import random import socket import sys -from collections import namedtuple +from collections import defaultdict, namedtuple from itertools import count from time import sleep @@ -13,14 +14,14 @@ from celery import group, VERSION_BANNER from celery.exceptions import TimeoutError -from celery.five import range, values, monotonic +from celery.five import items, monotonic, range, values from celery.utils.debug import blockdetection from celery.utils.text import pluralize, truncate from celery.utils.timeutils import humanize_seconds from .app import ( marker, _marker, add, any_, exiting, kill, sleeping, - sleeping_ignore_limits, segfault, any_returning, + sleeping_ignore_limits, any_returning, ) from .data import BIG, SMALL from .fbi import FBI @@ -83,7 +84,7 @@ def testgroup(*funs): return OrderedDict((fun.__name__, fun) for fun in funs) -class Suite(object): +class BaseSuite(object): def __init__(self, app, block_timeout=30 * 60): self.app = app @@ -92,30 +93,26 @@ def __init__(self, app, block_timeout=30 * 60): self.progress = None self.speaker = Speaker() self.fbi = FBI(app) + self.init_groups() - self.groups = { - 'all': testgroup( - self.manyshort, - self.termbysig, - self.bigtasks, - self.bigtasksbigvalue, - self.smalltasks, - self.timelimits, - self.timelimits_soft, - self.revoketermfast, - self.revoketermslow, - self.alwayskilled, - self.alwaysexits, - ), - 'green': testgroup( - self.manyshort, - self.bigtasks, - self.bigtasksbigvalue, - self.smalltasks, - self.alwaysexits, - self.group_with_exit, - ), - } + def init_groups(self): + acc = defaultdict(list) + for attr in dir(self): + if not _is_descriptor(self, attr): + meth = getattr(self, attr) + try: + groups = meth.__func__.__testgroup__ + except AttributeError: + pass + else: + for group in groups: + acc[group].append(meth) + # sort the tests by the order in which they are defined in the class + for group in values(acc): + group[:] = sorted(group, key=lambda m: m.__func__.__testsort__) + self.groups = dict( + (name, testgroup(*tests)) for name, tests in items(acc) + ) def run(self, names=None, iterations=50, offset=0, numtests=None, list_all=False, repeat=0, group='all', @@ -167,10 +164,6 @@ def banner(self, tests): total=len(tests), ) - def manyshort(self): - self.join(group(add.s(i, i) for i in range(1000))(), - timeout=10, propagate=True) - def runtest(self, fun, n=50, index=0, repeats=1): print('{0}: [[[{1}({2})]]]'.format(repeats, fun.__name__, n)) with blockdetection(self.block_timeout): @@ -211,26 +204,92 @@ def runtest(self, fun, n=50, index=0, repeats=1): fun, i + 1, n, index, repeats, runtime, elapsed, 1, ) + def missing_results(self, r): + return [res.id for res in r if res.id not in res.backend._cache] + + def join(self, r, propagate=False, max_retries=10, **kwargs): + if self.no_join: + return + received = [] + + def on_result(task_id, value): + received.append(task_id) + + for i in range(max_retries) if max_retries else count(0): + received[:] = [] + try: + return r.get(callback=on_result, propagate=propagate, **kwargs) + except (socket.timeout, TimeoutError) as exc: + waiting_for = self.missing_results(r) + self.speaker.beep() + marker( + 'Still waiting for {0}/{1}: [{2}]: {3!r}'.format( + len(r) - len(received), len(r), + truncate(', '.join(waiting_for)), exc), '!', + ) + self.fbi.diag(waiting_for) + except self.connerrors as exc: + self.speaker.beep() + marker('join: connection lost: {0!r}'.format(exc), '!') + raise StopSuite('Test failed: Missing task results') + + def dump_progress(self): + return pstatus(self.progress) if self.progress else 'No test running' + + +_creation_counter = count(0) +def testcase(*groups): + if not groups: + raise ValueError('@testcase requires at least one group name') + + def _mark_as_case(fun): + fun.__testgroup__ = groups + fun.__testsort__ = next(_creation_counter) + return fun + + return _mark_as_case + + +def _is_descriptor(obj, attr): + try: + cattr = getattr(obj.__class__, attr) + except AttributeError: + pass + else: + return not inspect.ismethod(cattr) and hasattr(cattr, '__get__') + return False + + +class Suite(BaseSuite): + + @testcase('all', 'green') + def manyshort(self): + self.join(group(add.s(i, i) for i in range(1000))(), + timeout=10, propagate=True) + + @testcase('all') def termbysig(self): self._evil_groupmember(kill) + @testcase('green') def group_with_exit(self): self._evil_groupmember(exiting) - def termbysegfault(self): - self._evil_groupmember(segfault) - + @testcase('all') def timelimits(self): self._evil_groupmember(sleeping, 2, time_limit=1) + @testcase('all') def timelimits_soft(self): self._evil_groupmember(sleeping_ignore_limits, 2, soft_time_limit=1, time_limit=1.1) + @testcase('all') def alwayskilled(self): g = group(kill.s() for _ in range(10)) self.join(g(), timeout=10) + @testcase('all', 'green') def alwaysexits(self): g = group(exiting.s() for _ in range(10)) self.join(g(), timeout=10) @@ -243,6 +302,7 @@ def _evil_groupmember(self, evil_t, *eargs, **opts): self.join(g1(), timeout=10) self.join(g2(), timeout=10) + @testcase('all', 'green') def bigtasksbigvalue(self): g = group(any_returning.s(BIG, sleep=0.3) for i in range(8)) r = g() @@ -255,15 +315,19 @@ def bigtasksbigvalue(self): except NotImplementedError: pass + @testcase('all', 'green') def bigtasks(self, wait=None): self._revoketerm(wait, False, False, BIG) + @testcase('all', 'green') def smalltasks(self, wait=None): self._revoketerm(wait, False, False, SMALL) + @testcase('all') def revoketermfast(self, wait=None): self._revoketerm(wait, True, False, SMALL) + @testcase('all') def revoketermslow(self, wait=5): self._revoketerm(wait, True, True, BIG) @@ -276,35 +340,3 @@ def _revoketerm(self, wait=None, terminate=True, sleep(random.choice(range(4))) r.revoke(terminate=True) self.join(r, timeout=10) - - def missing_results(self, r): - return [res.id for res in r if res.id not in res.backend._cache] - - def join(self, r, propagate=False, max_retries=10, **kwargs): - if self.no_join: - return - received = [] - - def on_result(task_id, value): - received.append(task_id) - - for i in range(max_retries) if max_retries else count(0): - received[:] = [] - try: - return r.get(callback=on_result, propagate=propagate, **kwargs) - except (socket.timeout, TimeoutError) as exc: - waiting_for = self.missing_results(r) - self.speaker.beep() - marker( - 'Still waiting for {0}/{1}: [{2}]: {3!r}'.format( - len(r) - len(received), len(r), - truncate(', '.join(waiting_for)), exc), '!', - ) - self.fbi.diag(waiting_for) - except self.connerrors as exc: - self.speaker.beep() - marker('join: connection lost: {0!r}'.format(exc), '!') - raise StopSuite('Test failed: Missing task results') - - def dump_progress(self): - return pstatus(self.progress) if self.progress else 'No test running' From 8b7e3f2e9a7e91b055885e14b90ccb0167871b98 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 27 Mar 2014 15:36:42 +0000 Subject: [PATCH 0031/1103] TaskProducer replaced by create_task_message and send_task_message --- celery/app/amqp.py | 411 +++++++++++------------- celery/app/base.py | 25 +- celery/app/task.py | 2 +- celery/beat.py | 2 +- celery/contrib/migrate.py | 4 +- celery/five.py | 1 - celery/task/base.py | 15 +- celery/task/sets.py | 2 +- celery/tests/app/test_amqp.py | 96 +----- celery/tests/app/test_app.py | 35 +- celery/tests/backends/test_amqp.py | 6 +- celery/tests/tasks/test_tasks.py | 5 - docs/reference/celery.app.amqp.rst | 19 +- examples/eventlet/bulk_task_producer.py | 26 +- 14 files changed, 267 insertions(+), 382 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index b0dae95e0..c5b253396 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -10,13 +10,14 @@ import numbers +from collections import Mapping, namedtuple from datetime import timedelta from weakref import WeakValueDictionary from kombu import Connection, Consumer, Exchange, Producer, Queue from kombu.common import Broadcast from kombu.pools import ProducerPool -from kombu.utils import cached_property, uuid +from kombu.utils import cached_property from kombu.utils.encoding import safe_repr from kombu.utils.functional import maybe_list @@ -25,10 +26,9 @@ from celery.utils.text import indent as textindent from celery.utils.timeutils import to_utc -from . import app_or_default from . import routes as _routes -__all__ = ['AMQP', 'Queues', 'TaskProducer', 'TaskConsumer'] +__all__ = ['AMQP', 'Queues', 'task_message'] #: Human readable queue declaration. QUEUE_FORMAT = """ @@ -36,6 +36,9 @@ key={0.routing_key} """ +task_message = namedtuple('task_message', + ('headers', 'properties', 'body', 'sent_event')) + class Queues(dict): """Queue name⇒ declaration mapping. @@ -184,204 +187,14 @@ def consume_from(self): return self -class TaskProducer(Producer): - app = None - auto_declare = False - retry = False - retry_policy = None - utc = True - event_dispatcher = None - send_sent_event = False - - def __init__(self, channel=None, exchange=None, *args, **kwargs): - self.retry = kwargs.pop('retry', self.retry) - self.retry_policy = kwargs.pop('retry_policy', - self.retry_policy or {}) - self.send_sent_event = kwargs.pop('send_sent_event', - self.send_sent_event) - exchange = exchange or self.exchange - self.queues = self.app.amqp.queues # shortcut - self.default_queue = self.app.amqp.default_queue - super(TaskProducer, self).__init__(channel, exchange, *args, **kwargs) - - def publish_task(self, task_name, task_args=None, task_kwargs=None, - countdown=None, eta=None, task_id=None, group_id=None, - taskset_id=None, # compat alias to group_id - expires=None, exchange=None, exchange_type=None, - event_dispatcher=None, retry=None, retry_policy=None, - queue=None, now=None, retries=0, chord=None, - callbacks=None, errbacks=None, routing_key=None, - serializer=None, delivery_mode=None, compression=None, - reply_to=None, time_limit=None, soft_time_limit=None, - declare=None, headers=None, - send_before_publish=signals.before_task_publish.send, - before_receivers=signals.before_task_publish.receivers, - send_after_publish=signals.after_task_publish.send, - after_receivers=signals.after_task_publish.receivers, - send_task_sent=signals.task_sent.send, # XXX deprecated - sent_receivers=signals.task_sent.receivers, - **kwargs): - """Send task message.""" - retry = self.retry if retry is None else retry - headers = {} if headers is None else headers - - qname = queue - if queue is None and exchange is None: - queue = self.default_queue - if queue is not None: - if isinstance(queue, string_t): - qname, queue = queue, self.queues[queue] - else: - qname = queue.name - exchange = exchange or queue.exchange.name - routing_key = routing_key or queue.routing_key - if declare is None and queue and not isinstance(queue, Broadcast): - declare = [queue] - - # merge default and custom policy - retry = self.retry if retry is None else retry - _rp = (dict(self.retry_policy, **retry_policy) if retry_policy - else self.retry_policy) - task_id = task_id or uuid() - task_args = task_args or [] - task_kwargs = task_kwargs or {} - if not isinstance(task_args, (list, tuple)): - raise ValueError('task args must be a list or tuple') - if not isinstance(task_kwargs, dict): - raise ValueError('task kwargs must be a dictionary') - if countdown: # Convert countdown to ETA. - now = now or self.app.now() - eta = now + timedelta(seconds=countdown) - if self.utc: - eta = to_utc(eta).astimezone(self.app.timezone) - if isinstance(expires, numbers.Real): - now = now or self.app.now() - expires = now + timedelta(seconds=expires) - if self.utc: - expires = to_utc(expires).astimezone(self.app.timezone) - eta = eta and eta.isoformat() - expires = expires and expires.isoformat() - - body = { - 'task': task_name, - 'id': task_id, - 'args': task_args, - 'kwargs': task_kwargs, - 'retries': retries or 0, - 'eta': eta, - 'expires': expires, - 'utc': self.utc, - 'callbacks': callbacks, - 'errbacks': errbacks, - 'timelimit': (time_limit, soft_time_limit), - 'taskset': group_id or taskset_id, - 'chord': chord, - } - - if before_receivers: - send_before_publish( - sender=task_name, body=body, - exchange=exchange, - routing_key=routing_key, - declare=declare, - headers=headers, - properties=kwargs, - retry_policy=retry_policy, - ) - - self.publish( - body, - exchange=exchange, routing_key=routing_key, - serializer=serializer or self.serializer, - compression=compression or self.compression, - headers=headers, - retry=retry, retry_policy=_rp, - reply_to=reply_to, - correlation_id=task_id, - delivery_mode=delivery_mode, declare=declare, - **kwargs - ) - - if after_receivers: - send_after_publish(sender=task_name, body=body, - exchange=exchange, routing_key=routing_key) - - if sent_receivers: # XXX deprecated - send_task_sent(sender=task_name, task_id=task_id, - task=task_name, args=task_args, - kwargs=task_kwargs, eta=eta, - taskset=group_id or taskset_id) - if self.send_sent_event: - evd = event_dispatcher or self.event_dispatcher - exname = exchange or self.exchange - if isinstance(exname, Exchange): - exname = exname.name - evd.publish( - 'task-sent', - { - 'uuid': task_id, - 'name': task_name, - 'args': safe_repr(task_args), - 'kwargs': safe_repr(task_kwargs), - 'retries': retries, - 'eta': eta, - 'expires': expires, - 'queue': qname, - 'exchange': exname, - 'routing_key': routing_key, - }, - self, retry=retry, retry_policy=retry_policy, - ) - return task_id - delay_task = publish_task # XXX Compat - - @cached_property - def event_dispatcher(self): - # We call Dispatcher.publish with a custom producer - # so don't need the dispatcher to be "enabled". - return self.app.events.Dispatcher(enabled=False) - - -class TaskPublisher(TaskProducer): - """Deprecated version of :class:`TaskProducer`.""" - - def __init__(self, channel=None, exchange=None, *args, **kwargs): - self.app = app_or_default(kwargs.pop('app', self.app)) - self.retry = kwargs.pop('retry', self.retry) - self.retry_policy = kwargs.pop('retry_policy', - self.retry_policy or {}) - exchange = exchange or self.exchange - if not isinstance(exchange, Exchange): - exchange = Exchange(exchange, - kwargs.pop('exchange_type', 'direct')) - self.queues = self.app.amqp.queues # shortcut - super(TaskPublisher, self).__init__(channel, exchange, *args, **kwargs) - - -class TaskConsumer(Consumer): - app = None - - def __init__(self, channel, queues=None, app=None, accept=None, **kw): - self.app = app or self.app - if accept is None: - accept = self.app.conf.CELERY_ACCEPT_CONTENT - super(TaskConsumer, self).__init__( - channel, - queues or list(self.app.amqp.queues.consume_from.values()), - accept=accept, - **kw - ) - - class AMQP(object): Connection = Connection Consumer = Consumer + Producer = Producer #: compat alias to Connection BrokerConnection = Connection - producer_cls = TaskProducer - consumer_cls = TaskConsumer queues_cls = Queues #: Cached and prepared routing table. @@ -400,6 +213,18 @@ class AMQP(object): def __init__(self, app): self.app = app + @cached_property + def _task_retry(self): + return self.app.conf.CELERY_TASK_PUBLISH_RETRY + + @cached_property + def _task_retry_policy(self): + return self.app.conf.CELERY_TASK_PUBLISH_RETRY_POLICY + + @cached_property + def _task_sent_event(self): + return self.app.conf.CELERY_SEND_TASK_SENT_EVENT + def flush_routes(self): self._rtable = _routes.prepare(self.app.conf.CELERY_ROUTES) @@ -429,35 +254,14 @@ def Router(self, queues=None, create_missing=None): self.app.either('CELERY_CREATE_MISSING_QUEUES', create_missing), app=self.app) - @cached_property - def TaskConsumer(self): - """Return consumer configured to consume from the queues - we are configured for (``app.amqp.queues.consume_from``).""" - return self.app.subclass_with_self(self.consumer_cls, - reverse='amqp.TaskConsumer') - get_task_consumer = TaskConsumer # XXX compat - - @cached_property - def TaskProducer(self): - """Return publisher used to send tasks. - - You should use `app.send_task` instead. - - """ - conf = self.app.conf - return self.app.subclass_with_self( - self.producer_cls, - reverse='amqp.TaskProducer', - exchange=self.default_exchange, - routing_key=conf.CELERY_DEFAULT_ROUTING_KEY, - serializer=conf.CELERY_TASK_SERIALIZER, - compression=conf.CELERY_MESSAGE_COMPRESSION, - retry=conf.CELERY_TASK_PUBLISH_RETRY, - retry_policy=conf.CELERY_TASK_PUBLISH_RETRY_POLICY, - send_sent_event=conf.CELERY_SEND_TASK_SENT_EVENT, - utc=conf.CELERY_ENABLE_UTC, + def TaskConsumer(self, channel, queues=None, accept=None, **kw): + if accept is None: + accept = self.app.conf.CELERY_ACCEPT_CONTENT + return self.Consumer( + channel, accept=accept, + queues=queues or list(self.queues.consume_from.values()), + **kw ) - TaskPublisher = TaskProducer # compat @cached_property def default_queue(self): @@ -488,7 +292,7 @@ def producer_pool(self): self._producer_pool = ProducerPool( self.app.pool, limit=self.app.pool.limit, - Producer=self.TaskProducer, + Producer=self.Producer, ) return self._producer_pool publisher_pool = producer_pool # compat alias @@ -497,3 +301,164 @@ def producer_pool(self): def default_exchange(self): return Exchange(self.app.conf.CELERY_DEFAULT_EXCHANGE, self.app.conf.CELERY_DEFAULT_EXCHANGE_TYPE) + + def create_task_message(self, task_id, name, args=None, kwargs=None, + countdown=None, eta=None, group_id=None, + expires=None, now=None, retries=0, chord=None, + callbacks=None, errbacks=None, reply_to=None, + time_limit=None, soft_time_limit=None, + create_sent_event=False): + args = args or () + kwargs = kwargs or {} + utc = self.utc + if not isinstance(args, (list, tuple)): + raise ValueError('task args must be a list or tuple') + if not isinstance(kwargs, Mapping): + raise ValueError('task keyword arguments must be a mapping') + if countdown: # convert countdown to ETA + now = now or self.app.now() + eta = now + timedelta(seconds=countdown) + if utc: + eta = to_utc(eta).astimezone(self.app.timezone) + if isinstance(expires, numbers.Real): + now = now or self.app.now() + expires = now + timedelta(seconds=expires) + if utc: + expires = to_utc(expires).astimezone(self.app.timezone) + eta = eta and eta.isoformat() + expires = expires and expires.isoformat() + + return task_message( + {}, + { + 'correlation_id': task_id, + 'reply_to': reply_to, + }, + { + 'task': name, + 'id': task_id, + 'args': args, + 'kwargs': kwargs, + 'retries': retries, + 'eta': eta, + 'expires': expires, + 'utc': utc, + 'callbacks': callbacks, + 'errbacks': errbacks, + 'timelimit': (time_limit, soft_time_limit), + 'taskset': group_id, + 'chord': chord, + }, + { + 'uuid': task_id, + 'name': name, + 'args': safe_repr(args), + 'kwargs': safe_repr(kwargs), + 'retries': retries, + 'eta': eta, + 'expires': expires, + } if create_sent_event else None, + ) + + def _create_task_sender(self): + default_retry = self.app.conf.CELERY_TASK_PUBLISH_RETRY + default_policy = self.app.conf.CELERY_TASK_PUBLISH_RETRY_POLICY + default_queue = self.default_queue + queues = self.queues + send_before_publish = signals.before_task_publish.send + before_receivers = signals.before_task_publish.receivers + send_after_publish = signals.after_task_publish.send + after_receivers = signals.after_task_publish.receivers + + send_task_sent = signals.task_sent.send # XXX compat + sent_receivers = signals.task_sent.receivers + + default_evd = self._event_dispatcher + default_exchange = self.default_exchange + + default_rkey = self.app.conf.CELERY_DEFAULT_ROUTING_KEY + default_serializer = self.app.conf.CELERY_TASK_SERIALIZER + default_compressor = self.app.conf.CELERY_MESSAGE_COMPRESSION + + def publish_task(producer, name, message, + exchange=None, routing_key=None, queue=None, + event_dispatcher=None, retry=None, retry_policy=None, + serializer=None, delivery_mode=None, + compression=None, declare=None, + headers=None, **kwargs): + retry = default_retry if retry is None else retry + headers, properties, body, sent_event = message + if kwargs: + properties.update(kwargs) + + qname = queue + if queue is None and exchange is None: + queue = default_queue + if queue is not None: + if isinstance(queue, string_t): + qname, queue = queue, queues[queue] + else: + qname = queue.name + exchange = exchange or queue.exchange.name + routing_key = routing_key or queue.routing_key + if declare is None and queue and not isinstance(queue, Broadcast): + declare = [queue] + + # merge default and custom policy + retry = default_retry if retry is None else retry + _rp = (dict(default_policy, **retry_policy) if retry_policy + else default_policy) + + if before_receivers: + send_before_publish( + sender=name, body=body, + exchange=exchange, routing_key=routing_key, + declare=declare, headers=headers, + properties=kwargs, retry_policy=retry_policy, + ) + ret = producer.publish( + body, + exchange=exchange or default_exchange, + routing_key=routing_key or default_rkey, + serializer=serializer or default_serializer, + compression=compression or default_compressor, + retry=retry, retry_policy=_rp, + delivery_mode=delivery_mode, declare=declare, + headers=headers, + **properties + ) + if after_receivers: + send_after_publish(sender=name, body=body, + exchange=exchange, routing_key=routing_key) + if sent_receivers: # XXX deprecated + send_task_sent(sender=name, task_id=body['id'], task=name, + args=body['args'], kwargs=body['kwargs'], + eta=body['eta'], taskset=body['taskset']) + if sent_event: + evd = event_dispatcher or default_evd + exname = exchange or self.exchange + if isinstance(name, Exchange): + exname = exname.name + sent_event.update({ + 'queue': qname, + 'exchange': exname, + 'routing_key': routing_key, + }) + evd.publish('task-sent', sent_event, + self, retry=retry, retry_policy=retry_policy) + return ret + return publish_task + + @cached_property + def send_task_message(self): + return self._create_task_sender() + + @cached_property + def utc(self): + return self.app.conf.CELERY_ENABLE_UTC + + @cached_property + def _event_dispatcher(self): + # We call Dispatcher.publish with a custom producer + # so don't need the diuspatcher to be enabled. + return self.app.events.Dispatcher(enabled=False) diff --git a/celery/app/base.py b/celery/app/base.py index 153a5575a..c934a7a94 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -302,26 +302,33 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, eta=None, task_id=None, producer=None, connection=None, router=None, result_cls=None, expires=None, publisher=None, link=None, link_error=None, - add_to_parent=True, reply_to=None, **options): + add_to_parent=True, group_id=None, retries=0, chord=None, + reply_to=None, time_limit=None, soft_time_limit=None, + **options): + amqp = self.amqp task_id = task_id or uuid() producer = producer or publisher # XXX compat - router = router or self.amqp.router + router = router or amqp.router conf = self.conf if conf.CELERY_ALWAYS_EAGER: # pragma: no cover warnings.warn(AlwaysEagerIgnored( 'CELERY_ALWAYS_EAGER has no effect on send_task', ), stacklevel=2) options = router.route(options, name, args, kwargs) + + message = amqp.create_task_message( + task_id, name, args, kwargs, countdown, eta, group_id, + expires, retries, chord, + maybe_list(link), maybe_list(link_error), + reply_to or self.oid, time_limit, soft_time_limit, + self.conf.CELERY_SEND_TASK_SENT_EVENT, + ) + if connection: - producer = self.amqp.TaskProducer(connection) + producer = amqp.Producer(connection) with self.producer_or_acquire(producer) as P: self.backend.on_task_call(P, task_id) - task_id = P.publish_task( - name, args, kwargs, countdown=countdown, eta=eta, - task_id=task_id, expires=expires, - callbacks=maybe_list(link), errbacks=maybe_list(link_error), - reply_to=reply_to or self.oid, **options - ) + amqp.send_task_message(P, name, message, **options) result = (result_cls or self.AsyncResult)(task_id) if add_to_parent: parent = get_current_worker_task() diff --git a/celery/app/task.py b/celery/app/task.py index 79f6d3fca..dcb32a34d 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -525,7 +525,7 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, :keyword link_error: A single, or a list of tasks to apply if an error occurs while executing the task. - :keyword producer: :class:~@amqp.TaskProducer` instance to use. + :keyword producer: :class:~@kombu.Producer` instance to use. :keyword add_to_parent: If set to True (default) and the task is applied while executing another task, then the result will be appended to the parent tasks ``request.children`` diff --git a/celery/beat.py b/celery/beat.py index 8205c2781..0b2ec97f2 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -179,7 +179,7 @@ def __init__(self, app, schedule=None, max_interval=None, self.sync_every_tasks = ( app.conf.CELERYBEAT_SYNC_EVERY if sync_every_tasks is None else sync_every_tasks) - self.Publisher = Publisher or app.amqp.TaskProducer + self.Publisher = Publisher or app.amqp.Producer if not lazy: self.setup_schedule() diff --git a/celery/contrib/migrate.py b/celery/contrib/migrate.py index e4a10e9b9..c50fba2c7 100644 --- a/celery/contrib/migrate.py +++ b/celery/contrib/migrate.py @@ -99,7 +99,7 @@ def migrate_tasks(source, dest, migrate=migrate_task, app=None, queues=None, **kwargs): app = app_or_default(app) queues = prepare_queues(queues) - producer = app.amqp.TaskProducer(dest) + producer = app.amqp.Producer(dest) migrate = partial(migrate, producer, queues=queues) def on_declare_queue(queue): @@ -186,7 +186,7 @@ def transform(value): app = app_or_default(app) queues = [_maybe_queue(app, queue) for queue in source or []] or None with app.connection_or_acquire(connection, pool=False) as conn: - producer = app.amqp.TaskProducer(conn) + producer = app.amqp.Producer(conn) state = State() def on_task(body, message): diff --git a/celery/five.py b/celery/five.py index dfee614e4..77ec1daa5 100644 --- a/celery/five.py +++ b/celery/five.py @@ -238,7 +238,6 @@ def _compat_periodic_task_decorator(*args, **kwargs): 'redirect_stdouts_to_logger': 'log.redirect_stdouts_to_logger', }, 'messaging': { - 'TaskPublisher': 'amqp.TaskPublisher', 'TaskConsumer': 'amqp.TaskConsumer', 'establish_connection': 'connection', 'get_consumer_set': 'amqp.TaskConsumer', diff --git a/celery/task/base.py b/celery/task/base.py index 9d466b57c..9e12d4f8c 100644 --- a/celery/task/base.py +++ b/celery/task/base.py @@ -106,12 +106,19 @@ def get_publisher(self, connection=None, exchange=None, exchange_type=None, **options): """Deprecated method to get the task publisher (now called producer). - Should be replaced with :class:`@amqp.TaskProducer`: + Should be replaced with :class:`@kombu.Producer`: .. code-block:: python - with celery.connection() as conn: - with celery.amqp.TaskProducer(conn) as prod: + with app.connection() as conn: + with app.amqp.Producer(conn) as prod: + my_task.apply_async(producer=prod) + + or event better is to use the :class:`@amqp.producer_pool`: + + .. code-block:: python + + with app.producer_or_acquire() as prod: my_task.apply_async(producer=prod) """ @@ -119,7 +126,7 @@ def get_publisher(self, connection=None, exchange=None, if exchange_type is None: exchange_type = self.exchange_type connection = connection or self.establish_connection() - return self._get_app().amqp.TaskProducer( + return self._get_app().amqp.Producer( connection, exchange=exchange and Exchange(exchange, exchange_type), routing_key=self.routing_key, **options diff --git a/celery/task/sets.py b/celery/task/sets.py index e277b796d..7d4355f62 100644 --- a/celery/task/sets.py +++ b/celery/task/sets.py @@ -46,7 +46,7 @@ def __init__(self, tasks=None, app=None, Publisher=None): super(TaskSet, self).__init__( maybe_signature(t, app=self.app) for t in tasks or [] ) - self.Publisher = Publisher or self.app.amqp.TaskProducer + self.Publisher = Publisher or self.app.amqp.Producer self.total = len(self) # XXX compat def apply_async(self, connection=None, publisher=None, taskset_id=None): diff --git a/celery/tests/app/test_amqp.py b/celery/tests/app/test_amqp.py index efb398ac6..cf2810d5b 100644 --- a/celery/tests/app/test_amqp.py +++ b/celery/tests/app/test_amqp.py @@ -1,86 +1,10 @@ from __future__ import absolute_import -import datetime - -import pytz - from kombu import Exchange, Queue -from celery.app.amqp import Queues, TaskPublisher +from celery.app.amqp import Queues from celery.five import keys -from celery.tests.case import AppCase, Mock - - -class test_TaskProducer(AppCase): - - def test__exit__(self): - publisher = self.app.amqp.TaskProducer(self.app.connection()) - publisher.release = Mock() - with publisher: - pass - publisher.release.assert_called_with() - - def test_declare(self): - publisher = self.app.amqp.TaskProducer(self.app.connection()) - publisher.exchange.name = 'foo' - publisher.declare() - publisher.exchange.name = None - publisher.declare() - - def test_retry_policy(self): - prod = self.app.amqp.TaskProducer(Mock()) - prod.channel.connection.client.declared_entities = set() - prod.publish_task('tasks.add', (2, 2), {}, - retry_policy={'frobulate': 32.4}) - - def test_publish_no_retry(self): - prod = self.app.amqp.TaskProducer(Mock()) - prod.channel.connection.client.declared_entities = set() - prod.publish_task('tasks.add', (2, 2), {}, retry=False, chord=123) - self.assertFalse(prod.connection.ensure.call_count) - - def test_publish_custom_queue(self): - prod = self.app.amqp.TaskProducer(Mock()) - self.app.amqp.queues['some_queue'] = Queue( - 'xxx', Exchange('yyy'), 'zzz', - ) - prod.channel.connection.client.declared_entities = set() - prod.publish = Mock() - prod.publish_task('tasks.add', (8, 8), {}, retry=False, - queue='some_queue') - self.assertEqual(prod.publish.call_args[1]['exchange'], 'yyy') - self.assertEqual(prod.publish.call_args[1]['routing_key'], 'zzz') - - def test_publish_with_countdown(self): - prod = self.app.amqp.TaskProducer(Mock()) - prod.channel.connection.client.declared_entities = set() - prod.publish = Mock() - now = datetime.datetime(2013, 11, 26, 16, 48, 46) - prod.publish_task('tasks.add', (1, 1), {}, retry=False, - countdown=10, now=now) - self.assertEqual( - prod.publish.call_args[0][0]['eta'], - '2013-11-26T16:48:56+00:00', - ) - - def test_publish_with_countdown_and_timezone(self): - # use timezone with fixed offset to be sure it won't be changed - self.app.conf.CELERY_TIMEZONE = pytz.FixedOffset(120) - prod = self.app.amqp.TaskProducer(Mock()) - prod.channel.connection.client.declared_entities = set() - prod.publish = Mock() - now = datetime.datetime(2013, 11, 26, 16, 48, 46) - prod.publish_task('tasks.add', (2, 2), {}, retry=False, - countdown=20, now=now) - self.assertEqual( - prod.publish.call_args[0][0]['eta'], - '2013-11-26T18:49:06+02:00', - ) - - def test_event_dispatcher(self): - prod = self.app.amqp.TaskProducer(Mock()) - self.assertTrue(prod.event_dispatcher) - self.assertFalse(prod.event_dispatcher.enabled) +from celery.tests.case import AppCase class test_TaskConsumer(AppCase): @@ -98,22 +22,6 @@ def test_accept_content(self): ) -class test_compat_TaskPublisher(AppCase): - - def test_compat_exchange_is_string(self): - producer = TaskPublisher(exchange='foo', app=self.app) - self.assertIsInstance(producer.exchange, Exchange) - self.assertEqual(producer.exchange.name, 'foo') - self.assertEqual(producer.exchange.type, 'direct') - producer = TaskPublisher(exchange='foo', exchange_type='topic', - app=self.app) - self.assertEqual(producer.exchange.type, 'topic') - - def test_compat_exchange_is_Exchange(self): - producer = TaskPublisher(exchange=Exchange('foo'), app=self.app) - self.assertEqual(producer.exchange.name, 'foo') - - class test_PublisherPool(AppCase): def test_setup_nolimit(self): diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index 113dedae1..36cdb67a2 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -8,7 +8,6 @@ from pickle import loads, dumps from amqp import promise -from kombu import Exchange from celery import shared_task, current_app from celery import app as _app @@ -336,10 +335,13 @@ def test_apply_async_has__self__(self): def aawsX(): pass - with patch('celery.app.amqp.TaskProducer.publish_task') as dt: - aawsX.apply_async((4, 5)) - args = dt.call_args[0][1] - self.assertEqual(args, ('hello', 4, 5)) + with patch('celery.app.amqp.AMQP.create_task_message') as create: + with patch('celery.app.amqp.AMQP.send_task_message') as send: + create.return_value = Mock(), Mock(), Mock(), Mock() + aawsX.apply_async((4, 5)) + args = create.call_args[0][2] + self.assertEqual(args, ('hello', 4, 5)) + self.assertTrue(send.called) def test_apply_async_adds_children(self): from celery._state import _task_stack @@ -609,22 +611,23 @@ def publish(self, type, fields, *args, **kwargs): chan.close() assert conn.transport_cls == 'memory' - prod = self.app.amqp.TaskProducer( - conn, exchange=Exchange('foo_exchange'), - send_sent_event=True, + message = self.app.amqp.create_task_message( + 'id', 'footask', (), {}, create_sent_event=True, ) + prod = self.app.amqp.Producer(conn) dispatcher = Dispatcher() - self.assertTrue(prod.publish_task('footask', (), {}, - exchange='moo_exchange', - routing_key='moo_exchange', - event_dispatcher=dispatcher)) + self.app.amqp.send_task_message( + prod, 'footask', message, + exchange='moo_exchange', routing_key='moo_exchange', + event_dispatcher=dispatcher, + ) self.assertTrue(dispatcher.sent) self.assertEqual(dispatcher.sent[0][0], 'task-sent') - self.assertTrue(prod.publish_task('footask', (), {}, - event_dispatcher=dispatcher, - exchange='bar_exchange', - routing_key='bar_exchange')) + self.app.amqp.send_task_message( + prod, 'footask', message, event_dispatcher=dispatcher, + exchange='bar_exchange', routing_key='bar_exchange', + ) def test_error_mail_sender(self): x = ErrorMail.subject % {'name': 'task_name', diff --git a/celery/tests/backends/test_amqp.py b/celery/tests/backends/test_amqp.py index 7e5a36196..23a6c46d7 100644 --- a/celery/tests/backends/test_amqp.py +++ b/celery/tests/backends/test_amqp.py @@ -108,8 +108,8 @@ def publish(*args, **kwargs): raise KeyError('foo') backend = AMQPBackend(self.app) - from celery.app.amqp import TaskProducer - prod, TaskProducer.publish = TaskProducer.publish, publish + from celery.app.amqp import Producer + prod, Producer.publish = Producer.publish, publish try: with self.assertRaises(KeyError): backend.retry_policy['max_retries'] = None @@ -119,7 +119,7 @@ def publish(*args, **kwargs): backend.retry_policy['max_retries'] = 10 backend.store_result('foo', 'bar', 'STARTED') finally: - TaskProducer.publish = prod + Producer.publish = prod def assertState(self, retval, state): self.assertEqual(retval['status'], state) diff --git a/celery/tests/tasks/test_tasks.py b/celery/tests/tasks/test_tasks.py index 93a782ecc..c01ffc16a 100644 --- a/celery/tests/tasks/test_tasks.py +++ b/celery/tests/tasks/test_tasks.py @@ -381,11 +381,6 @@ def test_after_return(self): finally: self.mytask.pop_request() - def test_send_task_sent_event(self): - with self.app.connection() as conn: - self.app.conf.CELERY_SEND_TASK_SENT_EVENT = True - self.assertTrue(self.app.amqp.TaskProducer(conn).send_sent_event) - def test_update_state(self): @self.app.task(shared=False) diff --git a/docs/reference/celery.app.amqp.rst b/docs/reference/celery.app.amqp.rst index 467552820..5257acdbf 100644 --- a/docs/reference/celery.app.amqp.rst +++ b/docs/reference/celery.app.amqp.rst @@ -17,7 +17,11 @@ .. attribute:: Consumer - Base Consumer class used. Default is :class:`kombu.compat.Consumer`. + Base Consumer class used. Default is :class:`kombu.Consumer`. + + .. attribute:: Producer + + Base Producer class used. Default is :class:`kombu.Producer`. .. attribute:: queues @@ -25,13 +29,13 @@ .. automethod:: Queues .. automethod:: Router - .. autoattribute:: TaskConsumer - .. autoattribute:: TaskProducer .. automethod:: flush_routes + .. autoattribute:: create_task_message + .. autoattribute:: send_task_message .. autoattribute:: default_queue .. autoattribute:: default_exchange - .. autoattribute:: publisher_pool + .. autoattribute:: producer_pool .. autoattribute:: router .. autoattribute:: routes @@ -41,10 +45,3 @@ .. autoclass:: Queues :members: :undoc-members: - - TaskPublisher - ------------- - - .. autoclass:: TaskPublisher - :members: - :undoc-members: diff --git a/examples/eventlet/bulk_task_producer.py b/examples/eventlet/bulk_task_producer.py index 2002160c0..4bc75a215 100644 --- a/examples/eventlet/bulk_task_producer.py +++ b/examples/eventlet/bulk_task_producer.py @@ -3,8 +3,6 @@ from eventlet.queue import LightQueue from eventlet.event import Event -from celery import current_app - monkey_patch() @@ -27,9 +25,16 @@ def wait(self, timeout=None): class ProducerPool(object): + """Usage:: + + >>> app = Celery(broker='amqp://') + >>> ProducerPool(app) + + """ Receipt = Receipt - def __init__(self, size=20): + def __init__(self, app, size=20): + self.app = app self.size = size self.inqueue = LightQueue() self._running = None @@ -48,13 +53,12 @@ def _run(self): ] def _producer(self): - connection = current_app.connection() - publisher = current_app.amqp.TaskProducer(connection) inqueue = self.inqueue - while 1: - task, args, kwargs, options, receipt = inqueue.get() - result = task.apply_async(args, kwargs, - publisher=publisher, - **options) - receipt.finished(result) + with self.app.producer_or_acquire() as producer: + while 1: + task, args, kwargs, options, receipt = inqueue.get() + result = task.apply_async(args, kwargs, + producer=producer, + **options) + receipt.finished(result) From f9e49a8f7ae0ad349705ee9b2ea79787b9e65d83 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 27 Mar 2014 16:18:54 +0000 Subject: [PATCH 0032/1103] 3.2: Use dict and set comprehensions (+ literals) --- celery/app/amqp.py | 7 +- celery/app/defaults.py | 2 +- celery/app/registry.py | 4 +- celery/app/task.py | 7 +- celery/app/utils.py | 8 +- celery/backends/base.py | 16 ++-- celery/backends/cache.py | 2 +- celery/beat.py | 7 +- celery/bin/base.py | 7 +- celery/bin/celery.py | 8 +- celery/bin/graph.py | 2 +- celery/bin/multi.py | 6 +- celery/concurrency/asynpool.py | 15 ++-- celery/contrib/migrate.py | 2 +- celery/datastructures.py | 6 +- celery/events/__init__.py | 4 +- celery/events/state.py | 25 +++--- celery/five.py | 21 +++-- celery/result.py | 6 +- celery/schedules.py | 4 +- celery/task/http.py | 10 +-- celery/tests/app/test_amqp.py | 4 +- celery/tests/app/test_schedules.py | 100 +++++++++++----------- celery/tests/concurrency/test_prefork.py | 18 ++-- celery/tests/utils/test_datastructures.py | 2 +- celery/tests/worker/test_control.py | 4 +- celery/utils/__init__.py | 13 +-- celery/utils/functional.py | 5 +- celery/worker/__init__.py | 9 +- celery/worker/autoreload.py | 13 +-- celery/worker/consumer.py | 4 +- celery/worker/control.py | 24 +++--- celery/worker/job.py | 4 +- 33 files changed, 189 insertions(+), 180 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index c5b253396..995171e6e 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -65,7 +65,7 @@ def __init__(self, queues=None, default_exchange=None, self.ha_policy = ha_policy self.autoexchange = Exchange if autoexchange is None else autoexchange if isinstance(queues, (tuple, list)): - queues = dict((q.name, q) for q in queues) + queues = {q.name: q for q in queues} for name, q in items(queues or {}): self.add(q) if isinstance(q, Queue) else self.add_compat(name, **q) @@ -156,8 +156,9 @@ def select(self, include): Can be iterable or string. """ if include: - self._consume_from = dict((name, self[name]) - for name in maybe_list(include)) + self._consume_from = { + name: self[name] for name in maybe_list(include) + } select_subset = select # XXX compat def deselect(self, exclude): diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 15f7fcfb6..a9cc79914 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -233,7 +233,7 @@ def flatten(d, ns=''): stack.append((name + key + '_', value)) else: yield name + key, value -DEFAULTS = dict((key, value.default) for key, value in flatten(NAMESPACES)) +DEFAULTS = {key: value.default for key, value in flatten(NAMESPACES)} def find_deprecated_settings(source): diff --git a/celery/app/registry.py b/celery/app/registry.py index 7046554d9..ce7b398e3 100644 --- a/celery/app/registry.py +++ b/celery/app/registry.py @@ -57,8 +57,8 @@ def periodic(self): return self.filter_types('periodic') def filter_types(self, type): - return dict((name, task) for name, task in items(self) - if getattr(task, 'type', 'regular') == type) + return {name: task for name, task in items(self) + if getattr(task, 'type', 'regular') == type} def _unpickle_task(name): diff --git a/celery/app/task.py b/celery/app/task.py index dcb32a34d..48a5b2be2 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -724,9 +724,10 @@ def apply(self, args=None, kwargs=None, 'loglevel': options.get('loglevel', 0), 'delivery_info': {'is_eager': True}} supported_keys = fun_takes_kwargs(task.run, default_kwargs) - extend_with = dict((key, val) - for key, val in items(default_kwargs) - if key in supported_keys) + extend_with = { + key: val for key, val in items(default_kwargs) + if key in supported_keys + } kwargs.update(extend_with) tb = None diff --git a/celery/app/utils.py b/celery/app/utils.py index defdca7b8..ba5e1bb8b 100644 --- a/celery/app/utils.py +++ b/celery/app/utils.py @@ -117,11 +117,11 @@ def get_by_parts(self, *parts): def table(self, with_defaults=False, censored=True): filt = filter_hidden_settings if censored else lambda v: v - return filt(dict( - (k, v) for k, v in items( + return filt({ + k: v for k, v in items( self if with_defaults else self.without_defaults()) if k.isupper() and not k.startswith('_') - )) + }) def humanize(self, with_defaults=False, censored=True): """Return a human readable string showing changes to the @@ -182,7 +182,7 @@ def maybe_censor(key, value, mask='*' * 8): return Connection(value).as_uri(mask=mask) return value - return dict((k, maybe_censor(k, v)) for k, v in items(conf)) + return {k: maybe_censor(k, v) for k, v in items(conf)} def bugreport(app): diff --git a/celery/backends/base.py b/celery/backends/base.py index 2a40f0dc9..437dd4c83 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -435,14 +435,16 @@ def _strip_prefix(self, key): def _mget_to_results(self, values, keys): if hasattr(values, 'items'): # client returns dict so mapping preserved. - return dict((self._strip_prefix(k), self.decode(v)) - for k, v in items(values) - if v is not None) + return { + self._strip_prefix(k): self.decode(v) + for k, v in items(values) if v is not None + } else: # client returns list so need to recreate mapping. - return dict((bytes_to_str(keys[i]), self.decode(value)) - for i, value in enumerate(values) - if value is not None) + return { + bytes_to_str(keys[i]): self.decode(value) + for i, value in enumerate(values) if value is not None + } def get_many(self, task_ids, timeout=None, interval=0.5, no_ack=True, READY_STATES=states.READY_STATES): @@ -467,7 +469,7 @@ def get_many(self, task_ids, timeout=None, interval=0.5, no_ack=True, r = self._mget_to_results(self.mget([self.get_key_for_task(k) for k in keys]), keys) cache.update(r) - ids.difference_update(set(bytes_to_str(v) for v in r)) + ids.difference_update({bytes_to_str(v) for v in r}) for key, value in items(r): yield bytes_to_str(key), value if timeout and iterations * interval >= timeout: diff --git a/celery/backends/cache.py b/celery/backends/cache.py index ac8710099..7062a001a 100644 --- a/celery/backends/cache.py +++ b/celery/backends/cache.py @@ -73,7 +73,7 @@ def get(self, key, *args, **kwargs): def get_multi(self, keys): cache = self.cache - return dict((k, cache[k]) for k in keys if k in cache) + return {k: cache[k] for k in keys if k in cache} def set(self, key, value, *args, **kwargs): self.cache[key] = value diff --git a/celery/beat.py b/celery/beat.py index 0b2ec97f2..03f6b3a5d 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -295,9 +295,10 @@ def _maybe_entry(self, name, entry): return self.Entry(**dict(entry, name=name, app=self.app)) def update_from_dict(self, dict_): - self.schedule.update(dict( - (name, self._maybe_entry(name, entry)) - for name, entry in items(dict_))) + self.schedule.update({ + name: self._maybe_entry(name, entry) + for name, entry in items(dict_) + }) def merge_inplace(self, b): schedule = self.schedule diff --git a/celery/bin/base.py b/celery/bin/base.py index 9ad794665..79bdb5c8a 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -373,9 +373,10 @@ def handle_argv(self, prog_name, argv, command=None): def prepare_args(self, options, args): if options: - options = dict((k, self.expanduser(v)) - for k, v in items(vars(options)) - if not k.startswith('_')) + options = { + k: self.expanduser(v) + for k, v in items(vars(options)) if not k.startswith('_') + } args = [self.expanduser(arg) for arg in args] self.check_args(args) return options, args diff --git a/celery/bin/celery.py b/celery/bin/celery.py index 10d7c0324..3d0cf5d8f 100644 --- a/celery/bin/celery.py +++ b/celery/bin/celery.py @@ -572,10 +572,10 @@ def run(self, force_ipython=False, force_bpython=False, 'signature': celery.signature} if not without_tasks: - self.locals.update(dict( - (task.__name__, task) for task in values(self.app.tasks) - if not task.name.startswith('celery.')), - ) + self.locals.update({ + task.__name__: task for task in values(self.app.tasks) + if not task.name.startswith('celery.') + }) if force_python: return self.invoke_fallback_shell() diff --git a/celery/bin/graph.py b/celery/bin/graph.py index 5d5847672..d8aa31187 100644 --- a/celery/bin/graph.py +++ b/celery/bin/graph.py @@ -34,7 +34,7 @@ def run(self, what=None, *args, **kwargs): def bootsteps(self, *args, **kwargs): worker = self.app.WorkController() - include = set(arg.lower() for arg in args or ['worker', 'consumer']) + include = {arg.lower() for arg in args or ['worker', 'consumer']} if 'worker' in include: graph = worker.blueprint.graph if 'consumer' in include: diff --git a/celery/bin/multi.py b/celery/bin/multi.py index ca14c0bfe..23ff496f7 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -116,9 +116,9 @@ __all__ = ['MultiTool'] -SIGNAMES = set(sig for sig in dir(signal) - if sig.startswith('SIG') and '_' not in sig) -SIGMAP = dict((getattr(signal, name), name) for name in SIGNAMES) +SIGNAMES = {sig for sig in dir(signal) + if sig.startswith('SIG') and '_' not in sig} +SIGMAP = {getattr(signal, name): name for name in SIGNAMES} USAGE = """\ usage: {prog_name} start [worker options] diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 5c4d5855c..a3906c492 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -347,8 +347,9 @@ def __init__(self, processes=None, synack=False, processes = self.cpu_count() if processes is None else processes self.synack = synack # create queue-pairs for all our processes in advance. - self._queues = dict((self.create_process_queues(), None) - for _ in range(processes)) + self._queues = { + self.create_process_queues(): None for _ in range(processes) + } # inqueue fileno -> process mapping self._fileno_to_inq = {} @@ -912,7 +913,7 @@ def flush(self): self._busy_workers.clear() def _flush_writer(self, proc, writer): - fds = set([proc.inq._writer]) + fds = {proc.inq._writer} try: while fds: if not proc._is_alive(): @@ -941,9 +942,9 @@ def on_grow(self, n): """Grow the pool by ``n`` proceses.""" diff = max(self._processes - len(self._queues), 0) if diff: - self._queues.update( - dict((self.create_process_queues(), None) for _ in range(diff)) - ) + self._queues.update({ + self.create_process_queues(): None for _ in range(diff) + }) def on_shrink(self, n): """Shrink the pool by ``n`` processes.""" @@ -1085,7 +1086,7 @@ def process_flush_queues(self, proc): """ resq = proc.outq._reader on_state_change = self._result_handler.on_state_change - fds = set([resq]) + fds = {resq} while fds and not resq.closed and self._state != TERMINATE: readable, _, again = _select(fds, None, fds, timeout=0.01) if readable: diff --git a/celery/contrib/migrate.py b/celery/contrib/migrate.py index c50fba2c7..c829cdb5a 100644 --- a/celery/contrib/migrate.py +++ b/celery/contrib/migrate.py @@ -250,7 +250,7 @@ def start_filter(app, conn, filter, limit=None, timeout=1.0, if isinstance(tasks, string_t): tasks = set(tasks.split(',')) if tasks is None: - tasks = set([]) + tasks = set() def update_state(body, message): state.count += 1 diff --git a/celery/datastructures.py b/celery/datastructures.py index 9c36a3957..1551ab861 100644 --- a/celery/datastructures.py +++ b/celery/datastructures.py @@ -186,9 +186,9 @@ def topsort(self): graph = DependencyGraph() components = self._tarjan72() - NC = dict((node, component) - for component in components - for node in component) + NC = { + node: component for component in components for node in component + } for component in components: graph.add_arc(component) for node in self: diff --git a/celery/events/__init__.py b/celery/events/__init__.py index 931f3953e..b4ca9045c 100644 --- a/celery/events/__init__.py +++ b/celery/events/__init__.py @@ -112,7 +112,7 @@ class EventDispatcher(object): You need to :meth:`close` this after use. """ - DISABLED_TRANSPORTS = set(['sql']) + DISABLED_TRANSPORTS = {'sql'} app = None @@ -300,7 +300,7 @@ def __init__(self, channel, handlers=None, routing_key='#', self.adjust_clock = self.clock.adjust self.forward_clock = self.clock.forward if accept is None: - accept = set([self.app.conf.CELERY_EVENT_SERIALIZER, 'json']) + accept = {self.app.conf.CELERY_EVENT_SERIALIZER, 'json'} self.accept = accept def _get_queue_arguments(self): diff --git a/celery/events/state.py b/celery/events/state.py index c78f2d08a..541f72226 100644 --- a/celery/events/state.py +++ b/celery/events/state.py @@ -222,7 +222,7 @@ def on_heartbeat(self, timestamp=None, local_received=None, **fields): def _defaults(cls): """Deprecated, to be removed in 3.3""" source = cls() - return dict((k, getattr(source, k)) for k in cls._fields) + return {k: getattr(source, k) for k in cls._fields} @with_unique_field('uuid') @@ -295,9 +295,9 @@ def event(self, type_, timestamp=None, local_received=None, fields=None, # this state logically happens-before the current state, so merge. keep = self.merge_rules.get(state) if keep is not None: - fields = dict( - (k, v) for k, v in items(fields) if k in keep - ) + fields = { + k: v for k, v in items(fields) if k in keep + } for key, value in items(fields): setattr(self, key, value) else: @@ -323,9 +323,9 @@ def __repr__(self): def as_dict(self): get = object.__getattribute__ - return dict( - (k, get(self, k)) for k in self._fields - ) + return { + k: get(self, k) for k in self._fields + } def __reduce__(self): return _depickle_task, (self.__class__, self.as_dict()) @@ -379,7 +379,7 @@ def update(self, state, timestamp, fields, def merge(self, state, timestamp, fields): keep = self.merge_rules.get(state) if keep is not None: - fields = dict((k, v) for k, v in items(fields) if k in keep) + fields = {k: v for k, v in items(fields) if k in keep} for key, value in items(fields): setattr(self, key, value) @@ -387,7 +387,7 @@ def merge(self, state, timestamp, fields): def _defaults(cls): """Deprecated, to be removed in 3.3.""" source = cls() - return dict((k, getattr(source, k)) for k in source._fields) + return {k: getattr(source, k) for k in source._fields} class State(object): @@ -436,9 +436,10 @@ def clear_tasks(self, ready=True): def _clear_tasks(self, ready=True): if ready: - in_progress = dict( - (uuid, task) for uuid, task in self.itertasks() - if task.state not in states.READY_STATES) + in_progress = { + uuid: task for uuid, task in self.itertasks() + if task.state not in states.READY_STATES + } self.tasks.clear() self.tasks.update(in_progress) else: diff --git a/celery/five.py b/celery/five.py index 77ec1daa5..99ecc28e3 100644 --- a/celery/five.py +++ b/celery/five.py @@ -127,7 +127,7 @@ def exec_(code, globs=None, locs=None): # pragma: no cover exec_("""def reraise(tp, value, tb=None): raise tp, value, tb""") -def with_metaclass(Type, skip_attrs=set(['__dict__', '__weakref__'])): +def with_metaclass(Type, skip_attrs={'__dict__', '__weakref__'}): """Class decorator to set metaclass. Works with both Python 2 and Python 3 and it does not add @@ -137,8 +137,8 @@ def with_metaclass(Type, skip_attrs=set(['__dict__', '__weakref__'])): """ def _clone_with_metaclass(Class): - attrs = dict((key, value) for key, value in items(vars(Class)) - if key not in skip_attrs) + attrs = {key: value for key, value in items(vars(Class)) + if key not in skip_attrs} return Type(Class.__name__, Class.__bases__, attrs) return _clone_with_metaclass @@ -191,7 +191,7 @@ def format_d(i): # noqa The module %s is deprecated and will be removed in a future version. """ -DEFAULT_ATTRS = set(['__file__', '__path__', '__doc__', '__all__']) +DEFAULT_ATTRS = {'__file__', '__path__', '__doc__', '__all__'} # im_func is no longer available in Py3. # instead the unbound method itself can be used. @@ -327,8 +327,10 @@ def create_module(name, attrs, cls_attrs=None, pkg=None, pkg, _, modname = name.rpartition('.') cls_attrs['__module__'] = pkg - attrs = dict((attr_name, prepare_attr(attr) if prepare_attr else attr) - for attr_name, attr in items(attrs)) + attrs = { + attr_name: (prepare_attr(attr) if prepare_attr else attr) + for attr_name, attr in items(attrs) + } module = sys.modules[fqdn] = type(modname, (base, ), cls_attrs)(fqdn) module.__dict__.update(attrs) return module @@ -350,8 +352,9 @@ def recreate_module(name, compat_modules=(), by_module={}, direct={}, ))), ) new_module = create_module(name, attrs, cls_attrs=cattrs, base=base) - new_module.__dict__.update(dict((mod, get_compat_module(new_module, mod)) - for mod in compat_modules)) + new_module.__dict__.update({ + mod: get_compat_module(new_module, mod) for mod in compat_modules + }) return old_module, new_module @@ -375,7 +378,7 @@ def prepare(attr): def get_origins(defs): origins = {} for module, attrs in items(defs): - origins.update(dict((attr, module) for attr in attrs)) + origins.update({attr: module for attr in attrs}) return origins diff --git a/celery/result.py b/celery/result.py index 3ea837c05..901d01933 100644 --- a/celery/result.py +++ b/celery/result.py @@ -676,9 +676,9 @@ def join_native(self, timeout=None, propagate=True, """ assert_will_not_block() - order_index = None if callback else dict( - (result.id, i) for i, result in enumerate(self.results) - ) + order_index = None if callback else { + result.id: i for i, result in enumerate(self.results) + } acc = None if callback else [None for _ in range(len(self))] for task_id, meta in self.iter_native(timeout, interval, no_ack): value = meta['result'] diff --git a/celery/schedules.py b/celery/schedules.py index 6424dfa04..18cf48190 100644 --- a/celery/schedules.py +++ b/celery/schedules.py @@ -383,7 +383,7 @@ def _expand_cronspec(cronspec, max_, min_=0): int (like 7) str (like '3-5,*/15', '*', or 'monday') - set (like set([0,15,30,45])) + set (like {0,15,30,45} list (like [8-17]) And convert it to an (expanded) set representing all time unit @@ -403,7 +403,7 @@ def _expand_cronspec(cronspec, max_, min_=0): """ if isinstance(cronspec, numbers.Integral): - result = set([cronspec]) + result = {cronspec} elif isinstance(cronspec, string_t): result = crontab_parser(max_, min_).parse(cronspec) elif isinstance(cronspec, set): diff --git a/celery/task/http.py b/celery/task/http.py index e170ec3a5..2c9d8604b 100644 --- a/celery/task/http.py +++ b/celery/task/http.py @@ -41,13 +41,13 @@ def utf8dict(tup): from urllib2 import Request, urlopen # noqa - def utf8dict(tup): # noqa + def utf8dict(tup, enc='utf-8'): # noqa """With a dict's items() tuple return a new dict with any utf-8 keys/values encoded.""" - return dict( - (k.encode('utf-8'), - v.encode('utf-8') if isinstance(v, unicode) else v) # noqa - for k, v in tup) + return { + k.encode(enc): (v.encode(enc) if isinstance(v, unicode) else v) + for k, v in tup + } class InvalidResponseError(Exception): diff --git a/celery/tests/app/test_amqp.py b/celery/tests/app/test_amqp.py index cf2810d5b..9ef9f572e 100644 --- a/celery/tests/app/test_amqp.py +++ b/celery/tests/app/test_amqp.py @@ -14,11 +14,11 @@ def test_accept_content(self): self.app.conf.CELERY_ACCEPT_CONTENT = ['application/json'] self.assertEqual( self.app.amqp.TaskConsumer(conn).accept, - set(['application/json']) + {'application/json'}, ) self.assertEqual( self.app.amqp.TaskConsumer(conn, accept=['json']).accept, - set(['application/json']), + {'application/json'}, ) diff --git a/celery/tests/app/test_schedules.py b/celery/tests/app/test_schedules.py index 8cb3d6d4c..90f49125b 100644 --- a/celery/tests/app/test_schedules.py +++ b/celery/tests/app/test_schedules.py @@ -54,65 +54,63 @@ def test_parse_range(self): def test_parse_range_wraps(self): self.assertEqual(crontab_parser(12).parse('11-1'), - set([11, 0, 1])) + {11, 0, 1}) self.assertEqual(crontab_parser(60, 1).parse('2-1'), set(range(1, 60 + 1))) def test_parse_groups(self): self.assertEqual(crontab_parser().parse('1,2,3,4'), - set([1, 2, 3, 4])) + {1, 2, 3, 4}) self.assertEqual(crontab_parser().parse('0,15,30,45'), - set([0, 15, 30, 45])) + {0, 15, 30, 45}) self.assertEqual(crontab_parser(min_=1).parse('1,2,3,4'), - set([1, 2, 3, 4])) + {1, 2, 3, 4}) def test_parse_steps(self): self.assertEqual(crontab_parser(8).parse('*/2'), - set([0, 2, 4, 6])) + {0, 2, 4, 6}) self.assertEqual(crontab_parser().parse('*/2'), - set(i * 2 for i in range(30))) + {i * 2 for i in range(30)}) self.assertEqual(crontab_parser().parse('*/3'), - set(i * 3 for i in range(20))) + {i * 3 for i in range(20)}) self.assertEqual(crontab_parser(8, 1).parse('*/2'), - set([1, 3, 5, 7])) + {1, 3, 5, 7}) self.assertEqual(crontab_parser(min_=1).parse('*/2'), - set(i * 2 + 1 for i in range(30))) + {i * 2 + 1 for i in range(30)}) self.assertEqual(crontab_parser(min_=1).parse('*/3'), - set(i * 3 + 1 for i in range(20))) + {i * 3 + 1 for i in range(20)}) def test_parse_composite(self): - self.assertEqual(crontab_parser(8).parse('*/2'), set([0, 2, 4, 6])) - self.assertEqual(crontab_parser().parse('2-9/5'), set([2, 7])) - self.assertEqual(crontab_parser().parse('2-10/5'), set([2, 7])) + self.assertEqual(crontab_parser(8).parse('*/2'), {0, 2, 4, 6}) + self.assertEqual(crontab_parser().parse('2-9/5'), {2, 7}) + self.assertEqual(crontab_parser().parse('2-10/5'), {2, 7}) self.assertEqual( crontab_parser(min_=1).parse('55-5/3'), - set([55, 58, 1, 4]), + {55, 58, 1, 4}, ) - self.assertEqual(crontab_parser().parse('2-11/5,3'), set([2, 3, 7])) + self.assertEqual(crontab_parser().parse('2-11/5,3'), {2, 3, 7}) self.assertEqual( crontab_parser().parse('2-4/3,*/5,0-21/4'), - set([0, 2, 4, 5, 8, 10, 12, 15, 16, - 20, 25, 30, 35, 40, 45, 50, 55]), + {0, 2, 4, 5, 8, 10, 12, 15, 16, 20, 25, 30, 35, 40, 45, 50, 55}, ) self.assertEqual( crontab_parser().parse('1-9/2'), - set([1, 3, 5, 7, 9]), + {1, 3, 5, 7, 9}, ) - self.assertEqual(crontab_parser(8, 1).parse('*/2'), set([1, 3, 5, 7])) - self.assertEqual(crontab_parser(min_=1).parse('2-9/5'), set([2, 7])) - self.assertEqual(crontab_parser(min_=1).parse('2-10/5'), set([2, 7])) + self.assertEqual(crontab_parser(8, 1).parse('*/2'), {1, 3, 5, 7}) + self.assertEqual(crontab_parser(min_=1).parse('2-9/5'), {2, 7}) + self.assertEqual(crontab_parser(min_=1).parse('2-10/5'), {2, 7}) self.assertEqual( crontab_parser(min_=1).parse('2-11/5,3'), - set([2, 3, 7]), + {2, 3, 7}, ) self.assertEqual( crontab_parser(min_=1).parse('2-4/3,*/5,1-21/4'), - set([1, 2, 5, 6, 9, 11, 13, 16, 17, - 21, 26, 31, 36, 41, 46, 51, 56]), + {1, 2, 5, 6, 9, 11, 13, 16, 17, 21, 26, 31, 36, 41, 46, 51, 56}, ) self.assertEqual( crontab_parser(min_=1).parse('1-9/2'), - set([1, 3, 5, 7, 9]), + {1, 3, 5, 7, 9}, ) def test_parse_errors_on_empty_string(self): @@ -148,11 +146,11 @@ def test_parse_errors_on_gt_max(self): def test_expand_cronspec_eats_iterables(self): self.assertEqual( crontab._expand_cronspec(iter([1, 2, 3]), 100), - set([1, 2, 3]), + {1, 2, 3}, ) self.assertEqual( crontab._expand_cronspec(iter([1, 2, 3]), 100, 1), - set([1, 2, 3]), + {1, 2, 3}, ) def test_expand_cronspec_invalid_type(self): @@ -408,7 +406,7 @@ def test_default_crontab_spec(self): def test_simple_crontab_spec(self): c = self.crontab(minute=30) - self.assertEqual(c.minute, set([30])) + self.assertEqual(c.minute, {30}) self.assertEqual(c.hour, set(range(24))) self.assertEqual(c.day_of_week, set(range(7))) self.assertEqual(c.day_of_month, set(range(1, 32))) @@ -416,13 +414,13 @@ def test_simple_crontab_spec(self): def test_crontab_spec_minute_formats(self): c = self.crontab(minute=30) - self.assertEqual(c.minute, set([30])) + self.assertEqual(c.minute, {30}) c = self.crontab(minute='30') - self.assertEqual(c.minute, set([30])) + self.assertEqual(c.minute, {30}) c = self.crontab(minute=(30, 40, 50)) - self.assertEqual(c.minute, set([30, 40, 50])) - c = self.crontab(minute=set([30, 40, 50])) - self.assertEqual(c.minute, set([30, 40, 50])) + self.assertEqual(c.minute, {30, 40, 50}) + c = self.crontab(minute={30, 40, 50}) + self.assertEqual(c.minute, {30, 40, 50}) def test_crontab_spec_invalid_minute(self): with self.assertRaises(ValueError): @@ -432,11 +430,11 @@ def test_crontab_spec_invalid_minute(self): def test_crontab_spec_hour_formats(self): c = self.crontab(hour=6) - self.assertEqual(c.hour, set([6])) + self.assertEqual(c.hour, {6}) c = self.crontab(hour='5') - self.assertEqual(c.hour, set([5])) + self.assertEqual(c.hour, {5}) c = self.crontab(hour=(4, 8, 12)) - self.assertEqual(c.hour, set([4, 8, 12])) + self.assertEqual(c.hour, {4, 8, 12}) def test_crontab_spec_invalid_hour(self): with self.assertRaises(ValueError): @@ -446,17 +444,17 @@ def test_crontab_spec_invalid_hour(self): def test_crontab_spec_dow_formats(self): c = self.crontab(day_of_week=5) - self.assertEqual(c.day_of_week, set([5])) + self.assertEqual(c.day_of_week, {5}) c = self.crontab(day_of_week='5') - self.assertEqual(c.day_of_week, set([5])) + self.assertEqual(c.day_of_week, {5}) c = self.crontab(day_of_week='fri') - self.assertEqual(c.day_of_week, set([5])) + self.assertEqual(c.day_of_week, {5}) c = self.crontab(day_of_week='tuesday,sunday,fri') - self.assertEqual(c.day_of_week, set([0, 2, 5])) + self.assertEqual(c.day_of_week, {0, 2, 5}) c = self.crontab(day_of_week='mon-fri') - self.assertEqual(c.day_of_week, set([1, 2, 3, 4, 5])) + self.assertEqual(c.day_of_week, {1, 2, 3, 4, 5}) c = self.crontab(day_of_week='*/2') - self.assertEqual(c.day_of_week, set([0, 2, 4, 6])) + self.assertEqual(c.day_of_week, {0, 2, 4, 6}) def test_crontab_spec_invalid_dow(self): with self.assertRaises(ValueError): @@ -470,13 +468,13 @@ def test_crontab_spec_invalid_dow(self): def test_crontab_spec_dom_formats(self): c = self.crontab(day_of_month=5) - self.assertEqual(c.day_of_month, set([5])) + self.assertEqual(c.day_of_month, {5}) c = self.crontab(day_of_month='5') - self.assertEqual(c.day_of_month, set([5])) + self.assertEqual(c.day_of_month, {5}) c = self.crontab(day_of_month='2,4,6') - self.assertEqual(c.day_of_month, set([2, 4, 6])) + self.assertEqual(c.day_of_month, {2, 4, 6}) c = self.crontab(day_of_month='*/5') - self.assertEqual(c.day_of_month, set([1, 6, 11, 16, 21, 26, 31])) + self.assertEqual(c.day_of_month, {1, 6, 11, 16, 21, 26, 31}) def test_crontab_spec_invalid_dom(self): with self.assertRaises(ValueError): @@ -490,15 +488,15 @@ def test_crontab_spec_invalid_dom(self): def test_crontab_spec_moy_formats(self): c = self.crontab(month_of_year=1) - self.assertEqual(c.month_of_year, set([1])) + self.assertEqual(c.month_of_year, {1}) c = self.crontab(month_of_year='1') - self.assertEqual(c.month_of_year, set([1])) + self.assertEqual(c.month_of_year, {1}) c = self.crontab(month_of_year='2,4,6') - self.assertEqual(c.month_of_year, set([2, 4, 6])) + self.assertEqual(c.month_of_year, {2, 4, 6}) c = self.crontab(month_of_year='*/2') - self.assertEqual(c.month_of_year, set([1, 3, 5, 7, 9, 11])) + self.assertEqual(c.month_of_year, {1, 3, 5, 7, 9, 11}) c = self.crontab(month_of_year='2-12/2') - self.assertEqual(c.month_of_year, set([2, 4, 6, 8, 10, 12])) + self.assertEqual(c.month_of_year, {2, 4, 6, 8, 10, 12}) def test_crontab_spec_invalid_moy(self): with self.assertRaises(ValueError): diff --git a/celery/tests/concurrency/test_prefork.py b/celery/tests/concurrency/test_prefork.py index 7ad247436..8216531ba 100644 --- a/celery/tests/concurrency/test_prefork.py +++ b/celery/tests/concurrency/test_prefork.py @@ -153,13 +153,13 @@ def test_select(self): with patch('select.select') as select: select.return_value = ([3], [], []) self.assertEqual( - asynpool._select(set([3])), + asynpool._select({3}), ([3], [], 0), ) select.return_value = ([], [], [3]) self.assertEqual( - asynpool._select(set([3]), None, set([3])), + asynpool._select({3}, None, {3}), ([3], [], 0), ) @@ -167,13 +167,13 @@ def test_select(self): eintr.errno = errno.EINTR select.side_effect = eintr - readers = set([3]) + readers = {3} self.assertEqual(asynpool._select(readers), ([], [], 1)) self.assertIn(3, readers) with patch('select.select') as select: select.side_effect = ebadf - readers = set([3]) + readers = {3} self.assertEqual(asynpool._select(readers), ([], [], 1)) select.assert_has_calls([call([3], [], [], 0)]) self.assertNotIn(3, readers) @@ -181,7 +181,7 @@ def test_select(self): with patch('select.select') as select: select.side_effect = MemoryError() with self.assertRaises(MemoryError): - asynpool._select(set([1])) + asynpool._select({1}) with patch('select.select') as select: @@ -190,7 +190,7 @@ def se(*args): raise ebadf select.side_effect = se with self.assertRaises(MemoryError): - asynpool._select(set([3])) + asynpool._select({3}) with patch('select.select') as select: @@ -200,14 +200,14 @@ def se2(*args): raise ebadf select.side_effect = se2 with self.assertRaises(socket.error): - asynpool._select(set([3])) + asynpool._select({3}) with patch('select.select') as select: select.side_effect = socket.error() select.side_effect.errno = 34134 with self.assertRaises(socket.error): - asynpool._select(set([3])) + asynpool._select({3}) def test_promise(self): fun = Mock() @@ -309,7 +309,7 @@ def test_restart(self): raise SkipTest('functional test') def get_pids(pool): - return set([p.pid for p in pool._pool._pool]) + return {p.pid for p in pool._pool._pool} tp = self.TaskPool(5) time.sleep(0.5) diff --git a/celery/tests/utils/test_datastructures.py b/celery/tests/utils/test_datastructures.py index f26fe86f7..e9ee0f7d8 100644 --- a/celery/tests/utils/test_datastructures.py +++ b/celery/tests/utils/test_datastructures.py @@ -220,7 +220,7 @@ def test_purge(self): s.purge() hp.assert_called_with(s._heap) with patch('celery.datastructures.heappop') as hp: - s._data = dict((i * 2, i * 2) for i in range(10)) + s._data = {i * 2: i * 2 for i in range(10)} s.purge() self.assertEqual(hp.call_count, 10) diff --git a/celery/tests/worker/test_control.py b/celery/tests/worker/test_control.py index e613440ff..bb7df0daf 100644 --- a/celery/tests/worker/test_control.py +++ b/celery/tests/worker/test_control.py @@ -141,7 +141,7 @@ def test_enable_events(self): evd.groups = set() panel.handle('enable_events') self.assertIn('task', evd.groups) - evd.groups = set(['task']) + evd.groups = {'task'} self.assertIn('already enabled', panel.handle('enable_events')['ok']) def test_disable_events(self): @@ -149,7 +149,7 @@ def test_disable_events(self): panel = self.create_panel(consumer=consumer) evd = consumer.event_dispatcher evd.enabled = True - evd.groups = set(['task']) + evd.groups = {'task'} panel.handle('disable_events') self.assertNotIn('task', evd.groups) self.assertIn('already disabled', panel.handle('disable_events')['ok']) diff --git a/celery/utils/__init__.py b/celery/utils/__init__.py index 24205090b..4045a85f8 100644 --- a/celery/utils/__init__.py +++ b/celery/utils/__init__.py @@ -176,8 +176,8 @@ def lpmerge(L, R): """In place left precedent dictionary merge. Keeps values from `L`, if the value in `R` is :const:`None`.""" - set = L.__setitem__ - [set(k, v) for k, v in items(R) if v is not None] + setitem = L.__setitem__ + [setitem(k, v) for k, v in items(R) if v is not None] return L @@ -214,7 +214,7 @@ def cry(out=None, sepchr='=', seplen=49): # pragma: no cover # get a map of threads by their ID so we can print their names # during the traceback dump - tmap = dict((t.ident, t) for t in threading.enumerate()) + tmap = {t.ident: t for t in threading.enumerate()} sep = sepchr * seplen for tid, frame in items(sys._current_frames()): @@ -276,9 +276,10 @@ def jsonify(obj, elif isinstance(obj, (tuple, list)): return [_jsonify(v) for v in obj] elif isinstance(obj, dict): - return dict((k, _jsonify(v, key=k)) - for k, v in items(obj) - if (keyfilter(k) if keyfilter else 1)) + return { + k: _jsonify(v, key=k) for k, v in items(obj) + if (keyfilter(k) if keyfilter else 1) + } elif isinstance(obj, datetime.datetime): # See "Date Time String Format" in the ECMA-262 specification. r = obj.isoformat() diff --git a/celery/utils/functional.py b/celery/utils/functional.py index faa272b32..8903ff08d 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -265,8 +265,7 @@ def padlist(container, size, default=None): def mattrgetter(*attrs): """Like :func:`operator.itemgetter` but return :const:`None` on missing attributes instead of raising :exc:`AttributeError`.""" - return lambda obj: dict((attr, getattr(obj, attr, None)) - for attr in attrs) + return lambda obj: {attr: getattr(obj, attr, None) for attr in attrs} def uniq(it): @@ -303,4 +302,4 @@ def data(self): def dictfilter(d=None, **kw): """Remove all keys from dict ``d`` whose value is :const:`None`""" d = kw if d is None else (dict(d, **kw) if kw else d) - return dict((k, v) for k, v in items(d) if v is not None) + return {k: v for k, v in items(d) if v is not None} diff --git a/celery/worker/__init__.py b/celery/worker/__init__.py index 217902d2e..29a095939 100644 --- a/celery/worker/__init__.py +++ b/celery/worker/__init__.py @@ -76,7 +76,7 @@ class WorkController(object): class Blueprint(bootsteps.Blueprint): """Worker bootstep blueprint.""" name = 'Worker' - default_steps = set([ + default_steps = { 'celery.worker.components:Hub', 'celery.worker.components:Queues', 'celery.worker.components:Pool', @@ -86,8 +86,7 @@ class Blueprint(bootsteps.Blueprint): 'celery.worker.components:Consumer', 'celery.worker.autoscale:WorkerComponent', 'celery.worker.autoreload:WorkerComponent', - - ]) + } def __init__(self, app=None, hostname=None, **kwargs): self.app = app or self.app @@ -190,8 +189,8 @@ def setup_includes(self, includes): prev += tuple(includes) [self.app.loader.import_task_module(m) for m in includes] self.include = includes - task_modules = set(task.__class__.__module__ - for task in values(self.app.tasks)) + task_modules = {task.__class__.__module__ + for task in values(self.app.tasks)} self.app.conf.CELERY_INCLUDE = tuple(set(prev) | task_modules) def prepare_args(self, **kwargs): diff --git a/celery/worker/autoreload.py b/celery/worker/autoreload.py index 8ade32fb2..03dcc8efd 100644 --- a/celery/worker/autoreload.py +++ b/celery/worker/autoreload.py @@ -107,8 +107,8 @@ def register_with_event_loop(self, hub): def find_changes(self): maybe_modified = self._maybe_modified - modified = dict((f, mt) for f, mt in self._mtimes() - if maybe_modified(f, mt)) + modified = {f: mt for f, mt in self._mtimes() + if maybe_modified(f, mt)} if modified: self.on_change(modified) self.modify_times.update(modified) @@ -131,7 +131,7 @@ class KQueueMonitor(BaseMonitor): def __init__(self, *args, **kwargs): super(KQueueMonitor, self).__init__(*args, **kwargs) - self.filemap = dict((f, None) for f in self.files) + self.filemap = {f: None for f in self.files} self.fdmap = {} def register_with_event_loop(self, hub): @@ -257,13 +257,14 @@ def __init__(self, controller, modules=None, monitor_cls=None, **options): def on_init(self): files = self.file_to_module - files.update(dict( - (module_file(sys.modules[m]), m) for m in self.modules)) + files.update({ + module_file(sys.modules[m]): m for m in self.modules + }) self._monitor = self.Monitor( files, self.on_change, shutdown_event=self._is_shutdown, **self.options) - self._hashes = dict([(f, file_hash(f)) for f in files]) + self._hashes = {f: file_hash(f) for f in files} def register_with_event_loop(self, hub): if self._monitor is None: diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 16fa0ff4e..16f0b2ff5 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -550,7 +550,7 @@ def stop(self, c): class Mingle(bootsteps.StartStopStep): label = 'Mingle' requires = (Events, ) - compatible_transports = set(['amqp', 'redis']) + compatible_transports = {'amqp', 'redis'} def __init__(self, c, without_mingle=False, **kwargs): self.enabled = not without_mingle and self.compatible_transport(c.app) @@ -643,7 +643,7 @@ class Gossip(bootsteps.ConsumerStep): _cons_stamp_fields = itemgetter( 'id', 'clock', 'hostname', 'pid', 'topic', 'action', 'cver', ) - compatible_transports = set(['amqp', 'redis']) + compatible_transports = {'amqp', 'redis'} def __init__(self, c, without_gossip=False, interval=5.0, **kwargs): self.enabled = not without_gossip and self.compatible_transport(c.app) diff --git a/celery/worker/control.py b/celery/worker/control.py index fcaf04081..8de8ac838 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -56,15 +56,14 @@ def query_task(state, ids, **kwargs): def reqinfo(state, req): return state, req.info() - reqs = dict((req.id, ('reserved', req.info())) - for req in _find_requests_by_id( - ids, worker_state.reserved_requests)) - reqs.update(dict( - (req.id, ('active', req.info())) - for req in _find_requests_by_id( - ids, worker_state.active_requests, - ) - )) + reqs = { + req.id: ('reserved', req.info()) + for req in _find_requests_by_id(ids, worker_state.reserved_requests) + } + reqs.update({ + req.id: ('active', req.info()) + for req in _find_requests_by_id(ids, worker_state.active_requests) + }) return reqs @@ -280,9 +279,10 @@ def dump_tasks(state, taskinfoitems=None, **kwargs): taskinfoitems = taskinfoitems or DEFAULT_TASK_INFO_ITEMS def _extract_info(task): - fields = dict((field, str(getattr(task, field, None))) - for field in taskinfoitems - if getattr(task, field, None) is not None) + fields = { + field: str(getattr(task, field, None)) for field in taskinfoitems + if getattr(task, field, None) is not None + } if fields: info = ['='.join(f) for f in items(fields)] return '{0} [{1}]'.format(task.name, ' '.join(info)) diff --git a/celery/worker/job.py b/celery/worker/job.py index 72946d0d3..8522d0091 100644 --- a/celery/worker/job.py +++ b/celery/worker/job.py @@ -221,8 +221,8 @@ def extend_with_default_kwargs(self): 'delivery_info': self.delivery_info} fun = self.task.run supported_keys = fun_takes_kwargs(fun, default_kwargs) - extend_with = dict((key, val) for key, val in items(default_kwargs) - if key in supported_keys) + extend_with = {key: val for key, val in items(default_kwargs) + if key in supported_keys} kwargs.update(extend_with) return kwargs From e80c545f6a0fce43676bcc18b3262909ce2023d7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 27 Mar 2014 16:19:52 +0000 Subject: [PATCH 0033/1103] Removes Py2.6 workaround for missing WeakSet --- celery/_state.py | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/celery/_state.py b/celery/_state.py index 3af39bf91..80a3b112d 100644 --- a/celery/_state.py +++ b/celery/_state.py @@ -19,31 +19,6 @@ from celery.local import Proxy from celery.utils.threads import LocalStack -try: - from weakref import WeakSet as AppSet -except ImportError: # XXX Py2.6 - - class AppSet(object): # noqa - - def __init__(self): - self._refs = set() - - def add(self, app): - self._refs.add(weakref.ref(app)) - - def __iter__(self): - dirty = [] - try: - for appref in self._refs: - app = appref() - if app is None: - dirty.append(appref) - else: - yield app - finally: - while dirty: - self._refs.discard(dirty.pop()) - __all__ = ['set_default_app', 'get_current_app', 'get_current_task', 'get_current_worker_task', 'current_app', 'current_task'] @@ -51,7 +26,7 @@ def __iter__(self): default_app = None #: List of all app instances (weakrefs), must not be used directly. -_apps = AppSet() +_apps = weakref.WeakSet() _task_join_will_block = False From 524421a36dcc838a5f51e5cf122902aab774bad1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 27 Mar 2014 16:21:09 +0000 Subject: [PATCH 0034/1103] [3.2] Requires Py2.7+ --- .travis.yml | 1 - setup.py | 5 ++--- tox.ini | 10 ---------- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index c8341f045..3690f624f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ env: global: PYTHONUNBUFFERED=yes matrix: - - TOXENV=2.6 - TOXENV=2.7 - TOXENV=3.3 - TOXENV=3.4 diff --git a/setup.py b/setup.py index 24ed03769..2767346f0 100644 --- a/setup.py +++ b/setup.py @@ -19,8 +19,8 @@ CELERY_COMPAT_PROGRAMS = int(os.environ.get('CELERY_COMPAT_PROGRAMS', 1)) -if sys.version_info < (2, 6): - raise Exception('Celery 3.1 requires Python 2.6 or higher.') +if sys.version_info < (2, 7): + raise Exception('Celery 3.2 requires Python 2.7 or higher.') downgrade_packages = [ 'celery.app.task', @@ -67,7 +67,6 @@ Topic :: Software Development :: Object Brokering Programming Language :: Python Programming Language :: Python :: 2 - Programming Language :: Python :: 2.6 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 Programming Language :: Python :: 3.3 diff --git a/tox.ini b/tox.ini index d8605c74d..bde53e19a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,5 @@ [tox] envlist = - 2.6, 2.7, 3.3, 3.4, @@ -37,15 +36,6 @@ setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] -[testenv:2.6] -basepython = python2.6 -deps = -r{toxinidir}/requirements/default.txt - -r{toxinidir}/requirements/test.txt - -r{toxinidir}/requirements/test-ci.txt -setenv = C_DEBUG_TEST = 1 -commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] - [testenv:pypy] basepython = pypy deps = -r{toxinidir}/requirements/default.txt From 663e4d3a0b457e02e0a92d5a751d4046da96c286 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 27 Mar 2014 17:06:04 +0000 Subject: [PATCH 0035/1103] create_task_message for task protocol v2 --- celery/app/amqp.py | 92 ++++++++++++++++++++++++++++++++++++------ celery/app/defaults.py | 1 + 2 files changed, 81 insertions(+), 12 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 995171e6e..60221a454 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -213,6 +213,14 @@ class AMQP(object): def __init__(self, app): self.app = app + self.task_protocols = { + 1: self.as_task_v1, + 2: self.as_task_v2, + } + + @cached_property + def create_task_message(self): + return self.task_protocols[self.app.conf.CELERY_TASK_PROTOCOL] @cached_property def _task_retry(self): @@ -303,12 +311,70 @@ def default_exchange(self): return Exchange(self.app.conf.CELERY_DEFAULT_EXCHANGE, self.app.conf.CELERY_DEFAULT_EXCHANGE_TYPE) - def create_task_message(self, task_id, name, args=None, kwargs=None, - countdown=None, eta=None, group_id=None, - expires=None, now=None, retries=0, chord=None, - callbacks=None, errbacks=None, reply_to=None, - time_limit=None, soft_time_limit=None, - create_sent_event=False): + def as_task_v2(self, task_id, name, args=None, kwargs=None, + countdown=None, eta=None, group_id=None, + expires=None, now=None, retries=0, chord=None, + callbacks=None, errbacks=None, reply_to=None, + time_limit=None, soft_time_limit=None, + create_sent_event=False, timezone=None): + args = args or () + kwargs = kwargs or {} + utc = self.utc + if not isinstance(args, (list, tuple)): + raise ValueError('task args must be a list or tuple') + if not isinstance(kwargs, Mapping): + raise ValueError('task keyword arguments must be a mapping') + if countdown: # convert countdown to ETA + now = now or self.app.now() + timezone = timezone or self.app.timezone + eta = now + timedelta(seconds=countdown) + if utc: + eta = to_utc(eta).astimezone(timezone) + if isinstance(expires, numbers.Real): + now = now or self.app.now() + timezone = timezone or self.app.timezone + expires = now + timedelta(seconds=expires) + if utc: + expires = to_utc(expires).astimezone(timezone) + eta = eta and eta.isoformat() + expires = expires and expires.isoformat() + + return task_message( + headers={ + 'lang': 'py', + 'c_type': name, + 'eta': eta, + 'expires': expires, + 'callbacks': callbacks, + 'errbacks': errbacks, + 'chain': None, # TODO + 'group': group_id, + 'chord': chord, + 'retries': retries, + 'timelimit': (time_limit, soft_time_limit), + }, + properties={ + 'correlation_id': task_id, + 'reply_to': reply_to, + }, + body=(args, kwargs), + sent_event={ + 'uuid': task_id, + 'name': name, + 'args': safe_repr(args), + 'kwargs': safe_repr(kwargs), + 'retries': retries, + 'eta': eta, + 'expires': expires, + } if create_sent_event else None, + ) + + def as_task_v1(self, task_id, name, args=None, kwargs=None, + countdown=None, eta=None, group_id=None, + expires=None, now=None, timezone=None, retries=0, + chord=None, callbacks=None, errbacks=None, reply_to=None, + time_limit=None, soft_time_limit=None, + create_sent_event=False): args = args or () kwargs = kwargs or {} utc = self.utc @@ -318,24 +384,26 @@ def create_task_message(self, task_id, name, args=None, kwargs=None, raise ValueError('task keyword arguments must be a mapping') if countdown: # convert countdown to ETA now = now or self.app.now() + timezone = timezone or self.app.timezone eta = now + timedelta(seconds=countdown) if utc: - eta = to_utc(eta).astimezone(self.app.timezone) + eta = to_utc(eta).astimezone(timezone) if isinstance(expires, numbers.Real): now = now or self.app.now() + timezone = timezone or self.app.timezone expires = now + timedelta(seconds=expires) if utc: - expires = to_utc(expires).astimezone(self.app.timezone) + expires = to_utc(expires).astimezone(timezone) eta = eta and eta.isoformat() expires = expires and expires.isoformat() return task_message( - {}, - { + headers={}, + properties={ 'correlation_id': task_id, 'reply_to': reply_to, }, - { + body={ 'task': name, 'id': task_id, 'args': args, @@ -350,7 +418,7 @@ def create_task_message(self, task_id, name, args=None, kwargs=None, 'taskset': group_id, 'chord': chord, }, - { + sent_event={ 'uuid': task_id, 'name': name, 'args': safe_repr(args), diff --git a/celery/app/defaults.py b/celery/app/defaults.py index a9cc79914..b09cc0256 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -146,6 +146,7 @@ def __repr__(self): 'SEND_TASK_ERROR_EMAILS': Option(False, type='bool'), 'SEND_TASK_SENT_EVENT': Option(False, type='bool'), 'STORE_ERRORS_EVEN_IF_IGNORED': Option(False, type='bool'), + 'TASK_PROTOCOL': Option(1, type='int'), 'TASK_PUBLISH_RETRY': Option(True, type='bool'), 'TASK_PUBLISH_RETRY_POLICY': Option({ 'max_retries': 3, From 0990a17b9ddb6ad37ed71dd8cffb559674110691 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 27 Mar 2014 17:43:19 +0000 Subject: [PATCH 0036/1103] Fixes amqp publish --- celery/app/amqp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 60221a454..a535ac7bb 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -355,7 +355,7 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, }, properties={ 'correlation_id': task_id, - 'reply_to': reply_to, + 'reply_to': reply_to or '', }, body=(args, kwargs), sent_event={ @@ -401,7 +401,7 @@ def as_task_v1(self, task_id, name, args=None, kwargs=None, headers={}, properties={ 'correlation_id': task_id, - 'reply_to': reply_to, + 'reply_to': reply_to or '', }, body={ 'task': name, From 79fd299a164b9b1b22c5bafa4f23c325c474cb56 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 27 Mar 2014 17:47:59 +0000 Subject: [PATCH 0037/1103] Fixes amqp publish #2 --- celery/app/amqp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index a535ac7bb..299775e2d 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -313,10 +313,10 @@ def default_exchange(self): def as_task_v2(self, task_id, name, args=None, kwargs=None, countdown=None, eta=None, group_id=None, - expires=None, now=None, retries=0, chord=None, + expires=None, retries=0, chord=None, callbacks=None, errbacks=None, reply_to=None, time_limit=None, soft_time_limit=None, - create_sent_event=False, timezone=None): + create_sent_event=False, now=None, timezone=None): args = args or () kwargs = kwargs or {} utc = self.utc @@ -371,10 +371,10 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, def as_task_v1(self, task_id, name, args=None, kwargs=None, countdown=None, eta=None, group_id=None, - expires=None, now=None, timezone=None, retries=0, + expires=None, retries=0, chord=None, callbacks=None, errbacks=None, reply_to=None, time_limit=None, soft_time_limit=None, - create_sent_event=False): + create_sent_event=False, now=None, timezone=None): args = args or () kwargs = kwargs or {} utc = self.utc From 5d88b18ea52e9c00b82102b189ff9524af27e450 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 28 Mar 2014 13:22:17 +0000 Subject: [PATCH 0038/1103] Reorganizes AMQP class --- celery/app/amqp.py | 96 ++++++++++++++++++++-------------------------- 1 file changed, 42 insertions(+), 54 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 299775e2d..6db1d7315 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -223,19 +223,8 @@ def create_task_message(self): return self.task_protocols[self.app.conf.CELERY_TASK_PROTOCOL] @cached_property - def _task_retry(self): - return self.app.conf.CELERY_TASK_PUBLISH_RETRY - - @cached_property - def _task_retry_policy(self): - return self.app.conf.CELERY_TASK_PUBLISH_RETRY_POLICY - - @cached_property - def _task_sent_event(self): - return self.app.conf.CELERY_SEND_TASK_SENT_EVENT - - def flush_routes(self): - self._rtable = _routes.prepare(self.app.conf.CELERY_ROUTES) + def send_task_message(self): + return self._create_task_sender() def Queues(self, queues, create_missing=None, ha_policy=None, autoexchange=None): @@ -263,6 +252,9 @@ def Router(self, queues=None, create_missing=None): self.app.either('CELERY_CREATE_MISSING_QUEUES', create_missing), app=self.app) + def flush_routes(self): + self._rtable = _routes.prepare(self.app.conf.CELERY_ROUTES) + def TaskConsumer(self, channel, queues=None, accept=None, **kw): if accept is None: accept = self.app.conf.CELERY_ACCEPT_CONTENT @@ -272,45 +264,6 @@ def TaskConsumer(self, channel, queues=None, accept=None, **kw): **kw ) - @cached_property - def default_queue(self): - return self.queues[self.app.conf.CELERY_DEFAULT_QUEUE] - - @cached_property - def queues(self): - """Queue name⇒ declaration mapping.""" - return self.Queues(self.app.conf.CELERY_QUEUES) - - @queues.setter # noqa - def queues(self, queues): - return self.Queues(queues) - - @property - def routes(self): - if self._rtable is None: - self.flush_routes() - return self._rtable - - @cached_property - def router(self): - return self.Router() - - @property - def producer_pool(self): - if self._producer_pool is None: - self._producer_pool = ProducerPool( - self.app.pool, - limit=self.app.pool.limit, - Producer=self.Producer, - ) - return self._producer_pool - publisher_pool = producer_pool # compat alias - - @cached_property - def default_exchange(self): - return Exchange(self.app.conf.CELERY_DEFAULT_EXCHANGE, - self.app.conf.CELERY_DEFAULT_EXCHANGE_TYPE) - def as_task_v2(self, task_id, name, args=None, kwargs=None, countdown=None, eta=None, group_id=None, expires=None, retries=0, chord=None, @@ -519,8 +472,43 @@ def publish_task(producer, name, message, return publish_task @cached_property - def send_task_message(self): - return self._create_task_sender() + def default_queue(self): + return self.queues[self.app.conf.CELERY_DEFAULT_QUEUE] + + @cached_property + def queues(self): + """Queue name⇒ declaration mapping.""" + return self.Queues(self.app.conf.CELERY_QUEUES) + + @queues.setter # noqa + def queues(self, queues): + return self.Queues(queues) + + @property + def routes(self): + if self._rtable is None: + self.flush_routes() + return self._rtable + + @cached_property + def router(self): + return self.Router() + + @property + def producer_pool(self): + if self._producer_pool is None: + self._producer_pool = ProducerPool( + self.app.pool, + limit=self.app.pool.limit, + Producer=self.Producer, + ) + return self._producer_pool + publisher_pool = producer_pool # compat alias + + @cached_property + def default_exchange(self): + return Exchange(self.app.conf.CELERY_DEFAULT_EXCHANGE, + self.app.conf.CELERY_DEFAULT_EXCHANGE_TYPE) @cached_property def utc(self): From 5d40a9dfd19d9994afe657bc62f278aeb20d1eea Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 28 Mar 2014 13:49:24 +0000 Subject: [PATCH 0039/1103] ColorFormatter should not modify record.msg. Closes #1939 --- celery/tests/app/test_log.py | 24 ++++++++++++++---------- celery/utils/log.py | 28 +++++++++++++++++----------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/celery/tests/app/test_log.py b/celery/tests/app/test_log.py index f430d8b5b..588e39bee 100644 --- a/celery/tests/app/test_log.py +++ b/celery/tests/app/test_log.py @@ -97,7 +97,7 @@ class test_ColorFormatter(AppCase): @patch('celery.utils.log.safe_str') @patch('logging.Formatter.formatException') def test_formatException_not_string(self, fe, safe_str): - x = ColorFormatter('HELLO') + x = ColorFormatter() value = KeyError() fe.return_value = value self.assertIs(x.formatException(value), value) @@ -106,16 +106,19 @@ def test_formatException_not_string(self, fe, safe_str): @patch('logging.Formatter.formatException') @patch('celery.utils.log.safe_str') - def test_formatException_string(self, safe_str, fe, value='HELLO'): - x = ColorFormatter(value) - fe.return_value = value - self.assertTrue(x.formatException(value)) + def test_formatException_string(self, safe_str, fe): + x = ColorFormatter() + fe.return_value = 'HELLO' + try: + raise Exception() + except Exception: + self.assertTrue(x.formatException(sys.exc_info())) if sys.version_info[0] == 2: self.assertTrue(safe_str.called) @patch('logging.Formatter.format') def test_format_object(self, _format): - x = ColorFormatter(object()) + x = ColorFormatter() x.use_color = True record = Mock() record.levelname = 'ERROR' @@ -124,7 +127,7 @@ def test_format_object(self, _format): @patch('celery.utils.log.safe_str') def test_format_raises(self, safe_str): - x = ColorFormatter('HELLO') + x = ColorFormatter() def on_safe_str(s): try: @@ -136,6 +139,7 @@ def on_safe_str(s): class Record(object): levelname = 'ERROR' msg = 'HELLO' + exc_info = 1 exc_text = 'error text' stack_info = None @@ -148,15 +152,15 @@ def getMessage(self): record = Record() safe_str.return_value = record - x.format(record) - self.assertIn(''.format( + type(msg), exc + ), + ) + try: + return logging.Formatter.format(self, record) + finally: + record.msg, record.exc_info = prev_msg, einfo else: - return safe_str(sformat(self, record)) + return safe_str(msg) class LoggingProxy(object): From 0ffe314b251628bac2e82d611778998ef8057bfe Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 28 Mar 2014 19:53:11 +0000 Subject: [PATCH 0040/1103] Tests: Make sure all threads get the current app trap --- celery/_state.py | 5 +++++ celery/app/base.py | 4 ++-- celery/tests/backends/test_mongodb.py | 4 ++-- celery/tests/case.py | 17 ++++++++++------- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/celery/_state.py b/celery/_state.py index 80a3b112d..e599a0fb9 100644 --- a/celery/_state.py +++ b/celery/_state.py @@ -67,6 +67,11 @@ def _get_current_app(): )) return _tls.current_app or default_app + +def _set_current_app(app): + _tls.current_app = app + + C_STRICT_APP = os.environ.get('C_STRICT_APP') if os.environ.get('C_STRICT_APP'): # pragma: no cover def get_current_app(): diff --git a/celery/app/base.py b/celery/app/base.py index c934a7a94..914b3204a 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -26,7 +26,7 @@ from celery import platforms from celery import signals from celery._state import ( - _task_stack, _tls, get_current_app, set_default_app, + _task_stack, get_current_app, _set_current_app, set_default_app, _register_app, get_current_worker_task, ) from celery.exceptions import AlwaysEagerIgnored, ImproperlyConfigured @@ -147,7 +147,7 @@ def __init__(self, main=None, loader=None, backend=None, _register_app(self) def set_current(self): - _tls.current_app = self + _set_current_app(self) def set_default(self): set_default_app(self) diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index e260d87f0..f7546d31e 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -298,7 +298,7 @@ def test_cleanup(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) - mock_collection = Mock() + self.backend.collections = mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection @@ -309,7 +309,7 @@ def test_cleanup(self, mock_get_database): mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_COLLECTION) - mock_collection.assert_called_once_with() + self.assertTrue(mock_collection.remove.called) def test_get_database_authfailure(self): x = MongoBackend(app=self.app) diff --git a/celery/tests/case.py b/celery/tests/case.py index c96fd8ec0..808347817 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -412,8 +412,12 @@ def setUp(self): self._current_app = current_app() self._default_app = _state.default_app trap = Trap() + self._prev_tls = _state._tls _state.set_default_app(trap) - _state._tls.current_app = trap + + class NonTLS(object): + current_app = trap + _state._tls = NonTLS() self.app = self.Celery(set_as_current=False) if not self.contained: @@ -447,13 +451,12 @@ def _teardown_app(self): if isinstance(backend.client, DummyClient): backend.client.cache.clear() backend._cache.clear() - from celery._state import ( - _tls, set_default_app, _set_task_join_will_block, - ) - _set_task_join_will_block(False) + from celery import _state + _state._set_task_join_will_block(False) - set_default_app(self._default_app) - _tls.current_app = self._current_app + _state.set_default_app(self._default_app) + _state._tls = self._prev_tls + _state._tls.current_app = self._current_app if self.app is not self._current_app: self.app.close() self.app = None From bb5194295e0f087d6465ecb114fec72d3f0be52d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 31 Mar 2014 15:29:15 +0100 Subject: [PATCH 0041/1103] Result: .forget() should also clear local cache --- celery/result.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/result.py b/celery/result.py index 901d01933..706a8ec4a 100644 --- a/celery/result.py +++ b/celery/result.py @@ -96,6 +96,7 @@ def as_tuple(self): def forget(self): """Forget about (and possibly remove the result of) this task.""" + self._cache = None self.backend.forget(self.id) def revoke(self, connection=None, terminate=False, signal=None, From d70a4498fb9b988121dab5ea54a04100fbb24071 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 1 Apr 2014 16:43:38 +0100 Subject: [PATCH 0042/1103] Fixes test weirdness --- celery/tests/tasks/test_tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/celery/tests/tasks/test_tasks.py b/celery/tests/tasks/test_tasks.py index c01ffc16a..087cd88bc 100644 --- a/celery/tests/tasks/test_tasks.py +++ b/celery/tests/tasks/test_tasks.py @@ -128,6 +128,8 @@ def test_retry_kwargs_can_be_empty(self): self.retry_task_mockapply.push_request() try: with self.assertRaises(Retry): + import sys + sys.exc_clear() self.retry_task_mockapply.retry(args=[4, 4], kwargs=None) finally: self.retry_task_mockapply.pop_request() From b04cfd5eae44049f081356b651ab9908812b7c78 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 1 Apr 2014 17:00:01 +0100 Subject: [PATCH 0043/1103] Implements ResultSet.backend (Issue #1936) --- celery/result.py | 6 +++++- celery/tests/tasks/test_result.py | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/celery/result.py b/celery/result.py index 706a8ec4a..eb7364a84 100644 --- a/celery/result.py +++ b/celery/result.py @@ -658,7 +658,7 @@ def iter_native(self, timeout=None, interval=0.5, no_ack=True): results = self.results if not results: return iter([]) - return results[0].backend.get_many( + return self.backend.get_many( set(r.id for r in results), timeout=timeout, interval=interval, no_ack=no_ack, ) @@ -720,6 +720,10 @@ def subtasks(self): def supports_native_join(self): return self.results[0].supports_native_join + @property + def backend(self): + return self.app.backend if self.app else self.results[0].backend + class GroupResult(ResultSet): """Like :class:`ResultSet`, but with an associated id. diff --git a/celery/tests/tasks/test_result.py b/celery/tests/tasks/test_result.py index dbaf3f4d4..ee3c9bb1a 100644 --- a/celery/tests/tasks/test_result.py +++ b/celery/tests/tasks/test_result.py @@ -489,6 +489,7 @@ def test_join_native(self): subtasks = [self.app.AsyncResult(uuid(), backend=backend) for i in range(10)] ts = self.app.GroupResult(uuid(), subtasks) + ts.app.backend = backend backend.ids = [subtask.id for subtask in subtasks] res = ts.join_native() self.assertEqual(res, list(range(10))) @@ -526,6 +527,7 @@ def test_iter_native(self): subtasks = [self.app.AsyncResult(uuid(), backend=backend) for i in range(10)] ts = self.app.GroupResult(uuid(), subtasks) + ts.app.backend = backend backend.ids = [subtask.id for subtask in subtasks] self.assertEqual(len(list(ts.iter_native())), 10) From e07ea51f3a41d912c2fa3731552011556c1abd05 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 1 Apr 2014 17:01:09 +0100 Subject: [PATCH 0044/1103] Task callbacks applied as group means trail (.children) stored multiple times. Closes #1936. Closes #1943 --- celery/app/builtins.py | 5 +++-- celery/app/trace.py | 26 +++++++++++++++++++++----- celery/canvas.py | 6 +++--- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/celery/app/builtins.py b/celery/app/builtins.py index a60920840..9240537cf 100644 --- a/celery/app/builtins.py +++ b/celery/app/builtins.py @@ -171,7 +171,8 @@ class Group(app.Task): accept_magic_kwargs = False _decorated = True - def run(self, tasks, result, group_id, partial_args): + def run(self, tasks, result, group_id, partial_args, + add_to_parent=True): app = self.app result = result_from_tuple(result, app) # any partial args are added to all tasks in the group @@ -186,7 +187,7 @@ def run(self, tasks, result, group_id, partial_args): [stask.apply_async(group_id=group_id, producer=pub, add_to_parent=False) for stask in taskit] parent = get_current_worker_task() - if parent: + if add_to_parent and parent: parent.add_trail(result) return result diff --git a/celery/app/trace.py b/celery/app/trace.py index b4c271631..45e24c170 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -257,11 +257,27 @@ def trace_task(uuid, args, kwargs, request=None): try: # callback tasks must be applied before the result is # stored, so that result.children is populated. - group( - [signature(callback, app=app) - for callback in task.request.callbacks or []], - app=app, - ).apply_async((retval, )) + + # groups are called inline and will store trail + # separately, so need to call them separately + # so that the trail's not added multiple times :( + # (Issue #1936) + callbacks = task.request.callbacks + if callbacks: + if len(task.request.callbacks) > 1: + sigs, groups = [], [] + for sig in callbacks: + sig = signature(sig, app=app) + if isinstance(sig, group): + groups.append(sig) + else: + sigs.append(sig) + for group_ in groups: + group.apply_async((retval, )) + if sigs: + group(sigs).apply_async(retval, ) + else: + signature(callbacks[0], app=app).delay(retval) if publish_result: store_result( uuid, retval, SUCCESS, request=task_request, diff --git a/celery/canvas.py b/celery/canvas.py index cabc5070c..5efb75b09 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -477,13 +477,13 @@ def from_dict(self, d, app=None): task['args'] = task._merge(d['args'])[0] return group(tasks, app=app, **kwdict(d['options'])) - def apply_async(self, args=(), kwargs=None, **options): + def apply_async(self, args=(), kwargs=None, add_to_parent=True, **options): tasks = _maybe_clone(self.tasks, app=self._app) if not tasks: return self.freeze() type = self.type - return type(*type.prepare(dict(self.options, **options), - tasks, args)) + return type(*type.prepare(dict(self.options, **options), tasks, args), + add_to_parent=add_to_parent) def set_immutable(self, immutable): for task in self.tasks: From ec905fadc41f541c92098c037dbd94c7dac5bb6a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 1 Apr 2014 19:53:57 +0100 Subject: [PATCH 0045/1103] Multi: %n is now the same as %N to be consistent with "celery worker". Closes #1938 --- celery/bin/multi.py | 58 +++++++++++++++------------------- celery/utils/__init__.py | 7 ++-- docs/tutorials/daemonizing.rst | 22 ++++++------- 3 files changed, 40 insertions(+), 47 deletions(-) diff --git a/celery/bin/multi.py b/celery/bin/multi.py index 23ff496f7..9e6dacdf7 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -13,19 +13,19 @@ # Pidfiles and logfiles are stored in the current directory # by default. Use --pidfile and --logfile argument to change - # this. The abbreviation %N will be expanded to the current + # this. The abbreviation %n will be expanded to the current # node name. - $ celery multi start Leslie -E --pidfile=/var/run/celery/%N.pid - --logfile=/var/log/celery/%N.log + $ celery multi start Leslie -E --pidfile=/var/run/celery/%n.pid + --logfile=/var/log/celery/%n.log # You need to add the same arguments when you restart, # as these are not persisted anywhere. - $ celery multi restart Leslie -E --pidfile=/var/run/celery/%N.pid - --logfile=/var/run/celery/%N.log + $ celery multi restart Leslie -E --pidfile=/var/run/celery/%n.pid + --logfile=/var/run/celery/%n.log # To stop the node, you need to specify the same pidfile. - $ celery multi stop Leslie --pidfile=/var/run/celery/%N.pid + $ celery multi stop Leslie --pidfile=/var/run/celery/%n.pid # 3 workers, with 3 processes each $ celery multi start 3 -c 3 @@ -101,6 +101,7 @@ import sys from collections import defaultdict, namedtuple +from functools import partial from subprocess import Popen from time import sleep @@ -111,7 +112,8 @@ from celery import VERSION_BANNER from celery.five import items from celery.platforms import Pidfile, IS_WINDOWS -from celery.utils import term, nodesplit +from celery.utils import term +from celery.utils import host_format, node_format, nodesplit from celery.utils.text import pluralize __all__ = ['MultiTool'] @@ -247,8 +249,8 @@ def start(self, argv, cmd): self.retcode = int(any(retcodes)) def with_detacher_default_options(self, p): - _setdefaultopt(p.options, ['--pidfile', '-p'], '%N.pid') - _setdefaultopt(p.options, ['--logfile', '-f'], '%N.log') + _setdefaultopt(p.options, ['--pidfile', '-p'], '%n.pid') + _setdefaultopt(p.options, ['--logfile', '-f'], '%n.log') p.options.setdefault( '--cmd', '-m {0}'.format(celery_exe('worker', '--detach')), @@ -320,7 +322,7 @@ def note_waiting(): self.note('') def getpids(self, p, cmd, callback=None): - _setdefaultopt(p.options, ['--pidfile', '-p'], '%N.pid') + _setdefaultopt(p.options, ['--pidfile', '-p'], '%n.pid') nodes = [] for node in multi_args(p, cmd): @@ -491,25 +493,27 @@ def multi_args(p, cmd='celery worker', append='', prefix='', suffix=''): raise KeyError('No node at index %r' % (ns_name, )) for name in names: - this_suffix = suffix + hostname = suffix if '@' in name: - this_name = options['-n'] = name - nodename, this_suffix = nodesplit(name) - name = nodename + nodename = options['-n'] = host_format(name) + shortname, hostname = nodesplit(nodename) + name = shortname else: - nodename = '%s%s' % (prefix, name) - this_name = options['-n'] = '%s@%s' % (nodename, this_suffix) - expand = abbreviations({'%h': this_name, - '%n': name, - '%N': nodename, - '%d': this_suffix}) + shortname = '%s%s' % (prefix, name) + nodename = options['-n'] = host_format( + '{0}@{1}'.format(shortname, hostname), + ) + + expand = partial( + node_format, nodename=nodename, N=shortname, d=hostname, + ) argv = ([expand(cmd)] + [format_opt(opt, expand(value)) for opt, value in items(p.optmerge(name, options))] + [passthrough]) if append: argv.append(expand(append)) - yield multi_args_t(this_name, argv, expand, name) + yield multi_args_t(nodename, argv, expand, name) class NamespacedOptionParser(object): @@ -591,18 +595,6 @@ def parse_ns_range(ns, ranges=False): return ret -def abbreviations(mapping): - - def expand(S): - ret = S - if S is not None: - for short_opt, long_opt in items(mapping): - ret = ret.replace(short_opt, long_opt) - return ret - - return expand - - def findsig(args, default=signal.SIGTERM): for arg in reversed(args): if len(arg) == 2 and arg[0] == '-': diff --git a/celery/utils/__init__.py b/celery/utils/__init__.py index 4045a85f8..7bf6c3137 100644 --- a/celery/utils/__init__.py +++ b/celery/utils/__init__.py @@ -352,7 +352,7 @@ def default_nodename(hostname): def node_format(s, nodename, **extra): name, host = nodesplit(nodename) return host_format( - s, host, n=name or NODENAME_DEFAULT, **extra) + s, host, name or NODENAME_DEFAULT, **extra) def _fmt_process_index(prefix='', default='0'): @@ -362,9 +362,10 @@ def _fmt_process_index(prefix='', default='0'): _fmt_process_index_with_prefix = partial(_fmt_process_index, '-', '') -def host_format(s, host=None, **extra): +def host_format(s, host=None, name=None, **extra): host = host or socket.gethostname() - name, _, domain = host.partition('.') + hname, _, domain = host.partition('.') + name = name or hname keys = dict({ 'h': host, 'n': name, 'd': domain, 'i': _fmt_process_index, 'I': _fmt_process_index_with_prefix, diff --git a/docs/tutorials/daemonizing.rst b/docs/tutorials/daemonizing.rst index 0c644584c..600dd4e8f 100644 --- a/docs/tutorials/daemonizing.rst +++ b/docs/tutorials/daemonizing.rst @@ -96,9 +96,9 @@ This is an example configuration for a Python project. # Extra command-line arguments to the worker CELERYD_OPTS="--time-limit=300 --concurrency=8" - # %N will be replaced with the first part of the nodename. - CELERYD_LOG_FILE="/var/log/celery/%N.log" - CELERYD_PID_FILE="/var/run/celery/%N.pid" + # %n will be replaced with the first part of the nodename. + CELERYD_LOG_FILE="/var/log/celery/%n.log" + CELERYD_PID_FILE="/var/run/celery/%n.pid" # Workers should run as an unprivileged user. # You need to create this user manually (or you can choose @@ -153,10 +153,10 @@ Available options directory. * CELERYD_PID_FILE - Full path to the PID file. Default is /var/run/celery/%N.pid + Full path to the PID file. Default is /var/run/celery/%n.pid * CELERYD_LOG_FILE - Full path to the worker log file. Default is /var/log/celery/%N.log + Full path to the worker log file. Default is /var/log/celery/%n.log * CELERYD_LOG_LEVEL Worker log level. Default is INFO. @@ -311,9 +311,9 @@ This is an example configuration for a Python project: # Extra command-line arguments to the worker CELERYD_OPTS="--time-limit=300 --concurrency=8" - # %N will be replaced with the first part of the nodename. - CELERYD_LOG_FILE="/var/log/celery/%N.log" - CELERYD_PID_FILE="/var/run/celery/%N.pid" + # %n will be replaced with the first part of the nodename. + CELERYD_LOG_FILE="/var/log/celery/%n.log" + CELERYD_PID_FILE="/var/run/celery/%n.pid" .. _generic-systemd-celeryd-django-example: @@ -339,9 +339,9 @@ This is an example configuration for those using `django-celery`: # Extra command-line arguments to the worker CELERYD_OPTS="--time-limit=300 --concurrency=8" - # %N will be replaced with the first part of the nodename. - CELERYD_LOG_FILE="/var/log/celery/%N.log" - CELERYD_PID_FILE="/var/run/celery/%N.pid" + # %n will be replaced with the first part of the nodename. + CELERYD_LOG_FILE="/var/log/celery/%n.log" + CELERYD_PID_FILE="/var/run/celery/%n.pid" To add an environment variable such as DJANGO_SETTINGS_MODULE use the Environment in celery.service. From be65e4ad25239348b23c0ab29d295da0aa131631 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Apr 2014 15:08:02 +0100 Subject: [PATCH 0046/1103] More tests for empty groups --- celery/tests/tasks/test_canvas.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/celery/tests/tasks/test_canvas.py b/celery/tests/tasks/test_canvas.py index 8ecbbbbc9..4c071a8a1 100644 --- a/celery/tests/tasks/test_canvas.py +++ b/celery/tests/tasks/test_canvas.py @@ -273,6 +273,9 @@ def test_from_dict(self): def test_call_empty_group(self): x = group(app=self.app) self.assertFalse(len(x())) + x.delay() + x.apply_async() + x() def test_skew(self): g = group([self.add.s(i, i) for i in range(10)]) From 8d8f26d02bd8710832931949b78b7619cef66273 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Apr 2014 15:08:17 +0100 Subject: [PATCH 0047/1103] Multi doc update --- celery/bin/multi.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/celery/bin/multi.py b/celery/bin/multi.py index 9e6dacdf7..1ceb50356 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -47,6 +47,9 @@ # specify fully qualified nodenames $ celery multi start foo@worker.example.com bar@worker.example.com -c 3 + # fully qualified nodenames but using the current hostname + $ celery multi start foo@%h bar@%h + # Advanced example starting 10 workers in the background: # * Three of the workers processes the images and video queue # * Two of the workers processes the data queue with loglevel DEBUG From 2bcb3357cea139bac46e6ca044dd9dc33a74e66b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Apr 2014 15:15:46 +0100 Subject: [PATCH 0048/1103] get_backend_cls cannot be memoized as it keeps reference to app.loader. Issue #1949 --- celery/backends/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/celery/backends/__init__.py b/celery/backends/__init__.py index 421f7f480..fbe8a9c62 100644 --- a/celery/backends/__init__.py +++ b/celery/backends/__init__.py @@ -16,7 +16,6 @@ from celery._state import current_app from celery.five import reraise from celery.utils.imports import symbol_by_name -from celery.utils.functional import memoize __all__ = ['get_backend_cls', 'get_backend_by_url'] @@ -41,7 +40,6 @@ default_backend = Proxy(lambda: current_app.backend) -@memoize(100) def get_backend_cls(backend=None, loader=None): """Get backend class by name/alias""" backend = backend or 'disabled' From 283a75690122ed65b5ddfd11d98b0d98e99ed534 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Apr 2014 16:00:25 +0100 Subject: [PATCH 0049/1103] Fixes multi tests --- celery/bin/multi.py | 1 + celery/tests/bin/test_multi.py | 13 ++----------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/celery/bin/multi.py b/celery/bin/multi.py index 1ceb50356..2f2055ece 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -509,6 +509,7 @@ def multi_args(p, cmd='celery worker', append='', prefix='', suffix=''): expand = partial( node_format, nodename=nodename, N=shortname, d=hostname, + h=nodename, ) argv = ([expand(cmd)] + [format_opt(opt, expand(value)) diff --git a/celery/tests/bin/test_multi.py b/celery/tests/bin/test_multi.py index 0b2ecd981..76a6c1b64 100644 --- a/celery/tests/bin/test_multi.py +++ b/celery/tests/bin/test_multi.py @@ -8,7 +8,6 @@ main, MultiTool, findsig, - abbreviations, parse_ns_range, format_opt, quote, @@ -30,14 +29,6 @@ def test_findsig(self): self.assertEqual(findsig(['-s']), signal.SIGTERM) self.assertEqual(findsig(['-log']), signal.SIGTERM) - def test_abbreviations(self): - expander = abbreviations({'%s': 'START', - '%x': 'STOP'}) - self.assertEqual(expander('foo%s'), 'fooSTART') - self.assertEqual(expander('foo%x'), 'fooSTOP') - self.assertEqual(expander('foo%y'), 'foo%y') - self.assertIsNone(expander(None)) - def test_parse_ns_range(self): self.assertEqual(parse_ns_range('1-3', True), ['1', '2', '3']) self.assertEqual(parse_ns_range('1-3', False), ['1-3']) @@ -78,6 +69,7 @@ class test_multi_args(AppCase): @patch('socket.gethostname') def test_parse(self, gethostname): + gethostname.return_value = 'example.com' p = NamespacedOptionParser([ '-c:jerry,elaine', '5', '--loglevel:kramer=DEBUG', @@ -120,12 +112,11 @@ def assert_line_in(name, args): ) expand = names[0][2] self.assertEqual(expand('%h'), '*P*jerry@*S*') - self.assertEqual(expand('%n'), 'jerry') + self.assertEqual(expand('%n'), '*P*jerry') names2 = list(multi_args(p, cmd='COMMAND', append='', prefix='*P*', suffix='*S*')) self.assertEqual(names2[0][1][-1], '-- .disable_rate_limits=1') - gethostname.return_value = 'example.com' p2 = NamespacedOptionParser(['10', '-c:1', '5']) names3 = list(multi_args(p2, cmd='COMMAND')) self.assertEqual(len(names3), 10) From 63a69938dca692ae95d720a58f14ff63562b105a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Apr 2014 16:01:01 +0100 Subject: [PATCH 0050/1103] Fixes backend tests for Issue #1949 --- celery/tests/backends/test_backends.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/celery/tests/backends/test_backends.py b/celery/tests/backends/test_backends.py index bba612770..c6a936b93 100644 --- a/celery/tests/backends/test_backends.py +++ b/celery/tests/backends/test_backends.py @@ -19,15 +19,6 @@ def test_get_backend_aliases(self): expect_cls, ) - def test_get_backend_cache(self): - backends.get_backend_cls.clear() - hits = backends.get_backend_cls.hits - misses = backends.get_backend_cls.misses - self.assertTrue(backends.get_backend_cls('amqp', self.app.loader)) - self.assertEqual(backends.get_backend_cls.misses, misses + 1) - self.assertTrue(backends.get_backend_cls('amqp', self.app.loader)) - self.assertEqual(backends.get_backend_cls.hits, hits + 1) - def test_unknown_backend(self): with self.assertRaises(ImportError): backends.get_backend_cls('fasodaopjeqijwqe', self.app.loader) From 9384218ba5c2bda496fd1be220eb4ef0f1d50f58 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Apr 2014 16:01:59 +0100 Subject: [PATCH 0051/1103] Fixes memory leak when app is registered in multiprocessing after fork registry. Closes #1949 --- celery/app/base.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/celery/app/base.py b/celery/app/base.py index 914b3204a..a134cb84c 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -58,6 +58,8 @@ Please set this variable and make it point to a configuration module.""" +_after_fork_registered = False + def app_has_custom(app, attr): return mro_lookup(app.__class__, attr, stop=(Celery, object), @@ -70,6 +72,29 @@ def _unpickle_appattr(reverse_name, args): return get_current_app()._rgetattr(reverse_name)(*args) +def _global_after_fork(): + # Previously every app would call: + # `register_after_fork(app, app._after_fork)` + # but this created a leak as `register_after_fork` stores concrete object + # references and once registered an object cannot be removed without + # touching and iterating over the private afterfork registry list. + # + # See Issue #1949 + from celery import _state + from multiprocessing.util import info + for app in _state.apps: + try: + app._after_fork() + except Exception as exc: + info('after forker raised exception: %r' % (exc, ), exc_info=1) + + +def _ensure_after_fork(): + global _after_fork_registered + _after_fork_registered = True + register_after_fork(_global_after_fork, _global_after_fork) + + class Celery(object): #: This is deprecated, use :meth:`reduce_keys` instead Pickler = AppPickler @@ -590,7 +615,7 @@ def TaskSetResult(self): # XXX compat @property def pool(self): if self._pool is None: - register_after_fork(self, self._after_fork) + _ensure_after_fork() limit = self.conf.BROKER_POOL_LIMIT self._pool = self.connection().Pool(limit=limit) return self._pool From f9d2f21e6b585c707a82c9f10086a3ea101d4e4d Mon Sep 17 00:00:00 2001 From: Ankur Dedania Date: Mon, 7 Apr 2014 13:16:45 -0500 Subject: [PATCH 0052/1103] Update periodic-tasks.rst Midnight added as an occurance --- docs/userguide/periodic-tasks.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/userguide/periodic-tasks.rst b/docs/userguide/periodic-tasks.rst index 6b829887f..92d065b6d 100644 --- a/docs/userguide/periodic-tasks.rst +++ b/docs/userguide/periodic-tasks.rst @@ -188,7 +188,8 @@ The syntax of these crontab expressions are very flexible. Some examples: | ``crontab(minute=0, hour=0)`` | Execute daily at midnight. | +-----------------------------------------+--------------------------------------------+ | ``crontab(minute=0, hour='*/3')`` | Execute every three hours: | -| | 3am, 6am, 9am, noon, 3pm, 6pm, 9pm. | +| | midnight, 3am, 6am, 9am, | +| | noon, 3pm, 6pm, 9pm. | +-----------------------------------------+--------------------------------------------+ | ``crontab(minute=0,`` | Same as previous. | | ``hour='0,3,6,9,12,15,18,21')`` | | From 9dbda6a5f9a6f6b7e9b304858cc82bcb8faf770b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 8 Apr 2014 13:39:27 +0100 Subject: [PATCH 0053/1103] Docs: generic celerybeat options should not write schedule to root owned directory. Thanks to Nikos Fertakis. --- docs/tutorials/daemonizing.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/tutorials/daemonizing.rst b/docs/tutorials/daemonizing.rst index 600dd4e8f..6ba461ee0 100644 --- a/docs/tutorials/daemonizing.rst +++ b/docs/tutorials/daemonizing.rst @@ -211,7 +211,7 @@ This is an example configuration for a Python project: CELERYBEAT_CHDIR="/opt/Myproject/" # Extra arguments to celerybeat - CELERYBEAT_OPTS="--schedule=/var/run/celerybeat-schedule" + CELERYBEAT_OPTS="--schedule=/var/run/celery/celerybeat-schedule" .. _generic-initd-celerybeat-django-example: @@ -265,7 +265,7 @@ Available options * CELERY_CREATE_LOGDIR Always create logfile directory. By default only enable when no custom logfile location set. - + .. _daemon-systemd-generic: Usage systemd @@ -279,10 +279,10 @@ Service file: celery.service :Usage: `systemctl {start|stop|restart|status} celery.service` :Configuration file: /etc/conf.d/celery -To create a temporary folders for the log and pid files change user and group in +To create a temporary folders for the log and pid files change user and group in /usr/lib/tmpfiles.d/celery.conf. -To configure user, group, chdir change settings User, Group and WorkingDirectory defines -in /usr/lib/systemd/system/celery.service. +To configure user, group, chdir change settings User, Group and WorkingDirectory defines +in /usr/lib/systemd/system/celery.service. .. _generic-systemd-celery-example: From 574559dd435c1303bfdc06e78211771c241ee0f8 Mon Sep 17 00:00:00 2001 From: Matthew Duggan Date: Wed, 9 Apr 2014 14:09:29 +0900 Subject: [PATCH 0054/1103] Add option to worker to control heartbeat interval. Thanks to my colleague Craig Northway for the patch. --- celery/bin/worker.py | 5 +++++ celery/tests/bin/test_worker.py | 4 +++- celery/tests/worker/test_consumer.py | 21 ++++++++++++++++++++- celery/worker/consumer.py | 7 +++++-- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/celery/bin/worker.py b/celery/bin/worker.py index bdc564d4f..44be17e4d 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -86,6 +86,10 @@ Do not send event heartbeats. +.. cmdoption:: --heartbeat-interval + + Interval in seconds at which to send worker heartbeat + .. cmdoption:: --purge Purges all waiting tasks before the daemon is started. @@ -245,6 +249,7 @@ def get_options(self): Option('--without-gossip', action='store_true', default=False), Option('--without-mingle', action='store_true', default=False), Option('--without-heartbeat', action='store_true', default=False), + Option('--heartbeat-interval', type='int'), Option('-O', dest='optimization'), Option('-D', '--detach', action='store_true'), ) + daemon_options() + tuple(self.app.user_options['worker']) diff --git a/celery/tests/bin/test_worker.py b/celery/tests/bin/test_worker.py index fbb7c52c2..e4ebf7157 100644 --- a/celery/tests/bin/test_worker.py +++ b/celery/tests/bin/test_worker.py @@ -443,8 +443,10 @@ def test_set_process_status(self): def test_parse_options(self): cmd = worker() cmd.app = self.app - opts, args = cmd.parse_options('worker', ['--concurrency=512']) + opts, args = cmd.parse_options('worker', ['--concurrency=512', + '--heartbeat-interval=10']) self.assertEqual(opts.concurrency, 512) + self.assertEqual(opts.heartbeat_interval, 10) @disable_stdouts def test_main(self): diff --git a/celery/tests/worker/test_consumer.py b/celery/tests/worker/test_consumer.py index 81199b85d..b9962a49c 100644 --- a/celery/tests/worker/test_consumer.py +++ b/celery/tests/worker/test_consumer.py @@ -164,11 +164,30 @@ def test_start(self): with patch('celery.worker.heartbeat.Heart') as hcls: h = Heart(c) self.assertTrue(h.enabled) + self.assertEqual(h.heartbeat_interval, None) self.assertIsNone(c.heart) h.start(c) self.assertTrue(c.heart) - hcls.assert_called_with(c.timer, c.event_dispatcher) + hcls.assert_called_with(c.timer, c.event_dispatcher, + h.heartbeat_interval) + c.heart.start.assert_called_with() + + def test_start_heartbeat_interval(self): + c = Mock() + c.timer = Mock() + c.event_dispatcher = Mock() + + with patch('celery.worker.heartbeat.Heart') as hcls: + h = Heart(c, False, 20) + self.assertTrue(h.enabled) + self.assertEqual(h.heartbeat_interval, 20) + self.assertIsNone(c.heart) + + h.start(c) + self.assertTrue(c.heart) + hcls.assert_called_with(c.timer, c.event_dispatcher, + h.heartbeat_interval) c.heart.start.assert_called_with() diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 16f0b2ff5..22d768649 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -534,12 +534,15 @@ def shutdown(self, c): class Heart(bootsteps.StartStopStep): requires = (Events, ) - def __init__(self, c, without_heartbeat=False, **kwargs): + def __init__(self, c, without_heartbeat=False, heartbeat_interval=None, + **kwargs): self.enabled = not without_heartbeat + self.heartbeat_interval = heartbeat_interval c.heart = None def start(self, c): - c.heart = heartbeat.Heart(c.timer, c.event_dispatcher) + c.heart = heartbeat.Heart(c.timer, c.event_dispatcher, + self.heartbeat_interval) c.heart.start() def stop(self, c): From 547b2530866629c878d0594f959223b3229e3821 Mon Sep 17 00:00:00 2001 From: dtheodor Date: Wed, 9 Apr 2014 11:07:40 +0200 Subject: [PATCH 0055/1103] Update task.py --- celery/app/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/task.py b/celery/app/task.py index 48a5b2be2..e8a4697a2 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -313,7 +313,7 @@ class Task(object): #: :setting:`CELERY_ACKS_LATE` setting. acks_late = None - #: List/tuple of expected exceptions. + #: Tuple of expected exceptions. #: #: These are errors that are expected in normal operation #: and that should not be regarded as a real error by the worker. From 606e7ee90e178348b777ed4653af5d3d5b3dc806 Mon Sep 17 00:00:00 2001 From: dtheodor Date: Wed, 9 Apr 2014 11:08:12 +0200 Subject: [PATCH 0056/1103] Update tasks.rst --- docs/userguide/tasks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index b984d5480..06351d5da 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -472,7 +472,7 @@ General .. attribute:: Task.throws - Optional list of expected error classes that should not be regarded + Optional tuple of expected error classes that should not be regarded as an actual error. Errors in this list will be reported as a failure to the result backend, From 7cbb7755c73db68963d3be0019f6498dc8056ae7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 9 Apr 2014 15:49:56 +0100 Subject: [PATCH 0057/1103] Try to fix CI tests --- celery/tests/tasks/test_tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/celery/tests/tasks/test_tasks.py b/celery/tests/tasks/test_tasks.py index 087cd88bc..5607c255d 100644 --- a/celery/tests/tasks/test_tasks.py +++ b/celery/tests/tasks/test_tasks.py @@ -129,7 +129,10 @@ def test_retry_kwargs_can_be_empty(self): try: with self.assertRaises(Retry): import sys - sys.exc_clear() + try: + sys.exc_clear() + except AttributeError: + pass self.retry_task_mockapply.retry(args=[4, 4], kwargs=None) finally: self.retry_task_mockapply.pop_request() From 2f110d2f13204d413d14d366e9ff2d61427396f2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 9 Apr 2014 16:03:49 +0100 Subject: [PATCH 0058/1103] Make celery.five importable from outside. Closes #1968 --- celery/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/__init__.py b/celery/__init__.py index 848907cf3..7f5c7a7c4 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -144,7 +144,7 @@ def maybe_patch_concurrency(argv=sys.argv, __package__='celery', __file__=__file__, __path__=__path__, __doc__=__doc__, __version__=__version__, __author__=__author__, __contact__=__contact__, - __homepage__=__homepage__, __docformat__=__docformat__, + __homepage__=__homepage__, __docformat__=__docformat__, five=five, VERSION=VERSION, SERIES=SERIES, VERSION_BANNER=VERSION_BANNER, maybe_patch_concurrency=maybe_patch_concurrency, _find_option_with_arg=_find_option_with_arg, From 0dcb8fe0b1ee822e8e8921263fe850557b9e19c5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 9 Apr 2014 18:21:42 +0100 Subject: [PATCH 0059/1103] Fixes tests --- celery/__init__.py | 4 ++-- celery/five.py | 6 +++--- celery/task/__init__.py | 4 ++-- celery/tests/app/test_loaders.py | 2 ++ 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/celery/__init__.py b/celery/__init__.py index 7f5c7a7c4..ddf8af385 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -127,9 +127,9 @@ def maybe_patch_concurrency(argv=sys.argv, concurrency.get_implementation(pool) # Lazy loading -from .five import recreate_module +from celery import five -old_module, new_module = recreate_module( # pragma: no cover +old_module, new_module = five.recreate_module( # pragma: no cover __name__, by_module={ 'celery.app': ['Celery', 'bugreport', 'shared_task'], diff --git a/celery/five.py b/celery/five.py index 99ecc28e3..56c640ac8 100644 --- a/celery/five.py +++ b/celery/five.py @@ -295,7 +295,7 @@ def reclassmethod(method): return classmethod(fun_of_method(method)) -class MagicModule(ModuleType): +class LazyModule(ModuleType): _compat_modules = () _all_by_module = {} _direct = {} @@ -321,7 +321,7 @@ def __reduce__(self): def create_module(name, attrs, cls_attrs=None, pkg=None, - base=MagicModule, prepare_attr=None): + base=LazyModule, prepare_attr=None): fqdn = '.'.join([pkg.__name__, name]) if pkg else name cls_attrs = {} if cls_attrs is None else cls_attrs pkg, _, modname = name.rpartition('.') @@ -337,7 +337,7 @@ def create_module(name, attrs, cls_attrs=None, pkg=None, def recreate_module(name, compat_modules=(), by_module={}, direct={}, - base=MagicModule, **attrs): + base=LazyModule, **attrs): old_module = sys.modules[name] origins = get_origins(by_module) compat_modules = COMPAT_MODULES.get(name, ()) diff --git a/celery/task/__init__.py b/celery/task/__init__.py index f8326e887..4ab1a2feb 100644 --- a/celery/task/__init__.py +++ b/celery/task/__init__.py @@ -12,7 +12,7 @@ from __future__ import absolute_import from celery._state import current_app, current_task as current -from celery.five import MagicModule, recreate_module +from celery.five import LazyModule, recreate_module from celery.local import Proxy __all__ = [ @@ -32,7 +32,7 @@ from .sets import TaskSet -class module(MagicModule): +class module(LazyModule): def __call__(self, *args, **kwargs): return self.task(*args, **kwargs) diff --git a/celery/tests/app/test_loaders.py b/celery/tests/app/test_loaders.py index f1b1bb037..ab69e501d 100644 --- a/celery/tests/app/test_loaders.py +++ b/celery/tests/app/test_loaders.py @@ -206,9 +206,11 @@ def test_import_from_cwd(self): except ValueError: pass celery = sys.modules.pop('celery', None) + sys.modules.pop('celery.five', None) try: self.assertTrue(l.import_from_cwd('celery')) sys.modules.pop('celery', None) + sys.modules.pop('celery.five', None) sys.path.insert(0, os.getcwd()) self.assertTrue(l.import_from_cwd('celery')) finally: From fb48b1f357f7a416d1413d0056158a74191185af Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Apr 2014 15:37:20 +0100 Subject: [PATCH 0060/1103] Fixes test for #1964 --- celery/tests/backends/test_mongodb.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index f7546d31e..a32d9ed27 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -196,9 +196,10 @@ def test_get_task_meta_for(self, mock_get_database): mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual( - ['status', 'task_id', 'date_done', 'traceback', 'result', - 'children'], - list(ret_val.keys())) + list(sorted(['status', 'task_id', 'date_done', 'traceback', + 'result', 'children'])), + list(sorted(ret_val.keys())), + ) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for_no_result(self, mock_get_database): From ae277443cfe381a0c855a67338afe08a34cff1b5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Apr 2014 15:37:27 +0100 Subject: [PATCH 0061/1103] Wording --- docs/userguide/optimizing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/optimizing.rst b/docs/userguide/optimizing.rst index 590143c63..459069fd2 100644 --- a/docs/userguide/optimizing.rst +++ b/docs/userguide/optimizing.rst @@ -223,5 +223,5 @@ worker option: $ celery -A proj worker -l info -Ofair -With this option enabled the worker will only write to workers that are +With this option enabled the worker will only write to processes that are available for work, disabling the prefetch behavior. From 4ec8b25ca96f7982b27174d27cc247a7c1a6fc59 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 14 Apr 2014 21:44:40 +0100 Subject: [PATCH 0062/1103] Adds celery.version_info_t --- celery/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/__init__.py b/celery/__init__.py index ddf8af385..86a3e450f 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -146,6 +146,7 @@ def maybe_patch_concurrency(argv=sys.argv, __author__=__author__, __contact__=__contact__, __homepage__=__homepage__, __docformat__=__docformat__, five=five, VERSION=VERSION, SERIES=SERIES, VERSION_BANNER=VERSION_BANNER, + version_info_t=version_info_t, maybe_patch_concurrency=maybe_patch_concurrency, _find_option_with_arg=_find_option_with_arg, ) From ae04f684e9e3c0f92e221ac3b35f148515d2ff61 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 14 Apr 2014 21:46:07 +0100 Subject: [PATCH 0063/1103] Fix for QoS when using RabbitMQ 3.3 or later. (Issue celery/kombu#339) --- celery/worker/consumer.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 22d768649..3f627edce 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -592,11 +592,29 @@ def __init__(self, c, **kwargs): def start(self, c): c.update_strategies() + + # - RabbitMQ 3.3 completely redefines how basic_qos works.. + # This will detect if the new qos smenatics is in effect, + # and if so make sure the 'apply_global' flag is set on qos updates. + qos_global = not ( + c.connection.transport.qos_semantics_matches_spec( + c.connection.connection)) + + # set initial prefetch count + c.connection.default_channel.basic_qos( + 0, c.initial_prefetch_count, qos_global, + ) + c.task_consumer = c.app.amqp.TaskConsumer( c.connection, on_decode_error=c.on_decode_error, ) - c.qos = QoS(c.task_consumer.qos, c.initial_prefetch_count) - c.qos.update() # set initial prefetch count + + def set_prefetch_count(prefetch_count): + return c.task_consumer.qos( + prefetch_count=prefetch_count, + apply_global=qos_global, + ) + c.qos = QoS(set_prefetch_count, c.initial_prefetch_count) def stop(self, c): if c.task_consumer: From a6f01727abc2e594280cf3e6748cf0e11406528a Mon Sep 17 00:00:00 2001 From: Matthew Duggan Date: Tue, 15 Apr 2014 10:05:06 +0900 Subject: [PATCH 0064/1103] Add my name to contributors --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index b11226613..ccfe96062 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -158,3 +158,4 @@ Dan McGee, 2014/01/27 Paul Kilgo, 2014/01/28 Martin Davidsson, 2014/02/08 Chris Clark, 2014/02/20 +Matthew Duggan, 2014/04/10 From 5ed3aa492459f244d19c01f4d7b155d488c590b6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 15 Apr 2014 14:50:10 +0100 Subject: [PATCH 0065/1103] Changelog stub for 3.1.11 --- Changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Changelog b/Changelog index 62abdf694..92b8dd506 100644 --- a/Changelog +++ b/Changelog @@ -8,6 +8,13 @@ This document contains change notes for bugfix releases in the 3.1.x series (Cipater), please see :ref:`whatsnew-3.1` for an overview of what's new in Celery 3.1. +.. _version-3.1.11: + +3.1.11 +====== +:release-date: 2014-04-XX XX:XX X.X UTC +:release-by: XXX + .. _version-3.1.10: 3.1.10 From 72899335c4b19ba2d7c3520f51fe5fc54e51e526 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 15 Apr 2014 15:31:40 +0100 Subject: [PATCH 0066/1103] Batches: Do not attempt to cancel non-existing tref. Closes #1984 --- celery/contrib/batches.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/contrib/batches.py b/celery/contrib/batches.py index 0248ebf8d..8cabc6f61 100644 --- a/celery/contrib/batches.py +++ b/celery/contrib/batches.py @@ -226,7 +226,8 @@ def _do_flush(self): self.flush(requests) if not requests: logger.debug('Batches: Cancelling timer: Nothing in buffer.') - self._tref.cancel() # cancel timer. + if self._tref: + self._tref.cancel() # cancel timer. self._tref = None def apply_buffer(self, requests, args=(), kwargs={}): From abf1ff963b854e7114ea98c58a4562d49597fbcc Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 15 Apr 2014 17:25:48 +0100 Subject: [PATCH 0067/1103] Use the Connection.qos_semantics_matches_spec property instead --- celery/worker/consumer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 3f627edce..c761d043a 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -596,9 +596,7 @@ def start(self, c): # - RabbitMQ 3.3 completely redefines how basic_qos works.. # This will detect if the new qos smenatics is in effect, # and if so make sure the 'apply_global' flag is set on qos updates. - qos_global = not ( - c.connection.transport.qos_semantics_matches_spec( - c.connection.connection)) + qos_global = not c.connection.qos_semantics_matches_spec # set initial prefetch count c.connection.default_channel.basic_qos( From daccf8c79107967c524cc3ba3d28b9e209cfe0a4 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Tue, 15 Apr 2014 16:54:53 -0400 Subject: [PATCH 0068/1103] Fix typo in comment ("if if") --- docs/tutorials/task-cookbook.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/task-cookbook.rst b/docs/tutorials/task-cookbook.rst index ad772a751..c47c09a7d 100644 --- a/docs/tutorials/task-cookbook.rst +++ b/docs/tutorials/task-cookbook.rst @@ -45,7 +45,7 @@ The cache key expires after some time in case something unexpected happens feed_url_digest = md5(feed_url).hexdigest() lock_id = '{0}-lock-{1}'.format(self.name, feed_url_hexdigest) - # cache.add fails if if the key already exists + # cache.add fails if the key already exists acquire_lock = lambda: cache.add(lock_id, 'true', LOCK_EXPIRE) # memcache delete is very slow, but we have to use it to take # advantage of using add() for atomic locking From 8a812a8a13dadf06d60eade8b7c4d17c928a4687 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 15 Apr 2014 22:16:20 +0100 Subject: [PATCH 0069/1103] Bundle celery[librabbitmq] now depends on librabbitmq 1.5.0 --- requirements/extras/librabbitmq.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/librabbitmq.txt b/requirements/extras/librabbitmq.txt index e3ac1690b..8f9a2dbca 100644 --- a/requirements/extras/librabbitmq.txt +++ b/requirements/extras/librabbitmq.txt @@ -1 +1 @@ -librabbitmq>=1.0.2 +librabbitmq>=1.5.0 From a39745199900a8f69998af7de494d496c3878594 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 15 Apr 2014 22:16:54 +0100 Subject: [PATCH 0070/1103] Now depends on kombu 3.0.15 --- requirements/default.txt | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/default.txt b/requirements/default.txt index 9895c2c55..0573e01dc 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,3 +1,3 @@ pytz>dev billiard>=3.3.0.14,<3.4 -kombu>=3.0.14,<4.0 +kombu>=3.0.15,<4.0 diff --git a/setup.cfg b/setup.cfg index c8da3bd33..31cfaeaa4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,4 +12,4 @@ upload-dir = docs/.build/html [bdist_rpm] requires = pytz >= 2011b billiard >= 3.3.0.14 - kombu >= 3.0.14 + kombu >= 3.0.15 From 33fbec9f4d6b6b4d6a63f838e13800dd142f1131 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 16 Apr 2014 14:18:50 +0100 Subject: [PATCH 0071/1103] Now depends on billiard 3.3.0.17 --- requirements/default.txt | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/default.txt b/requirements/default.txt index 0573e01dc..da64babcf 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,3 +1,3 @@ pytz>dev -billiard>=3.3.0.14,<3.4 +billiard>=3.3.0.17,<3.4 kombu>=3.0.15,<4.0 diff --git a/setup.cfg b/setup.cfg index 31cfaeaa4..2a032e4d1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,5 +11,5 @@ upload-dir = docs/.build/html [bdist_rpm] requires = pytz >= 2011b - billiard >= 3.3.0.14 + billiard >= 3.3.0.17 kombu >= 3.0.15 From 40d9c8f652787c89505f64aac415c6f125708346 Mon Sep 17 00:00:00 2001 From: Brian Bouterse Date: Thu, 10 Apr 2014 13:43:43 -0400 Subject: [PATCH 0072/1103] Stops MongoDB Backend from using BROKER_USE_SSL. This change effectively disables SSL as set using BROKER_USE_SSL. SSL support should be functional through existing mechanisms, but requires doc changes and testing. I've documented this potential feature and the necessary work with celery/celery#1974. --- celery/backends/mongodb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index c3229d51c..44c1c2252 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -92,7 +92,6 @@ def __init__(self, *args, **kwargs): self.options = dict(config, **config.pop('options', None) or {}) # Set option defaults - self.options.setdefault('ssl', self.app.conf.BROKER_USE_SSL) self.options.setdefault('max_pool_size', self.max_pool_size) self.options.setdefault('auto_start_request', False) From c24415d5496acafe86a37be69bb4bb51a9fcf336 Mon Sep 17 00:00:00 2001 From: Brian Bouterse Date: Thu, 10 Apr 2014 14:20:26 -0400 Subject: [PATCH 0073/1103] Add Brian Bouterse to AUTHORS and CONTRIBUTORS Conflicts: CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + docs/AUTHORS.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index ccfe96062..edf2f120a 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -159,3 +159,4 @@ Paul Kilgo, 2014/01/28 Martin Davidsson, 2014/02/08 Chris Clark, 2014/02/20 Matthew Duggan, 2014/04/10 +Brian Bouterse, 2014/04/10 diff --git a/docs/AUTHORS.txt b/docs/AUTHORS.txt index 3d53ce911..8caea46a7 100644 --- a/docs/AUTHORS.txt +++ b/docs/AUTHORS.txt @@ -21,6 +21,7 @@ Ben Firshman Brad Jasper Branko Čibej Brendon Crawford +Brian Bouterse Brian Rosner Bryan Berg Chase Seibert From 840b0c683660ac37705bae90bb95179d981c0c06 Mon Sep 17 00:00:00 2001 From: Brian Bouterse Date: Tue, 15 Apr 2014 10:12:31 -0400 Subject: [PATCH 0074/1103] Adding changelog entry --- Changelog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Changelog b/Changelog index 92b8dd506..7726a13f8 100644 --- a/Changelog +++ b/Changelog @@ -15,6 +15,11 @@ new in Celery 3.1. :release-date: 2014-04-XX XX:XX X.X UTC :release-by: XXX +- **MongoDB**: SSL configuration with non-MongoDB transport breaks MongoDB + results backend (Issue #1973). + + Fix contributed by Brian Bouterse. + .. _version-3.1.10: 3.1.10 From f4658e6082945ec52f30bccabd555981a6707a3d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 16 Apr 2014 18:44:10 +0100 Subject: [PATCH 0075/1103] Fix for chord inside chord when using json serializer. Closes #1987 --- celery/backends/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 437dd4c83..41ce1ef17 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -167,8 +167,10 @@ def exception_to_python(self, exc): """Convert serialized exception to Python exception.""" if self.serializer in EXCEPTION_ABLE_CODECS: return get_pickled_exception(exc) - return create_exception_cls( - from_utf8(exc['exc_type']), __name__)(exc['exc_message']) + elif not isinstance(exc, BaseException): + return create_exception_cls( + from_utf8(exc['exc_type']), __name__)(exc['exc_message']) + return exc def prepare_value(self, result): """Prepare value for storage.""" From ede043f9bb4a9c2461552db7c967645ab15d5a85 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 16 Apr 2014 18:53:26 +0100 Subject: [PATCH 0076/1103] Tests passing; --- celery/tests/backends/test_mongodb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index a32d9ed27..f3449f793 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -98,7 +98,7 @@ def test_get_connection_no_connection_host(self): connection = self.backend._get_connection() mock_Connection.assert_called_once_with( - host='mongodb://localhost:27017', ssl=False, max_pool_size=10, + host='mongodb://localhost:27017', max_pool_size=10, auto_start_request=False) self.assertEqual(sentinel.connection, connection) @@ -113,7 +113,7 @@ def test_get_connection_no_connection_mongodb_uri(self): connection = self.backend._get_connection() mock_Connection.assert_called_once_with( - host=mongodb_uri, ssl=False, max_pool_size=10, + host=mongodb_uri, max_pool_size=10, auto_start_request=False) self.assertEqual(sentinel.connection, connection) From ec011029846a23f7a0e932190a6eb4d62ac7f197 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 16 Apr 2014 20:23:10 +0100 Subject: [PATCH 0077/1103] Regression: DEFAULT_DELIVERY_MODE no longer honored, also use Exchange.delivery_mode if set. Closes #1953 --- celery/app/amqp.py | 6 ++++++ docs/userguide/optimizing.rst | 13 +++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 6db1d7315..a23f1d63b 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -385,6 +385,7 @@ def as_task_v1(self, task_id, name, args=None, kwargs=None, def _create_task_sender(self): default_retry = self.app.conf.CELERY_TASK_PUBLISH_RETRY default_policy = self.app.conf.CELERY_TASK_PUBLISH_RETRY_POLICY + default_delivery_mode = self.app.conf.CELERY_DEFAULT_DELIVERY_MODE default_queue = self.default_queue queues = self.queues send_before_publish = signals.before_task_publish.send @@ -421,6 +422,11 @@ def publish_task(producer, name, message, qname, queue = queue, queues[queue] else: qname = queue.name + if delivery_mode is None: + try: + delivery_mode = queue.exchange.delivery_mode + except AttributeError: + delivery_mode = default_delivery_mode exchange = exchange or queue.exchange.name routing_key = routing_key or queue.routing_key if declare is None and queue and not isinstance(queue, Broadcast): diff --git a/docs/userguide/optimizing.rst b/docs/userguide/optimizing.rst index 459069fd2..e5ab4b312 100644 --- a/docs/userguide/optimizing.rst +++ b/docs/userguide/optimizing.rst @@ -98,11 +98,20 @@ to improve performance: CELERY_QUEUES = ( Queue('celery', routing_key='celery'), - Queue('transient', routing_key='transient', - delivery_mode=1), + Queue('transient', Exchange('transient', delivery_mode=1), + routing_key='transient', durable=False), ) +or by using :setting:`CELERY_ROUTES`: + +.. code-block:: python + + CELERY_ROUTES = { + 'proj.tasks.add': {'queue': 'celery', 'delivery_mode': 'transient'} + } + + The ``delivery_mode`` changes how the messages to this queue are delivered. A value of 1 means that the message will not be written to disk, and a value of 2 (default) means that the message can be written to disk. From d028eed5115e03967dfa3fe273fd8c71818a2b0a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 16 Apr 2014 22:30:15 +0100 Subject: [PATCH 0078/1103] Renames celery.app.builtins.shared_task -> celery._state.connect_on_app_finalize. Closes #1937 --- celery/_state.py | 19 +++++++++++++++- celery/app/__init__.py | 9 ++++---- celery/app/base.py | 12 +++++----- celery/app/builtins.py | 43 +++++++++--------------------------- celery/app/task.py | 4 ++-- celery/tests/app/test_app.py | 2 +- 6 files changed, 41 insertions(+), 48 deletions(-) diff --git a/celery/_state.py b/celery/_state.py index e599a0fb9..a76b3f4b7 100644 --- a/celery/_state.py +++ b/celery/_state.py @@ -20,7 +20,8 @@ from celery.utils.threads import LocalStack __all__ = ['set_default_app', 'get_current_app', 'get_current_task', - 'get_current_worker_task', 'current_app', 'current_task'] + 'get_current_worker_task', 'current_app', 'current_task', + 'connect_on_app_finalize'] #: Global default app used when no current app. default_app = None @@ -28,9 +29,25 @@ #: List of all app instances (weakrefs), must not be used directly. _apps = weakref.WeakSet() +#: global set of functions to call whenever a new app is finalized +#: E.g. Shared tasks, and builtin tasks are created +#: by adding callbacks here. +_on_app_finalizers = set() + _task_join_will_block = False +def connect_on_app_finalize(callback): + _on_app_finalizers.add(callback) + return callback + + +def _announce_app_finalized(app): + callbacks = set(_on_app_finalizers) + for callback in callbacks: + callback(app) + + def _set_task_join_will_block(blocks): global _task_join_will_block _task_join_will_block = blocks diff --git a/celery/app/__init__.py b/celery/app/__init__.py index 426fed6ce..952a8746d 100644 --- a/celery/app/__init__.py +++ b/celery/app/__init__.py @@ -13,15 +13,12 @@ from celery.local import Proxy from celery import _state from celery._state import ( - set_default_app, get_current_app as current_app, get_current_task as current_task, - _get_active_apps, - _task_stack, + connect_on_app_finalize, set_default_app, _get_active_apps, _task_stack, ) from celery.utils import gen_task_name -from .builtins import shared_task as _shared_task from .base import Celery, AppPickler __all__ = ['Celery', 'AppPickler', 'default_app', 'app_or_default', @@ -128,7 +125,9 @@ def __inner(fun): name = options.get('name') # Set as shared task so that unfinalized apps, # and future apps will load the task. - _shared_task(lambda app: app._task_from_fun(fun, **options)) + connect_on_app_finalize( + lambda app: app._task_from_fun(fun, **options) + ) # Force all finalized apps to take this task as well. for app in _get_active_apps(): diff --git a/celery/app/base.py b/celery/app/base.py index a134cb84c..22e4a480b 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -27,7 +27,8 @@ from celery import signals from celery._state import ( _task_stack, get_current_app, _set_current_app, set_default_app, - _register_app, get_current_worker_task, + _register_app, get_current_worker_task, connect_on_app_finalize, + _announce_app_finalized, ) from celery.exceptions import AlwaysEagerIgnored, ImproperlyConfigured from celery.five import items, values @@ -38,7 +39,6 @@ from celery.utils.objects import mro_lookup from .annotations import prepare as prepare_annotations -from .builtins import shared_task, load_shared_tasks from .defaults import DEFAULTS, find_deprecated_settings from .registry import TaskRegistry from .utils import ( @@ -208,8 +208,8 @@ def task(self, *args, **opts): # a differnt task instance. This makes sure it will always use # the task instance from the current app. # Really need a better solution for this :( - from . import shared_task as proxies_to_curapp - return proxies_to_curapp(*args, _force_evaluate=True, **opts) + from . import shared_task + return shared_task(*args, _force_evaluate=True, **opts) def inner_create_task_cls(shared=True, filter=None, **opts): _filt = filter # stupid 2to3 @@ -218,7 +218,7 @@ def _create_task_cls(fun): if shared: cons = lambda app: app._task_from_fun(fun, **opts) cons.__name__ = fun.__name__ - shared_task(cons) + connect_on_app_finalize(cons) if self.accept_magic_kwargs: # compat mode task = self._task_from_fun(fun, **opts) if filter: @@ -271,7 +271,7 @@ def finalize(self, auto=False): if auto and not self.autofinalize: raise RuntimeError('Contract breach: app not finalized') self.finalized = True - load_shared_tasks(self) + _announce_app_finalized(self) pending = self._pending while pending: diff --git a/celery/app/builtins.py b/celery/app/builtins.py index 9240537cf..e42e0b25c 100644 --- a/celery/app/builtins.py +++ b/celery/app/builtins.py @@ -11,39 +11,16 @@ from collections import deque -from celery._state import get_current_worker_task +from celery._state import get_current_worker_task, connect_on_app_finalize from celery.utils import uuid from celery.utils.log import get_logger -__all__ = ['shared_task', 'load_shared_tasks'] +__all__ = [] logger = get_logger(__name__) -#: global list of functions defining tasks that should be -#: added to all apps. -_shared_tasks = set() - -def shared_task(constructor): - """Decorator that specifies a function that generates a built-in task. - - The function will then be called for every new app instance created - (lazily, so more exactly when the task registry for that app is needed). - - The function must take a single ``app`` argument. - """ - _shared_tasks.add(constructor) - return constructor - - -def load_shared_tasks(app): - """Create built-in tasks for an app instance.""" - constructors = set(_shared_tasks) - for constructor in constructors: - constructor(app) - - -@shared_task +@connect_on_app_finalize def add_backend_cleanup_task(app): """The backend cleanup task can be used to clean up the default result backend. @@ -60,7 +37,7 @@ def backend_cleanup(): return backend_cleanup -@shared_task +@connect_on_app_finalize def add_unlock_chord_task(app): """This task is used by result backends without native chord support. @@ -127,7 +104,7 @@ def unlock_chord(group_id, callback, interval=None, propagate=None, return unlock_chord -@shared_task +@connect_on_app_finalize def add_map_task(app): from celery.canvas import signature @@ -138,7 +115,7 @@ def xmap(task, it): return xmap -@shared_task +@connect_on_app_finalize def add_starmap_task(app): from celery.canvas import signature @@ -149,7 +126,7 @@ def xstarmap(task, it): return xstarmap -@shared_task +@connect_on_app_finalize def add_chunk_task(app): from celery.canvas import chunks as _chunks @@ -159,7 +136,7 @@ def chunks(task, it, n): return chunks -@shared_task +@connect_on_app_finalize def add_group_task(app): _app = app from celery.canvas import maybe_signature, signature @@ -226,7 +203,7 @@ def apply(self, args=(), kwargs={}, **options): return Group -@shared_task +@connect_on_app_finalize def add_chain_task(app): from celery.canvas import ( Signature, chain, chord, group, maybe_signature, maybe_unroll_group, @@ -322,7 +299,7 @@ def apply(self, args=(), kwargs={}, signature=maybe_signature, return Chain -@shared_task +@connect_on_app_finalize def add_chord_task(app): """Every chord is executed in a dedicated task, so that the chord can be used as a signature, and this generates the task diff --git a/celery/app/task.py b/celery/app/task.py index e8a4697a2..b20974424 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -176,14 +176,14 @@ def __new__(cls, name, bases, attrs): # Hairy stuff, here to be compatible with 2.x. # People should not use non-abstract task classes anymore, # use the task decorator. - from celery.app.builtins import shared_task + from celery._state import connect_on_app_finalize unique_name = '.'.join([task_module, name]) if unique_name not in cls._creation_count: # the creation count is used as a safety # so that the same task is not added recursively # to the set of constructors. cls._creation_count[unique_name] = 1 - shared_task(_CompatShared( + connect_on_app_finalize(_CompatShared( unique_name, lambda app: TaskType.__new__(cls, name, bases, dict(attrs, _app=app)), diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index 36cdb67a2..5bb1ef61e 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -251,7 +251,7 @@ def foo(shared=False): _state._task_stack.pop() def test_task_not_shared(self): - with patch('celery.app.base.shared_task') as sh: + with patch('celery.app.base.connect_on_app_finalize') as sh: @self.app.task(shared=False) def foo(): pass From 6f3dd4b512459855036c1e51eb252f71add55a02 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 16 Apr 2014 23:40:28 +0100 Subject: [PATCH 0079/1103] Updates Changelog --- Changelog | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/Changelog b/Changelog index 7726a13f8..a44812cd0 100644 --- a/Changelog +++ b/Changelog @@ -12,14 +12,77 @@ new in Celery 3.1. 3.1.11 ====== -:release-date: 2014-04-XX XX:XX X.X UTC -:release-by: XXX +:release-date: 2014-04-16 11:00 P.M UTC +:release-by: Ask Solem + +- Now compatible with RabbitMQ 3.3.0 + + You need to run Celery 3.1.11 or later when using RabbitMQ 3.3, + and if you use the ``librabbitmq`` module you also have to upgrade + to librabbitmq 1.5.0: + + .. code-block:: bash + + $ pip install -U librabbitmq + +- **Requirements**: + + - Now depends on :ref:`Kombu 3.0.15 `. + + - Now depends on `billiard 3.3.0.17`_. + + - Bundle ``celery[librabbitmq]`` now depends on :mod:`librabbitmq` 1.5.0. + +.. _`billiard 3.3.0.17`: + https://github.com/celery/billiard/blob/master/CHANGES.txt + +- **Tasks**: The :setting:`CELERY_DEFAULT_DELIVERY_MODE` setting was being + ignored (Issue #1953). + +- **Worker**: New :option:`--heartbeat-interval` can be used to change the + time (in seconds) between sending event heartbeats. + + Contributed by Matthew Duggan and Craig Northway. + +- **App**: Fixed memory leaks occurring when creating lots of temporary + app instances (Issue #1949). - **MongoDB**: SSL configuration with non-MongoDB transport breaks MongoDB results backend (Issue #1973). Fix contributed by Brian Bouterse. +- **Logging**: The color formatter accidentally modified ``record.msg`` + (Issue #1939). + +- **Results**: Fixed problem with task trails being stored multiple times, + causing ``result.collect()`` to hang (Issue #1936, Issue #1943). + +- **Results**: ``ResultSet`` now implements a ``.backend`` attribute for + compatibility with ``AsyncResult``. + +- **Results**: ``.forget()`` now also clears the local cache. + +- **Results**: Fixed problem with multiple calls to ``result._set_cache`` + (Issue #1940). + +- **Results**: ``join_native`` populated result cache even if disabled. + +- **Results**: The YAML result serializer should now be able to handle storing + exceptions. + +- **Worker**: No longer sends task error emails for expected errors (in + ``@task(throws=(..., )))``. + +- **Canvas**: Fixed problem with exception deserialization when using + the JSON serializer (Issue #1987). + +- **Eventlet**: Fixes crash when ``celery.contrib.batches`` attempted to + cancel a non-existing timer (Issue #1984). + +- Can now import ``celery.version_info_t``, and ``celery.five`` (Issue #1968). + + .. _version-3.1.10: 3.1.10 From b05df838c07f1f88fb755ff9bd1bad376a7821fe Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 17 Apr 2014 00:33:24 +0100 Subject: [PATCH 0080/1103] Must import builtins --- celery/app/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/celery/app/base.py b/celery/app/base.py index 22e4a480b..1a1300ca1 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -45,6 +45,9 @@ AppPickler, Settings, bugreport, _unpickle_app, _unpickle_app_v2, appstr, ) +# Load all builtin tasks +from . import builtins # noqa + __all__ = ['Celery'] _EXECV = os.environ.get('FORKED_BY_MULTIPROCESSING') From 68ac8968a4eabac95a424297b51e3c46a4b4c7aa Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 17 Apr 2014 00:45:59 +0100 Subject: [PATCH 0081/1103] Changelog cosmetics --- Changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog b/Changelog index a44812cd0..7c7e35165 100644 --- a/Changelog +++ b/Changelog @@ -15,7 +15,7 @@ new in Celery 3.1. :release-date: 2014-04-16 11:00 P.M UTC :release-by: Ask Solem -- Now compatible with RabbitMQ 3.3.0 +- **Now compatible with RabbitMQ 3.3.0** You need to run Celery 3.1.11 or later when using RabbitMQ 3.3, and if you use the ``librabbitmq`` module you also have to upgrade From 2a0ff2641b081a7b6a43ba6e3d0533dd5f82d393 Mon Sep 17 00:00:00 2001 From: ffeast Date: Thu, 17 Apr 2014 13:55:30 +0400 Subject: [PATCH 0082/1103] removed centos/celeryd CELERYD_CHDIR quoting --- extra/centos/celeryd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/centos/celeryd b/extra/centos/celeryd index 8b43b6112..879a99f63 100644 --- a/extra/centos/celeryd +++ b/extra/centos/celeryd @@ -95,7 +95,7 @@ if [ -n "$CELERYD_GROUP" ]; then fi if [ -n "$CELERYD_CHDIR" ]; then - DAEMON_OPTS="$DAEMON_OPTS --workdir=\"$CELERYD_CHDIR\"" + DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYD_CHDIR" fi check_dev_null() { From 4f5e8b80b77d867546dd7bede5ef71536d87dcd5 Mon Sep 17 00:00:00 2001 From: ocean1 Date: Sat, 12 Apr 2014 01:07:14 +0200 Subject: [PATCH 0083/1103] add a groupmeta_collection option to save group results in a different collection than results and add a 'bson' fake kombu encoder to allow pymongo to serialize natively data in mongodb --- celery/backends/mongodb.py | 107 ++++++++++++++++++++++++++++++++----- 1 file changed, 95 insertions(+), 12 deletions(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index c3229d51c..70af35aae 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -20,21 +20,48 @@ from bson.binary import Binary except ImportError: # pragma: no cover from pymongo.binary import Binary # noqa + from pymongo.errors import InvalidDocument # noqa else: # pragma: no cover Binary = None # noqa + InvalidDocument = None # noqa from kombu.syn import detect_environment from kombu.utils import cached_property - +from kombu.exceptions import EncodeError +from kombu.serialization import register, disable_insecure_serializers from celery import states from celery.exceptions import ImproperlyConfigured from celery.five import string_t from celery.utils.timeutils import maybe_timedelta +from celery.result import AsyncResult from .base import BaseBackend __all__ = ['MongoBackend'] +BINARY_CODECS = frozenset(['pickle','msgpack']) + +#register a fake bson serializer which will return the document as it is +class bson_serializer(): + @staticmethod + def loads(obj, *args, **kwargs): + if isinstance(obj,string_t): + try: + from anyjson import loads + return loads(obj) + except: + pass + return obj + + @staticmethod + def dumps(obj, *args, **kwargs): + return obj + +register('bson', bson_serializer.loads, bson_serializer.dumps, + content_type='application/data', + content_encoding='utf-8') + +disable_insecure_serializers(['json','bson']) class Bunch(object): @@ -43,6 +70,7 @@ def __init__(self, **kw): class MongoBackend(BaseBackend): + host = 'localhost' port = 27017 user = None @@ -64,10 +92,16 @@ def __init__(self, *args, **kwargs): """ self.options = {} + super(MongoBackend, self).__init__(*args, **kwargs) self.expires = kwargs.get('expires') or maybe_timedelta( self.app.conf.CELERY_TASK_RESULT_EXPIRES) + # little hack to get over standard kombu loads because + # mongo return strings which don't get decoded! + if self.serializer == 'bson': + self.decode = self.decode_bson + if not pymongo: raise ImproperlyConfigured( 'You need to install the pymongo library to use the ' @@ -88,6 +122,9 @@ def __init__(self, *args, **kwargs): self.taskmeta_collection = config.pop( 'taskmeta_collection', self.taskmeta_collection, ) + self.groupmeta_collection = config.pop( + 'groupmeta_collection', self.taskmeta_collection, + ) self.options = dict(config, **config.pop('options', None) or {}) @@ -101,6 +138,7 @@ def __init__(self, *args, **kwargs): # Specifying backend as an URL self.host = url + def _get_connection(self): """Connect to the MongoDB server.""" if self._connection is None: @@ -132,25 +170,50 @@ def process_cleanup(self): del(self.database) self._connection = None + def encode(self, data): + payload = super(MongoBackend, self).encode(data) + #serializer which are in a unsupported format (pickle/binary) + if self.serializer in BINARY_CODECS: + payload = Binary(payload) + + return payload + + def decode_bson(self, data): + return bson_serializer.loads(data) + + def encode_result(self, result, status): + if status in self.EXCEPTION_STATES and isinstance(result, Exception): + return self.prepare_exception(result) + else: + return self.prepare_value(result) + def _store_result(self, task_id, result, status, traceback=None, request=None, **kwargs): """Store return value and status of an executed task.""" + meta = {'_id': task_id, 'status': status, - 'result': Binary(self.encode(result)), + 'result': self.encode(result), 'date_done': datetime.utcnow(), - 'traceback': Binary(self.encode(traceback)), - 'children': Binary(self.encode( + 'traceback': self.encode(traceback), + 'children': self.encode( self.current_task_children(request), - ))} - self.collection.save(meta) + )} + + try: + self.collection.save(meta) + except InvalidDocument as exc: + raise EncodeError(exc) return result def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" - obj = self.collection.find_one({'_id': task_id}) + # if collection don't contain it try searching in the + # group_collection it could be a groupresult instead + obj = self.collection.find_one({'_id': task_id}) or \ + self.group_collection.find_one({'_id': task_id}) if not obj: return {'status': states.PENDING, 'result': None} @@ -167,22 +230,29 @@ def _get_task_meta_for(self, task_id): def _save_group(self, group_id, result): """Save the group result.""" + + task_ids = [ i.id for i in result ] + meta = {'_id': group_id, - 'result': Binary(self.encode(result)), + 'result': self.encode(task_ids), 'date_done': datetime.utcnow()} - self.collection.save(meta) + self.group_collection.save(meta) return result def _restore_group(self, group_id): """Get the result for a group by id.""" - obj = self.collection.find_one({'_id': group_id}) + obj = self.group_collection.find_one({'_id': group_id}) if not obj: return + tasks = self.decode(obj['result']) + + tasks = [ AsyncResult(task) for task in tasks ] + meta = { 'task_id': obj['_id'], - 'result': self.decode(obj['result']), + 'result': tasks, 'date_done': obj['date_done'], } @@ -190,7 +260,7 @@ def _restore_group(self, group_id): def _delete_group(self, group_id): """Delete a group by id.""" - self.collection.remove({'_id': group_id}) + self.group_collection.remove({'_id': group_id}) def _forget(self, task_id): """ @@ -209,6 +279,9 @@ def cleanup(self): self.collection.remove( {'date_done': {'$lt': self.app.now() - self.expires}}, ) + self.group_collection.remove( + {'date_done': {'$lt': self.app.now() - self.expires}}, + ) def __reduce__(self, args=(), kwargs={}): kwargs.update( @@ -240,3 +313,13 @@ def collection(self): # in the background. Once completed cleanup will be much faster collection.ensure_index('date_done', background='true') return collection + + @cached_property + def group_collection(self): + """Get the metadata task collection.""" + collection = self.database[self.groupmeta_collection] + + # Ensure an index on date_done is there, if not process the index + # in the background. Once completed cleanup will be much faster + collection.ensure_index('date_done', background='true') + return collection From 6373b5a8e0dee9faeabcc48b721c744816fae830 Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Thu, 17 Apr 2014 17:08:37 -0500 Subject: [PATCH 0084/1103] Correct import in security docs. --- docs/userguide/security.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/security.rst b/docs/userguide/security.rst index 027ad5489..ef3cd9635 100644 --- a/docs/userguide/security.rst +++ b/docs/userguide/security.rst @@ -168,7 +168,7 @@ with the private key and certificate files located in `/etc/ssl`. CELERY_SECURITY_KEY = '/etc/ssl/private/worker.key' CELERY_SECURITY_CERTIFICATE = '/etc/ssl/certs/worker.pem' CELERY_SECURITY_CERT_STORE = '/etc/ssl/certs/*.pem' - from celery import setup_security + from celery.security import setup_security setup_security() .. note:: From 2edea37f6aeb37b7ceb150c4ebc9cfbf85fdefa2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 18 Apr 2014 15:43:51 +0100 Subject: [PATCH 0085/1103] Update AbortableTask docs. Closes #1993 --- celery/contrib/abortable.py | 67 +++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/celery/contrib/abortable.py b/celery/contrib/abortable.py index 37dc30d92..dcdc61566 100644 --- a/celery/contrib/abortable.py +++ b/celery/contrib/abortable.py @@ -28,49 +28,52 @@ .. code-block:: python - from celery.contrib.abortable import AbortableTask - from celery.utils.log import get_task_logger - - logger = get_logger(__name__) - - class MyLongRunningTask(AbortableTask): - - def run(self, **kwargs): - results = [] - for x in range(100): - # Check after every 5 loops.. - if x % 5 == 0: # alternatively, check when some timer is due - if self.is_aborted(**kwargs): - # Respect the aborted status and terminate - # gracefully - logger.warning('Task aborted.') - return - y = do_something_expensive(x) - results.append(y) - logger.info('Task finished.') - return results - + from __future__ import absolute_import + + from celery.contrib.abortable import AbortableTask + from celery.utils.log import get_task_logger + + from proj.celery import app + + logger = get_logger(__name__) + + @app.task(bind=True, base=AbortableTask) + def long_running_task(self): + results = [] + for i in range(100): + # check after every 5 iterations... + # (or alternatively, check when some timer is due) + if not i % 5: + if self.is_aborted(): + # respect aborted state, and terminate gracefully. + logger.warning('Task aborted') + return + value = do_something_expensive(i) + results.append(y) + logger.info('Task complete') + return results In the producer: .. code-block:: python - from myproject.tasks import MyLongRunningTask + from __future__ import absolute_import - def myview(request): + import time - async_result = MyLongRunningTask.delay() - # async_result is of type AbortableAsyncResult + from proj.tasks import MyLongRunningTask - # After 10 seconds, abort the task - time.sleep(10) - async_result.abort() + def myview(request): + # result is of type AbortableAsyncResult + result = long_running_task.delay() - ... + # abort the task after 10 seconds + time.sleep(10) + result.abort() -After the `async_result.abort()` call, the task execution is not +After the `result.abort()` call, the task execution is not aborted immediately. In fact, it is not guaranteed to abort at all. Keep -checking the `async_result` status, or call `async_result.wait()` to +checking `result.state` status, or call `result.get(timeout=)` to have it block until the task is finished. .. note:: From 5a3014663c90e94a846a495f9619fb1ff8cdd30b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 22 Apr 2014 14:48:52 +0100 Subject: [PATCH 0086/1103] Adds flower to intersphinx --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 914aee712..2cee3992a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -81,6 +81,7 @@ def linkcode_resolve(domain, info): 'djcelery': ('http://django-celery.readthedocs.org/en/latest', None), 'cyme': ('http://cyme.readthedocs.org/en/latest', None), 'amqp': ('http://amqp.readthedocs.org/en/latest', None), + 'flower': ('http://flower.readthedocs.org/en/latest', None), } # The name of the Pygments (syntax highlighting) style to use. From 0fe113513bd5779115d5ec80fccdadf77ef9543c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 23 Apr 2014 15:45:24 +0100 Subject: [PATCH 0087/1103] Fixes broken rst ref. Closes #1998 --- docs/userguide/calling.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index 5d2150cbb..bfddf408a 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -318,7 +318,12 @@ or for each individual task, or even per message. There's built-in support for :mod:`pickle`, `JSON`, `YAML` and `msgpack`, and you can also add your own custom serializers by registering -them into the Kombu serializer registry (see ref:`kombu:guide-serialization`). +them into the Kombu serializer registry + +.. seealso:: + + :ref:`Message Serialization ` in the Kombu user + guide. Each option has its advantages and disadvantages. From c9e217a6a881393076b02ac6a733fa2b1f04fbd3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 23 Apr 2014 16:02:10 +0100 Subject: [PATCH 0088/1103] Travis: Only get IRC notification when build fixed and failed --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3690f624f..e6b5146be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,5 +37,5 @@ notifications: irc: channels: - "chat.freenode.net#celery" - on_success: always + on_success: change on_failure: always From 04a12eb1ff5d83bd8df0d6acfdd40486724c138a Mon Sep 17 00:00:00 2001 From: ffeast Date: Tue, 22 Apr 2014 13:29:58 +0400 Subject: [PATCH 0089/1103] Support of missing CELERY_BIN and CELERY_APP, described in http://celery.readthedocs.org/en/latest/tutorials/daemonizing.html#id11 --- extra/centos/celeryd | 3 ++- extra/centos/celeryd.sysconfig | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/extra/centos/celeryd b/extra/centos/celeryd index 879a99f63..c5e3b555c 100644 --- a/extra/centos/celeryd +++ b/extra/centos/celeryd @@ -71,7 +71,7 @@ if [ -z "$CELERYD_LOG_FILE" ]; then fi CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} -CELERYD_MULTI=${CELERYD_MULTI:-"celeryd-multi"} +CELERYD_MULTI=${CELERYD_MULTI:-"${CELERY_BIN} multi"} CELERYD=${CELERYD:-$DEFAULT_CELERYD} CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES} @@ -85,6 +85,7 @@ fi CELERYD_LOG_DIR=`dirname $CELERYD_LOG_FILE` CELERYD_PID_DIR=`dirname $CELERYD_PID_FILE` +CELERYD_OPTS=${CELERYD_OPTS:-"--app=$CELERY_APP"} # Extra start-stop-daemon options, like user/group. if [ -n "$CELERYD_USER" ]; then diff --git a/extra/centos/celeryd.sysconfig b/extra/centos/celeryd.sysconfig index e1d98bd4d..c6f2d54c6 100644 --- a/extra/centos/celeryd.sysconfig +++ b/extra/centos/celeryd.sysconfig @@ -1,4 +1,5 @@ # In CentOS, contents should be placed in the file /etc/sysconfig/celeryd +# Available options: http://celery.readthedocs.org/en/latest/tutorials/daemonizing.html#available-options # Names of nodes to start (space-separated) #CELERYD_NODES="my_application-node_1" @@ -6,11 +7,11 @@ # Where to chdir at start. This could be the root of a virtualenv. #CELERYD_CHDIR="/path/to/my_application" -# How to call celeryd-multi -#CELERYD_MULTI="$CELERYD_CHDIR/bin/celeryd-multi" +# Absolute or relative path to the celery program +#CELERY_BIN="/usr/local/bin/celery" -# Extra arguments -#CELERYD_OPTS="--app=my_application.path.to.worker --time-limit=300 --concurrency=8 --loglevel=DEBUG" +# App instance to use (value for --app argument). +#CELERY_APP="my_application" # Create log/pid dirs, if they don't already exist #CELERY_CREATE_DIRS=1 From c67fae736eb485bfabd36bb3406fa9e706bb94ea Mon Sep 17 00:00:00 2001 From: ffeast Date: Tue, 22 Apr 2014 16:01:27 +0400 Subject: [PATCH 0090/1103] Added centos celerybeat init script + sysconfig example + simple tests --- extra/centos/celerybeat | 239 ++++++++++++++++++++++++++++++ extra/centos/celerybeat.sysconfig | 15 ++ extra/centos/test_celerybeat.sh | 6 + extra/centos/test_celeryd.sh | 37 +---- extra/centos/test_service.sh | 43 ++++++ 5 files changed, 304 insertions(+), 36 deletions(-) create mode 100644 extra/centos/celerybeat create mode 100644 extra/centos/celerybeat.sysconfig create mode 100755 extra/centos/test_celerybeat.sh create mode 100755 extra/centos/test_service.sh diff --git a/extra/centos/celerybeat b/extra/centos/celerybeat new file mode 100644 index 000000000..b51ab0762 --- /dev/null +++ b/extra/centos/celerybeat @@ -0,0 +1,239 @@ +#!/bin/sh +# ============================================ +# celerybeat - Starts the Celery periodic task scheduler. +# ============================================ +# +# :Usage: /etc/init.d/celerybeat {start|stop|restart|status} +# :Configuration file: /etc/sysconfig/celerybeat +# +# See http://docs.celeryproject.org/en/latest/tutorials/daemonizing.html + +### BEGIN INIT INFO +# Provides: celerybeat +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery task worker daemon +### END INIT INFO +# +# +# To implement separate init scripts, do NOT copy this script. Instead, +# symlink it. I.e., if my new application, "little-worker" needs an init, I +# should just use: +# +# ln -s /etc/init.d/celerybeat /etc/init.d/little-worker +# +# You can then configure this by manipulating /etc/sysconfig/little-worker. +# +# Setting `prog` here allows you to symlink this init script, making it easy +# to run multiple processes on the system. + +# If we're invoked via SysV-style runlevel scripts we need to follow the +# link from rcX.d before working out the script name. +if [[ `dirname $0` == /etc/rc*.d ]]; then + target="$(readlink $0)" +else + target=$0 +fi + +prog="$(basename $target)" + +# Source the centos service helper functions +source /etc/init.d/functions +# NOTE: "set -e" does not work with the above functions, +# which use non-zero return codes as non-error return conditions + +# some commands work asyncronously, so we'll wait this many seconds +SLEEP_SECONDS=5 + +DEFAULT_PID_FILE="/var/run/celery/$prog.pid" +DEFAULT_LOG_FILE="/var/log/celery/$prog.log" +DEFAULT_LOG_LEVEL="INFO" +DEFAULT_NODES="celery" + +CELERY_DEFAULTS=${CELERY_DEFAULTS:-"/etc/sysconfig/$prog"} + +test -f "$CELERY_DEFAULTS" && . "$CELERY_DEFAULTS" + +# Set CELERY_CREATE_DIRS to always create log/pid dirs. +CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} +CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS +CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS +if [ -z "$CELERYBEAT_PID_FILE" ]; then + CELERYBEAT_PID_FILE="$DEFAULT_PID_FILE" + CELERY_CREATE_RUNDIR=1 +fi +if [ -z "$CELERYBEAT_LOG_FILE" ]; then + CELERYBEAT_LOG_FILE="$DEFAULT_LOG_FILE" + CELERY_CREATE_LOGDIR=1 +fi + +CELERYBEAT_LOG_LEVEL=${CELERYBEAT_LOG_LEVEL:-${CELERYBEAT_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} +CELERYBEAT=${CELERYBEAT:-"${CELERY_BIN} beat"} +CELERYBEAT=${CELERYBEAT:-$DEFAULT_CELERYBEAT} +CELERYBEAT_NODES=${CELERYBEAT_NODES:-$DEFAULT_NODES} + +# This is used to change how Celery loads in the configs. It does not need to +# be set to be run. +export CELERY_LOADER + +if [ -n "$2" ]; then + CELERYBEAT_OPTS="$CELERYBEAT_OPTS $2" +fi + +CELERYBEAT_OPTS=${CELERYBEAT_OPTS:-"--app=$CELERY_APP"} +CELERYBEAT_LOG_DIR=`dirname $CELERYBEAT_LOG_FILE` +CELERYBEAT_PID_DIR=`dirname $CELERYBEAT_PID_FILE` + +# Extra start-stop-daemon options, like user/group. +if [ -n "$CELERYBEAT_USER" ]; then + DAEMON_OPTS="$DAEMON_OPTS --uid=$CELERYBEAT_USER" +fi +if [ -n "$CELERYBEAT_GROUP" ]; then + DAEMON_OPTS="$DAEMON_OPTS --gid=$CELERYBEAT_GROUP" +fi + +if [ -n "$CELERYBEAT_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYBEAT_CHDIR" +fi + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 75 # EX_TEMPFAIL + fi +} + + +maybe_die() { + if [ $? -ne 0 ]; then + echo "Exiting: $* (errno $?)" + exit 77 # EX_NOPERM + fi +} + +create_default_dir() { + if [ ! -d "$1" ]; then + echo "- Creating default directory: '$1'" + mkdir -p "$1" + maybe_die "Couldn't create directory $1" + echo "- Changing permissions of '$1' to 02755" + chmod 02755 "$1" + maybe_die "Couldn't change permissions for $1" + if [ -n "$CELERYBEAT_USER" ]; then + echo "- Changing owner of '$1' to '$CELERYBEAT_USER'" + chown "$CELERYBEAT_USER" "$1" + maybe_die "Couldn't change owner of $1" + fi + if [ -n "$CELERYBEAT_GROUP" ]; then + echo "- Changing group of '$1' to '$CELERYBEAT_GROUP'" + chgrp "$CELERYBEAT_GROUP" "$1" + maybe_die "Couldn't change group of $1" + fi + fi +} + + +check_paths() { + if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then + create_default_dir "$CELERYBEAT_LOG_DIR" + fi + if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then + create_default_dir "$CELERYBEAT_PID_DIR" + fi +} + +create_paths() { + create_default_dir "$CELERYBEAT_LOG_DIR" + create_default_dir "$CELERYBEAT_PID_DIR" +} + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + +stop() { + [[ ! -f "$CELERYBEAT_PID_FILE" ]] && echo "$prog is stopped" && return 0 + + local one_failed= + echo -n $"Stopping $prog: " + + # killproc comes from 'functions' and brings three nice features: + # 1. sending TERM, sleeping, then sleeping more if needed, then sending KILL + # 2. handling 'success' and 'failure' output + # 3. removes stale pid files, if any remain + killproc -p "$CELERYBEAT_PID_FILE" -d "$SLEEP_SECONDS" $prog || one_failed=true + echo + + [[ "$one_failed" ]] && return 1 || return 0 +} + +start() { + echo -n $"Starting $prog: " + + # If Celery is already running, bail out + if [[ -f "$CELERYBEAT_PID_FILE" ]]; then + echo -n "$prog is already running. Use 'restart'." + failure + echo + return 1 + fi + + $CELERYBEAT $CELERYBEAT_OPTS $DAEMON_OPTS --detach \ + --pidfile="$CELERYBEAT_PID_FILE" \ + --logfile="$CELERYBEAT_LOG_FILE" \ + --loglevel="$CELERYBEAT_LOG_LEVEL" + + if [[ "$?" == "0" ]]; then + # Sleep a few seconds to give Celery a chance to initialize itself. + # This is useful to prevent scripts following this one from trying to + # use Celery (or its pid files) too early. + sleep $SLEEP_SECONDS + if [[ -f "$CELERYBEAT_PID_FILE" ]]; then + success + echo + return 0 + else # celerybeat succeeded but no pid files found + failure + fi + else # celerybeat did not succeed + failure + fi + echo + return 1 +} + +check_status() { + status -p "$CELERYBEAT_PID_FILE" $"$prog" || return 1 + return 0 +} + +case "$1" in + start) + check_dev_null + check_paths + start + ;; + + stop) + check_dev_null + check_paths + stop + ;; + + status) + check_status + ;; + + restart) + check_dev_null + check_paths + stop && start + ;; + + *) + echo "Usage: /etc/init.d/$prog {start|stop|restart|status}" + exit 3 + ;; +esac + +exit $? diff --git a/extra/centos/celerybeat.sysconfig b/extra/centos/celerybeat.sysconfig new file mode 100644 index 000000000..50015151e --- /dev/null +++ b/extra/centos/celerybeat.sysconfig @@ -0,0 +1,15 @@ +# In CentOS, contents should be placed in the file /etc/sysconfig/celeryd +# Available options: http://celery.readthedocs.org/en/latest/tutorials/daemonizing.html#init-script-celerybeat + +# Where the Django project is. +#CELERYBEAT_CHDIR="/path/to/my_application" + +# Absolute or relative path to the celery program +#CELERY_BIN="/usr/local/bin/celery" + +# App instance to use (value for --app argument). +#CELERY_APP="my_application.path.to.worker" + +# Beat run as an unprivileged user +#CELERYBEAT_USER="brandings" +#CELERYBEAT_GROUP="brandings" diff --git a/extra/centos/test_celerybeat.sh b/extra/centos/test_celerybeat.sh new file mode 100755 index 000000000..d60829d2d --- /dev/null +++ b/extra/centos/test_celerybeat.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# If you make changes to the celerybeat init script, +# you can use this test script to verify you didn't break the universe + +./test_service.sh celerybeat diff --git a/extra/centos/test_celeryd.sh b/extra/centos/test_celeryd.sh index a331c2c83..89429e924 100755 --- a/extra/centos/test_celeryd.sh +++ b/extra/centos/test_celeryd.sh @@ -3,39 +3,4 @@ # If you make changes to the celeryd init script, # you can use this test script to verify you didn't break the universe -SERVICE="celeryd" -SERVICE_CMD="sudo /sbin/service $SERVICE" - -run_test() { - local msg="$1" - local cmd="$2" - local expected_retval="${3:-0}" - local n=${#msg} - - echo - echo `printf "%$((${n}+4))s" | tr " " "#"` - echo "# $msg #" - echo `printf "%$((${n}+4))s" | tr " " "#"` - - $cmd - local retval=$? - if [[ "$retval" == "$expected_retval" ]]; then - echo "[PASSED]" - else - echo "[FAILED]" - echo "Exit status: $retval, but expected: $expected_retval" - exit $retval - fi -} - -run_test "stop should succeed" "$SERVICE_CMD stop" 0 -run_test "status on a stopped service should return 1" "$SERVICE_CMD status" 1 -run_test "stopping a stopped celery should not fail" "$SERVICE_CMD stop" 0 -run_test "start should succeed" "$SERVICE_CMD start" 0 -run_test "status on a running service should return 0" "$SERVICE_CMD status" 0 -run_test "starting a running service should fail" "$SERVICE_CMD start" 1 -run_test "restarting a running service should succeed" "$SERVICE_CMD restart" 0 -run_test "status on a restarted service should return 0" "$SERVICE_CMD status" 0 -run_test "stop should succeed" "$SERVICE_CMD stop" 0 - -echo "All tests passed!" +./test_service.sh celeryd diff --git a/extra/centos/test_service.sh b/extra/centos/test_service.sh new file mode 100755 index 000000000..d5a33ba38 --- /dev/null +++ b/extra/centos/test_service.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +if [ -z "$1" ]; then + echo 'service name is not specified' + exit -1 +fi + +SERVICE="$1" +SERVICE_CMD="sudo /sbin/service $SERVICE" + +run_test() { + local msg="$1" + local cmd="$2" + local expected_retval="${3:-0}" + local n=${#msg} + + echo + echo `printf "%$((${n}+4))s" | tr " " "#"` + echo "# $msg #" + echo `printf "%$((${n}+4))s" | tr " " "#"` + + $cmd + local retval=$? + if [[ "$retval" == "$expected_retval" ]]; then + echo "[PASSED]" + else + echo "[FAILED]" + echo "Exit status: $retval, but expected: $expected_retval" + exit $retval + fi +} + +run_test "stop should succeed" "$SERVICE_CMD stop" 0 +run_test "status on a stopped service should return 1" "$SERVICE_CMD status" 1 +run_test "stopping a stopped celery should not fail" "$SERVICE_CMD stop" 0 +run_test "start should succeed" "$SERVICE_CMD start" 0 +run_test "status on a running service should return 0" "$SERVICE_CMD status" 0 +run_test "starting a running service should fail" "$SERVICE_CMD start" 1 +run_test "restarting a running service should succeed" "$SERVICE_CMD restart" 0 +run_test "status on a restarted service should return 0" "$SERVICE_CMD status" 0 +run_test "stop should succeed" "$SERVICE_CMD stop" 0 + +echo "All tests passed!" From 2dcc8de08ef8576d7dc924a43608410c213cd3c0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Apr 2014 14:02:07 +0100 Subject: [PATCH 0091/1103] Worker --detach must forward working_directory option. Closes #2003 --- celery/bin/celeryd_detach.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/bin/celeryd_detach.py b/celery/bin/celeryd_detach.py index 1db2ff041..12e1f6497 100644 --- a/celery/bin/celeryd_detach.py +++ b/celery/bin/celeryd_detach.py @@ -30,6 +30,7 @@ C_FAKEFORK = os.environ.get('C_FAKEFORK') OPTION_LIST = daemon_options(default_pidfile='celeryd.pid') + ( + Option('--workdir', default='/', dest='working_directory'), Option('--fake', default=False, action='store_true', dest='fake', help="Don't fork (for debugging purposes)"), From dba299a15277000d2860b510c80571721550e92c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Apr 2014 14:03:09 +0100 Subject: [PATCH 0092/1103] New signals: app.on_configure, .on_after_configure, .on_after_finalize --- celery/app/base.py | 30 ++++++++++++++++++++++++----- docs/reference/celery.rst | 12 ++++++++++-- funtests/stress/stress/app.py | 10 ++-------- funtests/stress/stress/templates.py | 7 ++++++- 4 files changed, 43 insertions(+), 16 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 1a1300ca1..b0079f7db 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -34,6 +34,7 @@ from celery.five import items, values from celery.loaders import get_loader_cls from celery.local import PromiseProxy, maybe_evaluate +from celery.utils.dispatch import Signal from celery.utils.functional import first, maybe_list from celery.utils.imports import instantiate, symbol_by_name from celery.utils.objects import mro_lookup @@ -117,6 +118,15 @@ class Celery(object): _pool = None builtin_fixups = BUILTIN_FIXUPS + #: Signal sent when app is loading configuration. + on_configure = None + + #: Signal sent after app has prepared the configuration. + on_after_configure = None + + #: Signal sent after app has been finalized. + on_after_finalize = None + def __init__(self, main=None, loader=None, backend=None, amqp=None, events=None, log=None, control=None, set_as_current=True, accept_magic_kwargs=False, @@ -171,6 +181,13 @@ def __init__(self, main=None, loader=None, backend=None, if self.set_as_current: self.set_current() + # Signals + if self.on_configure is None: + # used to be a method pre 3.2 + self.on_configure = Signal() + self.on_after_configure = Signal() + self.on_after_finalize = Signal() + self.on_init() _register_app(self) @@ -283,6 +300,8 @@ def finalize(self, auto=False): for task in values(self._tasks): task.bind(self) + self.on_after_finalize.send(sender=self) + def add_defaults(self, fun): if not callable(fun): d, fun = fun, lambda: d @@ -455,12 +474,12 @@ def _get_backend(self): self.loader) return backend(app=self, url=url) - def on_configure(self): - """Callback calld when the app loads configuration""" - pass - def _get_config(self): - self.on_configure() + if isinstance(self.on_configure, Signal): + self.on_configure.send(sender=self) + else: + # used to be a method pre 3.2 + self.on_configure() if self._config_source: self.loader.config_from_object(self._config_source) self.configured = True @@ -474,6 +493,7 @@ def _get_config(self): if self._preconf: for key, value in items(self._preconf): setattr(s, key, value) + self.on_after_configure.send(sender=self, source=s) return s def _after_fork(self, obj_): diff --git a/docs/reference/celery.rst b/docs/reference/celery.rst index d87cfdca1..a99c7963a 100644 --- a/docs/reference/celery.rst +++ b/docs/reference/celery.rst @@ -382,9 +382,17 @@ and creating Celery applications. Finalizes the app by loading built-in tasks, and evaluating pending task decorators - .. method:: Celery.on_configure() + .. signal:: on_configure - Optional callback for when the first time the configured is required. + Signal sent when app is loading configuration. + + .. signal:: on_after_configure + + Signal sent after app has prepared the configuration. + + .. signal:: on_after_finalize + + Signal sent after app has been finalized. .. attribute:: Celery.Pickler diff --git a/funtests/stress/stress/app.py b/funtests/stress/stress/app.py index 0a9690cfb..077437d89 100644 --- a/funtests/stress/stress/app.py +++ b/funtests/stress/stress/app.py @@ -33,7 +33,7 @@ def __init__(self, *args, **kwargs): ) ) signals.user_preload_options.connect(self.on_preload_parsed) - self.after_configure = None + self.on_configure.connect(self._maybe_use_default_template) def on_preload_parsed(self, options=None, **kwargs): self.use_template(options['template']) @@ -44,13 +44,7 @@ def use_template(self, name='default'): use_template(self, name) self.template_selected = True - def _get_config(self): - ret = super(App, self)._get_config() - if self.after_configure: - self.after_configure(ret) - return ret - - def on_configure(self): + def _maybe_use_default_template(self, **kwargs): if not self.template_selected: self.use_template('default') diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index 73b8cd3bf..09e820454 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -23,7 +23,12 @@ def _register(cls): def use_template(app, template='default'): template = template.split(',') - app.after_configure = partial(mixin_templates, template[1:]) + + # mixin the rest of the templates when the config is needed + @app.on_after_configure.connect + def load_template(sender, source, **kwargs): + mixin_templates(template[1:], source) + app.config_from_object(templates[template[0]]) From fd4701ce72f0fc17ad6e941cd25ffedfcdd03d32 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Apr 2014 14:33:10 +0100 Subject: [PATCH 0093/1103] Fixes bugs with bootsteps: requires attribute not inherited by subclasses and using module paths did not work properly. Closes #2002 --- celery/bootsteps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/bootsteps.py b/celery/bootsteps.py index 9c0427fe6..4471a4cb3 100644 --- a/celery/bootsteps.py +++ b/celery/bootsteps.py @@ -232,6 +232,8 @@ def _find_last(self): return next((C for C in values(self.steps) if C.last), None) def _firstpass(self, steps): + for step in values(steps): + step.requires = [symbol_by_name(dep) for dep in step.requires] stream = deque(step.requires for step in values(steps)) while stream: for node in stream.popleft(): @@ -283,7 +285,6 @@ def __new__(cls, name, bases, attrs): attrs.update( __qualname__=qname, name=attrs.get('name') or qname, - requires=attrs.get('requires', ()), ) return super(StepType, cls).__new__(cls, name, bases, attrs) From bc7eb64af22c1c8c482f5066483d52967d165ecf Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Mon, 28 Apr 2014 11:46:05 +0700 Subject: [PATCH 0094/1103] Fixed wrong arguments --- celery/bin/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/bin/worker.py b/celery/bin/worker.py index 44be17e4d..dc0407500 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -175,7 +175,7 @@ def run_from_argv(self, prog_name, argv=None, command=None): # parse options before detaching so errors can be handled. options, args = self.prepare_args( *self.parse_options(prog_name, argv, command)) - self.maybe_detach([command] + sys.argv[1:]) + self.maybe_detach([command] + argv) return self(*args, **options) def maybe_detach(self, argv, dopts=['-D', '--detach']): From 607567dbabc5794ecd3ba4a4ddc05e1338f70996 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 28 Apr 2014 13:04:31 +0100 Subject: [PATCH 0095/1103] Fixes tests --- celery/tests/bin/test_celeryd_detach.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/tests/bin/test_celeryd_detach.py b/celery/tests/bin/test_celeryd_detach.py index 2b6e5ae8d..000d2f633 100644 --- a/celery/tests/bin/test_celeryd_detach.py +++ b/celery/tests/bin/test_celeryd_detach.py @@ -85,6 +85,7 @@ def test_execute_from_commandline(self, detach, exit): detach.assert_called_with( path=x.execv_path, uid=None, gid=None, umask=0, fake=False, logfile='/var/log', pidfile='celeryd.pid', + working_directory='/', argv=x.execv_argv + [ '-c', '1', '-lDEBUG', '--logfile=/var/log', '--pidfile=celeryd.pid', From 2b4a3a7ff6daaa1f09b86b525ef03a117d6d7ce6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 30 Apr 2014 22:09:49 +0100 Subject: [PATCH 0096/1103] Stresstests: envvar C_SLEEP can now be used to add tracebacks to sleep calls --- funtests/stress/stress/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/funtests/stress/stress/__init__.py b/funtests/stress/stress/__init__.py index 747647ffe..089130cba 100644 --- a/funtests/stress/stress/__init__.py +++ b/funtests/stress/stress/__init__.py @@ -1,4 +1,19 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import os +import time + +if os.environ.get('C_SLEEP'): + + _orig_sleep = time.sleep + + def _sleep(n): + print('WARNING: Time sleep for {0}s'.format(n)) + import traceback + traceback.print_stack() + _orig_sleep(n) + time.sleep = _sleep + + from .app import app # noqa From a3360b5bf02e06cb5551e420938b020aab8f9cc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ionel=20Cristian=20M=C4=83rie=C8=99?= Date: Fri, 2 May 2014 16:42:56 +0300 Subject: [PATCH 0097/1103] Remove python3.4 travis fixups (it's preinstalled now) --- .travis.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index e6b5146be..f0c96caa6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,13 +17,6 @@ before_install: sudo apt-get install pypy source ~/virtualenv/pypy/bin/activate fi - if [[ $TOXENV = 3.4 ]]; then - sudo apt-get update - sudo apt-get install python3.4-dev - source ~/virtualenv/python3.4 - virtualenv ~/virtualenv/python3.4 --python=$(which python3.4) - source ~/virtualenv/python3.4/bin/activate - fi python --version uname -a lsb_release -a From cdb745343c4bde164d624163be04da9a9ae65db2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 6 May 2014 13:36:42 +0100 Subject: [PATCH 0098/1103] Fixes stress test templates and adds SQS template --- funtests/stress/stress/templates.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index 09e820454..931269807 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -25,7 +25,7 @@ def use_template(app, template='default'): template = template.split(',') # mixin the rest of the templates when the config is needed - @app.on_after_configure.connect + @app.on_after_configure.connect(weak=False) def load_template(sender, source, **kwargs): mixin_templates(template[1:], source) @@ -116,3 +116,11 @@ class events(default): @template() class execv(default): CELERYD_FORCE_EXECV = True + + +@template() +class sqs(default): + BROKER_URL='sqs://' + BROKER_TRANSPORT_OPTIONS = { + 'region': os.environ.get('AWS_REGION', 'us-east-1'), + } From 3e9119d4972545d9c2d4d210c07558305dfd6afe Mon Sep 17 00:00:00 2001 From: Jay Farrimond Date: Mon, 5 May 2014 15:03:50 -0700 Subject: [PATCH 0099/1103] non-string dict keys in django-celery configs This fix allows celery-flower to not have problems displaying configurations for projects that still use configurations embedded in django settings files. In this instance there are some int dict keys that are totally unrelated to celery but that are causing frequent error messages in the celeryd logs. --- celery/worker/control.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/worker/control.py b/celery/worker/control.py index 8de8ac838..6016543c7 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -14,7 +14,7 @@ from kombu.utils.encoding import safe_repr from celery.exceptions import WorkerShutdown -from celery.five import UserDict, items +from celery.five import UserDict, items, string_t from celery.platforms import signals as _signals from celery.utils import timeutils from celery.utils.functional import maybe_list @@ -364,7 +364,7 @@ def active_queues(state): def _wanted_config_key(key): - return key.isupper() and not key.startswith('__') + return isinstance(key, string_t) and key.isupper() and not key.startswith('__') @Panel.register From 0f111b1834c3c1ecfe2d40add9c334989f6ecaa4 Mon Sep 17 00:00:00 2001 From: Mher Movsisyan Date: Tue, 6 May 2014 16:59:00 +0400 Subject: [PATCH 0100/1103] Adds stats for eventlet pool --- celery/concurrency/eventlet.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/celery/concurrency/eventlet.py b/celery/concurrency/eventlet.py index e5319a9b8..613b28a86 100644 --- a/celery/concurrency/eventlet.py +++ b/celery/concurrency/eventlet.py @@ -142,3 +142,10 @@ def on_apply(self, target, args=None, kwargs=None, callback=None, self._quick_put(apply_target, target, args, kwargs, callback, accept_callback, self.getpid) + + def _get_info(self): + return { + 'max-concurrency': self.limit, + 'free-threads': self._pool.free(), + 'running-threads': self._pool.running(), + } From f1601c075564590112abcd344dde829b1e22b23e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 6 May 2014 16:02:56 +0100 Subject: [PATCH 0101/1103] Small doc stuff --- celery/bin/worker.py | 4 ++-- docs/configuration.rst | 3 ++- docs/internals/guide.rst | 27 +++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/celery/bin/worker.py b/celery/bin/worker.py index dc0407500..d5592f85f 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -71,8 +71,8 @@ .. cmdoption:: -E, --events - Send events that can be captured by monitors like :program:`celery events`, - `celerymon`, and others. + Send task-related events that can be captured by monitors like + :program:`celery events`, `celerymon`, and others. .. cmdoption:: --without-gossip diff --git a/docs/configuration.rst b/docs/configuration.rst index 3f787f270..864b255dd 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1509,7 +1509,8 @@ Events CELERY_SEND_EVENTS ~~~~~~~~~~~~~~~~~~ -Send events so the worker can be monitored by tools like `celerymon`. +Send task-related events so that tasks can be monitored using tools like +`flower`. Sets the default value for the workers :option:`-E` argument. .. setting:: CELERY_SEND_TASK_SENT_EVENT diff --git a/docs/internals/guide.rst b/docs/internals/guide.rst index 941f7b11d..6a4be2f34 100644 --- a/docs/internals/guide.rst +++ b/docs/internals/guide.rst @@ -305,3 +305,30 @@ Module Overview - celery.contrib Additional public code that doesn't fit into any other namespace. + +Worker overview +=============== + +* `celery.bin.worker:Worker` + + This is the command-line interface to the worker. + + Responsibilities: + * Daemonization when `--detach` set, + * dropping privileges when using `--uid`/`--gid` arguments + * Installs "concurrency patches" (eventlet/gevent monkey patches). + + ``app.worker_main(argv)`` calls + ``instantiate('celery.bin.worker:Worker')(app).execute_from_commandline(argv)`` + +* `app.Worker` -> `celery.apps.worker:Worker` + + Responsibilities: + * sets up logging and redirects stdouts + * installs signal handlers (`TERM`/`HUP`/`STOP`/`USR1` (cry)/`USR2` (rdb)) + * prints banner and warnings (e.g. pickle warning) + * handles the ``--purge`` argument + +* `app.WorkController` -> `celery.worker.WorkController` + + This is the real worker, built up around bootsteps. From 2c37c41b051a6c214e453668883e88fccafb7bda Mon Sep 17 00:00:00 2001 From: Luke Pomfrey Date: Fri, 2 May 2014 13:06:33 +0100 Subject: [PATCH 0102/1103] Fix handling of non-string keys in filter_hidden_settings --- celery/app/utils.py | 11 ++++++----- celery/tests/app/test_utils.py | 18 +++++++++++++++++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/celery/app/utils.py b/celery/app/utils.py index ba5e1bb8b..a409d8fac 100644 --- a/celery/app/utils.py +++ b/celery/app/utils.py @@ -175,11 +175,12 @@ def filter_hidden_settings(conf): def maybe_censor(key, value, mask='*' * 8): if isinstance(value, Mapping): return filter_hidden_settings(value) - if isinstance(value, string_t) and HIDDEN_SETTINGS.search(key): - return mask - if isinstance(key, string_t) and 'BROKER_URL' in key.upper(): - from kombu import Connection - return Connection(value).as_uri(mask=mask) + if isinstance(key, string_t): + if HIDDEN_SETTINGS.search(key): + return mask + if 'BROKER_URL' in key.upper(): + from kombu import Connection + return Connection(value).as_uri(mask=mask) return value return {k: maybe_censor(k, v) for k, v in items(conf)} diff --git a/celery/tests/app/test_utils.py b/celery/tests/app/test_utils.py index dc7e38110..b0ff108e8 100644 --- a/celery/tests/app/test_utils.py +++ b/celery/tests/app/test_utils.py @@ -2,7 +2,7 @@ from collections import Mapping, MutableMapping -from celery.app.utils import Settings, bugreport +from celery.app.utils import Settings, filter_hidden_settings, bugreport from celery.tests.case import AppCase, Mock @@ -20,6 +20,22 @@ def test_is_mutable_mapping(self): self.assertTrue(issubclass(Settings, MutableMapping)) +class test_filter_hidden_settings(AppCase): + + def test_handles_non_string_keys(self): + """filter_hidden_settings shouldn't raise an exception when handling + mappings with non-string keys""" + conf = { + 'STRING_KEY': 'VALUE1', + ('NON', 'STRING', 'KEY'): 'VALUE2', + 'STRING_KEY2': { + 'STRING_KEY3': 1, + ('NON', 'STRING', 'KEY', '2'): 2 + }, + } + filter_hidden_settings(conf) + + class test_bugreport(AppCase): def test_no_conn_driver_info(self): From dc9e9755ba41eb03556a135831040c91046b1eb6 Mon Sep 17 00:00:00 2001 From: Luke Pomfrey Date: Tue, 6 May 2014 16:16:32 +0100 Subject: [PATCH 0103/1103] Adding self to CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index edf2f120a..3484a6712 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -160,3 +160,4 @@ Martin Davidsson, 2014/02/08 Chris Clark, 2014/02/20 Matthew Duggan, 2014/04/10 Brian Bouterse, 2014/04/10 +Luke Pomfrey, 2014/05/06 From 61288aa2a8150b2affbaa6845a2c537d1aff3622 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 7 May 2014 11:12:21 +0100 Subject: [PATCH 0104/1103] Updates system configuration --- extra/systemd/celery.conf | 13 ++++++++++++- extra/systemd/celery.service | 16 +++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/extra/systemd/celery.conf b/extra/systemd/celery.conf index d490fe793..08b90cf28 100644 --- a/extra/systemd/celery.conf +++ b/extra/systemd/celery.conf @@ -1,2 +1,13 @@ +# See +# http://docs.celeryproject.org/en/latest/tutorials/daemonizing.html#available-options + +CELERY_APP="proj" +CELERYD_NODES="worker" +CELERYD_OPTS="" +CELERY_BIN="/usr/bin/python2 -m celery" +CELERYD_PID_FILE="/var/run/celery/%n.pid" +CELERYD_LOG_FILE="/var/log/celery/%n.log" +CELERYD_LOG_LEVEL="INFO" + d /run/celery 0755 user users - -d /var/log/celery 0755 user users - \ No newline at end of file +d /var/log/celery 0755 user users - diff --git a/extra/systemd/celery.service b/extra/systemd/celery.service index 31f17bdb1..5729d2924 100644 --- a/extra/systemd/celery.service +++ b/extra/systemd/celery.service @@ -1,17 +1,23 @@ [Unit] -Description=Celery Nodes Daemon +Description=Celery workers After=network.target [Service] Type=forking User=user Group=users -#Environment=DJANGO_SETTINGS_MODULE=MyProject.settings EnvironmentFile=-/etc/conf.d/celery WorkingDirectory=/opt/Myproject/ -ExecStart=/usr/bin/python2 ${CELERY_BIN} $CELERYD_MULTI start $CELERYD_NODES --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel="INFO" $CELERYD_OPTS -ExecStop=/usr/bin/python2 ${CELERY_BIN} $CELERYD_MULTI stopwait $CELERYD_NODES --pidfile=${CELERYD_PID_FILE} -ExecReload=/usr/bin/python2 ${CELERY_BIN} $CELERYD_MULTI restart $CELERYD_NODES --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel="INFO" $CELERYD_OPTS +ExecStart=${CELERY_BIN} multi start $CELERYD_NODES \ + -A $CELERY_APP --pidfile=${CELERYD_PID_FILE} \ + --logfile=${CELERYD_LOG_FILE} --loglevel="${CELERYD_LOG_LEVEL}" \ + $CELERYD_OPTS +ExecStop=${CELERY_BIN} multi stopwait $CELERYD_NODES \ + --pidfile=${CELERYD_PID_FILE} +ExecReload=${CELERY_BIN} multi restart $CELERYD_NODES \ + -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \ + --logfile=${CELERYD_LOG_FILE} --loglevel="${CELERYD_LOG_LEVEL}" \ + $CELERYD_OPTS [Install] WantedBy=multi-user.target From 661bbfe98452e67e23a98718dd034dd811712e74 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 7 May 2014 16:42:11 +0100 Subject: [PATCH 0105/1103] Renames Task.subtask -> .signature (with alias for compat) --- celery/app/task.py | 22 ++++++++++++-------- celery/task/base.py | 1 + celery/tests/tasks/test_chord.py | 10 ++++----- docs/getting-started/next-steps.rst | 32 ++++++++++++++--------------- docs/internals/guide.rst | 2 +- docs/internals/protocol.rst | 10 ++++----- docs/reference/celery.rst | 2 +- docs/userguide/calling.rst | 8 ++++---- docs/userguide/canvas.rst | 23 +++++++++------------ docs/userguide/tasks.rst | 6 +++--- examples/resultgraph/tasks.py | 2 +- 11 files changed, 60 insertions(+), 58 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index b20974424..38eef7b5d 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -555,8 +555,8 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, **dict(self._get_exec_options(), **options) ) - def subtask_from_request(self, request=None, args=None, kwargs=None, - queue=None, **extra_options): + def signature_from_request(self, request=None, args=None, kwargs=None, + queue=None, **extra_options): request = self.request if request is None else request args = request.args if args is None else args kwargs = request.kwargs if kwargs is None else kwargs @@ -573,7 +573,10 @@ def subtask_from_request(self, request=None, args=None, kwargs=None, options.update( {'queue': queue} if queue else (request.delivery_info or {}) ) - return self.subtask(args, kwargs, options, type=self, **extra_options) + return self.signature( + args, kwargs, options, type=self, **extra_options + ) + subtask_from_request = signature_from_request def retry(self, args=None, kwargs=None, exc=None, throw=True, eta=None, countdown=None, max_retries=None, **options): @@ -647,7 +650,7 @@ def retry(self, args=None, kwargs=None, exc=None, throw=True, countdown = self.default_retry_delay is_eager = request.is_eager - S = self.subtask_from_request( + S = self.signature_from_request( request, args, kwargs, countdown=countdown, eta=eta, retries=retries, **options @@ -748,20 +751,21 @@ def AsyncResult(self, task_id, **kwargs): return self._get_app().AsyncResult(task_id, backend=self.backend, task_name=self.name, **kwargs) - def subtask(self, args=None, *starargs, **starkwargs): + def signature(self, args=None, *starargs, **starkwargs): """Return :class:`~celery.signature` object for this task, wrapping arguments and execution options for a single task invocation.""" starkwargs.setdefault('app', self.app) return signature(self, args, *starargs, **starkwargs) + subtask = signature def s(self, *args, **kwargs): - """``.s(*a, **k) -> .subtask(a, k)``""" - return self.subtask(args, kwargs) + """``.s(*a, **k) -> .signature(a, k)``""" + return self.signature(args, kwargs) def si(self, *args, **kwargs): - """``.si(*a, **k) -> .subtask(a, k, immutable=True)``""" - return self.subtask(args, kwargs, immutable=True) + """``.si(*a, **k) -> .signature(a, k, immutable=True)``""" + return self.signature(args, kwargs, immutable=True) def chunks(self, it, n): """Creates a :class:`~celery.canvas.chunks` task for this task.""" diff --git a/celery/task/base.py b/celery/task/base.py index 9e12d4f8c..6feffc48d 100644 --- a/celery/task/base.py +++ b/celery/task/base.py @@ -24,6 +24,7 @@ #: list of methods that must be classmethods in the old API. _COMPAT_CLASSMETHODS = ( 'delay', 'apply_async', 'retry', 'apply', 'subtask_from_request', + 'signature_from_request', 'signature', 'AsyncResult', 'subtask', '_get_request', '_get_exec_options', ) diff --git a/celery/tests/tasks/test_chord.py b/celery/tests/tasks/test_chord.py index 47e771841..531d06467 100644 --- a/celery/tests/tasks/test_chord.py +++ b/celery/tests/tasks/test_chord.py @@ -142,7 +142,7 @@ def callback(*args, **kwargs): fail_current = self.app.backend.fail_from_current_stack = Mock() try: with patch_unlock_retry(self.app) as (unlock, retry): - subtask, canvas.maybe_signature = ( + signature, canvas.maybe_signature = ( canvas.maybe_signature, passthru, ) if setup: @@ -160,7 +160,7 @@ def callback(*args, **kwargs): except Retry: pass finally: - canvas.maybe_signature = subtask + canvas.maybe_signature = signature yield callback_s, retry, fail_current finally: result.GroupResult = pts @@ -211,7 +211,7 @@ def test_apply(self): body = self.add.s(2) result = x(body) self.assertTrue(result.id) - # does not modify original subtask + # does not modify original signature with self.assertRaises(KeyError): body.options['task_id'] self.assertTrue(chord._type.called) @@ -228,6 +228,6 @@ def test_run(self): Chord = self.app.tasks['celery.chord'] body = dict() - Chord(group(self.add.subtask((i, i)) for i in range(5)), body) - Chord([self.add.subtask((j, j)) for j in range(5)], body) + Chord(group(self.add.signature((i, i)) for i in range(5)), body) + Chord([self.add.signature((j, j)) for j in range(5)], body) self.assertEqual(self.app.backend.apply_chord.call_count, 2) diff --git a/docs/getting-started/next-steps.rst b/docs/getting-started/next-steps.rst index d25282d16..bc10f801a 100644 --- a/docs/getting-started/next-steps.rst +++ b/docs/getting-started/next-steps.rst @@ -275,7 +275,7 @@ so that no message is sent:: These three methods - :meth:`delay`, :meth:`apply_async`, and applying (``__call__``), represents the Celery calling API, which are also used for -subtasks. +signatures. A more detailed overview of the Calling API can be found in the :ref:`Calling User Guide `. @@ -380,16 +380,16 @@ Calling tasks is described in detail in the You just learned how to call a task using the tasks ``delay`` method, and this is often all you need, but sometimes you may want to pass the signature of a task invocation to another process or as an argument to another -function, for this Celery uses something called *subtasks*. +function, for this Celery uses something called *signatures*. -A subtask wraps the arguments and execution options of a single task +A signature wraps the arguments and execution options of a single task invocation in a way such that it can be passed to functions or even serialized and sent across the wire. -You can create a subtask for the ``add`` task using the arguments ``(2, 2)``, +You can create a signature for the ``add`` task using the arguments ``(2, 2)``, and a countdown of 10 seconds like this:: - >>> add.subtask((2, 2), countdown=10) + >>> add.signature((2, 2), countdown=10) tasks.add(2, 2) There is also a shortcut using star arguments:: @@ -400,12 +400,12 @@ There is also a shortcut using star arguments:: And there's that calling API again… ----------------------------------- -Subtask instances also supports the calling API, which means that they +Signature instances also supports the calling API, which means that they have the ``delay`` and ``apply_async`` methods. -But there is a difference in that the subtask may already have +But there is a difference in that the signature may already have an argument signature specified. The ``add`` task takes two arguments, -so a subtask specifying two arguments would make a complete signature:: +so a signature specifying two arguments would make a complete signature:: >>> s1 = add.s(2, 2) >>> res = s1.delay() @@ -418,8 +418,8 @@ But, you can also make incomplete signatures to create what we call # incomplete partial: add(?, 2) >>> s2 = add.s(2) -``s2`` is now a partial subtask that needs another argument to be complete, -and this can be resolved when calling the subtask:: +``s2`` is now a partial signature that needs another argument to be complete, +and this can be resolved when calling the signature:: # resolves the partial: add(8, 2) >>> res = s2.delay(8) @@ -435,14 +435,14 @@ existing keyword arguments, but with new arguments taking precedence:: >>> s3 = add.s(2, 2, debug=True) >>> s3.delay(debug=False) # debug is now False. -As stated subtasks supports the calling API, which means that: +As stated signatures supports the calling API, which means that: -- ``subtask.apply_async(args=(), kwargs={}, **options)`` +- ``sig.apply_async(args=(), kwargs={}, **options)`` - Calls the subtask with optional partial arguments and partial + Calls the signature with optional partial arguments and partial keyword arguments. Also supports partial execution options. -- ``subtask.delay(*args, **kwargs)`` +- ``sig.delay(*args, **kwargs)`` Star argument version of ``apply_async``. Any arguments will be prepended to the arguments in the signature, and keyword arguments is merged with any @@ -466,7 +466,7 @@ The Primitives - :ref:`starmap ` - :ref:`chunks ` -The primitives are subtasks themselves, so that they can be combined +These primitives are signature objects themselves, so they can be combined in any number of ways to compose complex workflows. .. note:: @@ -556,7 +556,7 @@ to a chord: 90 -Since these primitives are all of the subtask type they +Since these primitives are all of the signature type they can be combined almost however you want, e.g:: >>> upload_document.s(file) | group(apply_filter.s() for filter in filters) diff --git a/docs/internals/guide.rst b/docs/internals/guide.rst index 6a4be2f34..36e053864 100644 --- a/docs/internals/guide.rst +++ b/docs/internals/guide.rst @@ -64,7 +64,7 @@ Naming Sometimes it makes sense to have a class mask as a function, and there is precedence for this in the stdlib (e.g. :class:`~contextlib.contextmanager`). Celery examples include - :class:`~celery.subtask`, :class:`~celery.chord`, + :class:`~celery.signature`, :class:`~celery.chord`, ``inspect``, :class:`~kombu.utils.functional.promise` and more.. - Factory functions and methods must be `CamelCase` (excluding verbs): diff --git a/docs/internals/protocol.rst b/docs/internals/protocol.rst index f80e6e8ff..6b7360b31 100644 --- a/docs/internals/protocol.rst +++ b/docs/internals/protocol.rst @@ -71,7 +71,7 @@ to process it. The taskset this task is part of (if any). * chord - :`subtask`: + :`Signature`: .. versionadded:: 2.3 @@ -88,18 +88,18 @@ to process it. should be used. * callbacks - :`subtask`: + :`Signature`: .. versionadded:: 3.0 - A list of subtasks to apply if the task exited successfully. + A list of signatures to call if the task exited successfully. * errbacks - :`subtask`: + :`Signature`: .. versionadded:: 3.0 - A list of subtasks to apply if an error occurs while executing the task. + A list of signatures to call if an error occurs while executing the task. * timelimit :`(float, float)`: diff --git a/docs/reference/celery.rst b/docs/reference/celery.rst index a99c7963a..0363c446b 100644 --- a/docs/reference/celery.rst +++ b/docs/reference/celery.rst @@ -470,7 +470,7 @@ See :ref:`guide-canvas` for more about creating task workflows. Signatures can also be created from tasks:: - >>> add.subtask(args=(), kwargs={}, options={}) + >>> add.signature(args=(), kwargs={}, options={}) or the ``.s()`` shortcut:: diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index bfddf408a..9701c4a1a 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -95,7 +95,7 @@ called `add`, returning the sum of two arguments: .. topic:: There's another way… You will learn more about this later while reading about the :ref:`Canvas - `, but :class:`~celery.subtask`'s are objects used to pass around + `, but :class:`~celery.signature`'s are objects used to pass around the signature of a task invocation, (for example to send it over the network), and they also support the Calling API: @@ -118,8 +118,8 @@ as a partial argument: .. sidebar:: What is ``s``? - The ``add.s`` call used here is called a subtask, I talk - more about subtasks in the :ref:`canvas guide `, + The ``add.s`` call used here is called a signature, I talk + more about signatures in the :ref:`canvas guide `, where you can also learn about :class:`~celery.chain`, which is a simpler way to chain tasks together. @@ -447,7 +447,7 @@ Though this particular example is much better expressed as a group: >>> from celery import group >>> numbers = [(2, 2), (4, 4), (8, 8), (16, 16)] - >>> res = group(add.subtask(n) for i in numbers).apply_async() + >>> res = group(add.s(n) for i in numbers).apply_async() >>> res.get() [4, 8, 16, 32] diff --git a/docs/userguide/canvas.rst b/docs/userguide/canvas.rst index 0afff4dbf..f74e8e4be 100644 --- a/docs/userguide/canvas.rst +++ b/docs/userguide/canvas.rst @@ -26,9 +26,6 @@ A :func:`~celery.signature` wraps the arguments, keyword arguments, and executio of a single task invocation in a way such that it can be passed to functions or even serialized and sent across the wire. -Signatures are often nicknamed "subtasks" because they describe a task to be called -within a task. - - You can create a signature for the ``add`` task using its name like this:: >>> from celery import signature @@ -38,9 +35,9 @@ within a task. This task has a signature of arity 2 (two arguments): ``(2, 2)``, and sets the countdown execution option to 10. -- or you can create one using the task's ``subtask`` method:: +- or you can create one using the task's ``signature`` method:: - >>> add.subtask((2, 2), countdown=10) + >>> add.signature((2, 2), countdown=10) tasks.add(2, 2) - There is also a shortcut using star arguments:: @@ -55,7 +52,7 @@ within a task. - From any signature instance you can inspect the different fields:: - >>> s = add.subtask((2, 2), {'debug': True}, countdown=10) + >>> s = add.signature((2, 2), {'debug': True}, countdown=10) >>> s.args (2, 2) >>> s.kwargs @@ -82,10 +79,10 @@ within a task. ``apply_async`` takes the same arguments as the :meth:`Task.apply_async <@Task.apply_async>` method:: >>> add.apply_async(args, kwargs, **options) - >>> add.subtask(args, kwargs, **options).apply_async() + >>> add.signature(args, kwargs, **options).apply_async() >>> add.apply_async((2, 2), countdown=1) - >>> add.subtask((2, 2), countdown=1).apply_async() + >>> add.signature((2, 2), countdown=1).apply_async() - You can't define options with :meth:`~@Task.s`, but a chaining ``set`` call takes care of that:: @@ -125,7 +122,7 @@ creates partials: - Any options added will be merged with the options in the signature, with the new options taking precedence:: - >>> s = add.subtask((2, 2), countdown=10) + >>> s = add.signature((2, 2), countdown=10) >>> s.apply_async(countdown=1) # countdown is now 1 You can also clone signatures to create derivates: @@ -147,7 +144,7 @@ Sometimes you want to specify a callback that does not take additional arguments, and in that case you can set the signature to be immutable:: - >>> add.apply_async((2, 2), link=reset_buffers.subtask(immutable=True)) + >>> add.apply_async((2, 2), link=reset_buffers.signature(immutable=True)) The ``.si()`` shortcut can also be used to create immutable signatures:: @@ -289,7 +286,7 @@ Here's some examples: In that case you can mark the signature as immutable, so that the arguments cannot be changed:: - >>> add.subtask((2, 2), immutable=True) + >>> add.signature((2, 2), immutable=True) There's also an ``.si`` shortcut for this:: @@ -419,7 +416,7 @@ The linked task will be applied with the result of its parent task as the first argument, which in the above case will result in ``mul(4, 16)`` since the result is 4. -The results will keep track of what subtasks a task applies, +The results will keep track of any subtasks called by the original task, and this can be accessed from the result instance:: >>> res.children @@ -456,7 +453,7 @@ You can also add *error callbacks* using the ``link_error`` argument:: >>> add.apply_async((2, 2), link_error=log_error.s()) - >>> add.subtask((2, 2), link_error=log_error.s()) + >>> add.signature((2, 2), link_error=log_error.s()) Since exceptions can only be serialized when pickle is used the error callbacks take the id of the parent task as argument instead: diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 06351d5da..8e7cb0739 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -266,9 +266,9 @@ The request defines the following attributes: :called_directly: This flag is set to true if the task was not executed by the worker. -:callbacks: A list of subtasks to be called if this task returns successfully. +:callbacks: A list of signatures to be called if this task returns successfully. -:errback: A list of subtasks to be called if this task fails. +:errback: A list of signatures to be called if this task fails. :utc: Set to true the caller has utc enabled (:setting:`CELERY_ENABLE_UTC`). @@ -1297,7 +1297,7 @@ Make your design asynchronous instead, for example by using *callbacks*. Here I instead created a chain of tasks by linking together -different :func:`~celery.subtask`'s. +different :func:`~celery.signature`'s. You can read about chains and other powerful constructs at :ref:`designing-workflows`. diff --git a/examples/resultgraph/tasks.py b/examples/resultgraph/tasks.py index bb14d2798..3c6dd81b0 100644 --- a/examples/resultgraph/tasks.py +++ b/examples/resultgraph/tasks.py @@ -16,7 +16,7 @@ # when the second task is ready.) # # >>> unlock_graph.apply_async((A.apply_async(), -# ... A_callback.subtask()), countdown=1) +# ... A_callback.s()), countdown=1) from celery import chord, group, task, signature, uuid From d79dcd8e82c5e41f39abd07ffed81ca58052bcd2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 8 May 2014 16:48:43 +0100 Subject: [PATCH 0106/1103] Canvas refactor, parts of root_id and parent_id and group no longer evaluates generators --- celery/app/amqp.py | 10 +- celery/app/base.py | 3 +- celery/app/builtins.py | 187 ++------------------- celery/app/task.py | 2 + celery/canvas.py | 264 ++++++++++++++++++++++++------ celery/tests/app/test_builtins.py | 22 ++- celery/tests/tasks/test_chord.py | 8 +- docs/internals/protov2.rst | 3 + 8 files changed, 251 insertions(+), 248 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index a23f1d63b..e3f62b731 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -269,7 +269,8 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, expires=None, retries=0, chord=None, callbacks=None, errbacks=None, reply_to=None, time_limit=None, soft_time_limit=None, - create_sent_event=False, now=None, timezone=None): + create_sent_event=False, now=None, timezone=None, + root_id=None, parent_id=None): args = args or () kwargs = kwargs or {} utc = self.utc @@ -305,6 +306,8 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, 'chord': chord, 'retries': retries, 'timelimit': (time_limit, soft_time_limit), + 'root_id': root_id, + 'parent_id': parent_id, }, properties={ 'correlation_id': task_id, @@ -313,6 +316,8 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, body=(args, kwargs), sent_event={ 'uuid': task_id, + 'root': root_id, + 'parent': parent_id, 'name': name, 'args': safe_repr(args), 'kwargs': safe_repr(kwargs), @@ -327,7 +332,8 @@ def as_task_v1(self, task_id, name, args=None, kwargs=None, expires=None, retries=0, chord=None, callbacks=None, errbacks=None, reply_to=None, time_limit=None, soft_time_limit=None, - create_sent_event=False, now=None, timezone=None): + create_sent_event=False, now=None, timezone=None, + root_id=None, parent_id=None): args = args or () kwargs = kwargs or {} utc = self.utc diff --git a/celery/app/base.py b/celery/app/base.py index b0079f7db..02590025a 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -351,7 +351,7 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, publisher=None, link=None, link_error=None, add_to_parent=True, group_id=None, retries=0, chord=None, reply_to=None, time_limit=None, soft_time_limit=None, - **options): + root_id=None, parent_id=None, **options): amqp = self.amqp task_id = task_id or uuid() producer = producer or publisher # XXX compat @@ -369,6 +369,7 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, maybe_list(link), maybe_list(link_error), reply_to or self.oid, time_limit, soft_time_limit, self.conf.CELERY_SEND_TASK_SENT_EVENT, + root_id, parent_id, ) if connection: diff --git a/celery/app/builtins.py b/celery/app/builtins.py index e42e0b25c..f08bf5054 100644 --- a/celery/app/builtins.py +++ b/celery/app/builtins.py @@ -9,10 +9,7 @@ """ from __future__ import absolute_import -from collections import deque - from celery._state import get_current_worker_task, connect_on_app_finalize -from celery.utils import uuid from celery.utils.log import get_logger __all__ = [] @@ -44,7 +41,7 @@ def add_unlock_chord_task(app): It joins chords by creating a task chain polling the header for completion. """ - from celery.canvas import signature + from celery.canvas import maybe_signature from celery.exceptions import ChordError from celery.result import allow_join_result, result_from_tuple @@ -66,6 +63,8 @@ def unlock_chord(group_id, callback, interval=None, propagate=None, interval = unlock_chord.default_retry_delay # check if the task group is ready, and if so apply the callback. + callback = maybe_signature(callback, app) + root_id = callback.options.get('root_id') deps = GroupResult( group_id, [result_from_tuple(r, app=app) for r in result], @@ -73,7 +72,7 @@ def unlock_chord(group_id, callback, interval=None, propagate=None, j = deps.join_native if deps.supports_native_join else deps.join if deps.ready(): - callback = signature(callback, app=app) + callback = maybe_signature(callback, app=app) try: with allow_join_result(): ret = j(timeout=3.0, propagate=propagate) @@ -139,7 +138,7 @@ def chunks(task, it, n): @connect_on_app_finalize def add_group_task(app): _app = app - from celery.canvas import maybe_signature, signature + from celery.canvas import maybe_signature from celery.result import result_from_tuple class Group(app.Task): @@ -153,13 +152,8 @@ def run(self, tasks, result, group_id, partial_args, app = self.app result = result_from_tuple(result, app) # any partial args are added to all tasks in the group - taskit = (signature(task, app=app).clone(partial_args) + taskit = (maybe_signature(task, app=app).clone(partial_args) for i, task in enumerate(tasks)) - if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER: - return app.GroupResult( - result.id, - [stask.apply(group_id=group_id) for stask in taskit], - ) with app.producer_or_acquire() as pub: [stask.apply_async(group_id=group_id, producer=pub, add_to_parent=False) for stask in taskit] @@ -167,48 +161,11 @@ def run(self, tasks, result, group_id, partial_args, if add_to_parent and parent: parent.add_trail(result) return result - - def prepare(self, options, tasks, args, **kwargs): - options['group_id'] = group_id = ( - options.setdefault('task_id', uuid())) - - def prepare_member(task): - task = maybe_signature(task, app=self.app) - task.options['group_id'] = group_id - return task, task.freeze() - - try: - tasks, res = list(zip( - *[prepare_member(task) for task in tasks] - )) - except ValueError: # tasks empty - tasks, res = [], [] - return (tasks, self.app.GroupResult(group_id, res), group_id, args) - - def apply_async(self, partial_args=(), kwargs={}, **options): - if self.app.conf.CELERY_ALWAYS_EAGER: - return self.apply(partial_args, kwargs, **options) - tasks, result, gid, args = self.prepare( - options, args=partial_args, **kwargs - ) - super(Group, self).apply_async(( - list(tasks), result.as_tuple(), gid, args), **options - ) - return result - - def apply(self, args=(), kwargs={}, **options): - return super(Group, self).apply( - self.prepare(options, args=args, **kwargs), - **options).get() return Group @connect_on_app_finalize def add_chain_task(app): - from celery.canvas import ( - Signature, chain, chord, group, maybe_signature, maybe_unroll_group, - ) - _app = app class Chain(app.Task): @@ -217,85 +174,6 @@ class Chain(app.Task): accept_magic_kwargs = False _decorated = True - def prepare_steps(self, args, tasks): - app = self.app - steps = deque(tasks) - next_step = prev_task = prev_res = None - tasks, results = [], [] - i = 0 - while steps: - # First task get partial args from chain. - task = maybe_signature(steps.popleft(), app=app) - task = task.clone() if i else task.clone(args) - res = task.freeze() - i += 1 - - if isinstance(task, group): - task = maybe_unroll_group(task) - if isinstance(task, chain): - # splice the chain - steps.extendleft(reversed(task.tasks)) - continue - - elif isinstance(task, group) and steps and \ - not isinstance(steps[0], group): - # automatically upgrade group(..) | s to chord(group, s) - try: - next_step = steps.popleft() - # for chords we freeze by pretending it's a normal - # task instead of a group. - res = Signature.freeze(next_step) - task = chord(task, body=next_step, task_id=res.task_id) - except IndexError: - pass # no callback, so keep as group - if prev_task: - # link previous task to this task. - prev_task.link(task) - # set the results parent attribute. - if not res.parent: - res.parent = prev_res - - if not isinstance(prev_task, chord): - results.append(res) - tasks.append(task) - prev_task, prev_res = task, res - - return tasks, results - - def apply_async(self, args=(), kwargs={}, group_id=None, chord=None, - task_id=None, link=None, link_error=None, **options): - if self.app.conf.CELERY_ALWAYS_EAGER: - return self.apply(args, kwargs, **options) - options.pop('publisher', None) - tasks, results = self.prepare_steps(args, kwargs['tasks']) - result = results[-1] - if group_id: - tasks[-1].set(group_id=group_id) - if chord: - tasks[-1].set(chord=chord) - if task_id: - tasks[-1].set(task_id=task_id) - result = tasks[-1].type.AsyncResult(task_id) - # make sure we can do a link() and link_error() on a chain object. - if link: - tasks[-1].set(link=link) - # and if any task in the chain fails, call the errbacks - if link_error: - for task in tasks: - task.set(link_error=link_error) - tasks[0].apply_async(**options) - return result - - def apply(self, args=(), kwargs={}, signature=maybe_signature, - **options): - app = self.app - last, fargs = None, args # fargs passed to first task only - for task in kwargs['tasks']: - res = signature(task, app=app).clone(fargs).apply( - last and (last.get(), ), - ) - res.parent, last, fargs = last, res, None - return last return Chain @@ -304,10 +182,9 @@ def add_chord_task(app): """Every chord is executed in a dedicated task, so that the chord can be used as a signature, and this generates the task responsible for that.""" - from celery import group + from celery import group, chord as _chord from celery.canvas import maybe_signature _app = app - default_propagate = app.conf.CELERY_CHORD_PROPAGATES class Chord(app.Task): app = _app @@ -320,53 +197,13 @@ def run(self, header, body, partial_args=(), interval=None, countdown=1, max_retries=None, propagate=None, eager=False, **kwargs): app = self.app - propagate = default_propagate if propagate is None else propagate - group_id = uuid() - # - convert back to group if serialized tasks = header.tasks if isinstance(header, group) else header header = group([ - maybe_signature(s, app=app).clone() for s in tasks + maybe_signature(s, app=app) for s in tasks ], app=self.app) - # - eager applies the group inline - if eager: - return header.apply(args=partial_args, task_id=group_id) - - body.setdefault('chord_size', len(header.tasks)) - results = header.freeze(group_id=group_id, chord=body).results - - return self.backend.apply_chord( - header, partial_args, group_id, - body, interval=interval, countdown=countdown, - max_retries=max_retries, propagate=propagate, result=results, - ) - - def apply_async(self, args=(), kwargs={}, task_id=None, - group_id=None, chord=None, **options): - app = self.app - if app.conf.CELERY_ALWAYS_EAGER: - return self.apply(args, kwargs, **options) - header = kwargs.pop('header') - body = kwargs.pop('body') - header, body = (maybe_signature(header, app=app), - maybe_signature(body, app=app)) - # forward certain options to body - if chord is not None: - body.options['chord'] = chord - if group_id is not None: - body.options['group_id'] = group_id - [body.link(s) for s in options.pop('link', [])] - [body.link_error(s) for s in options.pop('link_error', [])] - body_result = body.freeze(task_id) - parent = super(Chord, self).apply_async((header, body, args), - kwargs, **options) - body_result.parent = parent - return body_result - - def apply(self, args=(), kwargs={}, propagate=True, **options): - body = kwargs['body'] - res = super(Chord, self).apply(args, dict(kwargs, eager=True), - **options) - return maybe_signature(body, app=self.app).apply( - args=(res.get(propagate=propagate).get(), )) + body = maybe_signature(body, app=app) + ch = _chord(header, body) + return ch.run(header, body, partial_args, app, interval, + countdown, max_retries, propagate, **kwargs) return Chord diff --git a/celery/app/task.py b/celery/app/task.py index 38eef7b5d..4687f29d2 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -93,6 +93,8 @@ class Context(object): headers = None delivery_info = None reply_to = None + root_id = None + parent_id = None correlation_id = None taskset = None # compat alias to group group = None diff --git a/celery/canvas.py b/celery/canvas.py index 5efb75b09..16924eeba 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -12,6 +12,7 @@ """ from __future__ import absolute_import +from collections import deque from copy import deepcopy from functools import partial as _partial, reduce from operator import itemgetter @@ -19,7 +20,7 @@ from kombu.utils import cached_property, fxrange, kwdict, reprcall, uuid -from celery._state import current_app +from celery._state import current_app, get_current_worker_task from celery.utils.functional import ( maybe_list, is_list, regen, chunks as _chunks, @@ -194,12 +195,13 @@ def clone(self, args=(), kwargs={}, **opts): return s partial = clone - def freeze(self, _id=None, group_id=None, chord=None): + def freeze(self, _id=None, group_id=None, chord=None, root_id=None): opts = self.options try: tid = opts['task_id'] except KeyError: tid = opts['task_id'] = _id or uuid() + root_id = opts.setdefault('root_id', root_id) if 'reply_to' not in opts: opts['reply_to'] = self.app.oid if group_id: @@ -348,6 +350,99 @@ def __call__(self, *args, **kwargs): if self.tasks: return self.apply_async(args, kwargs) + def apply_async(self, args=(), kwargs={}, group_id=None, chord=None, + task_id=None, link=None, link_error=None, + publisher=None, root_id=None, **options): + app = self.app + if app.conf.CELERY_ALWAYS_EAGER: + return self.apply(args, kwargs, **options) + tasks, results = self.prepare_steps( + args, self.tasks, root_id, link_error, + ) + if not results: + return + result = results[-1] + last_task = tasks[-1] + if group_id: + last_task.set(group_id=group_id) + if chord: + last_task.set(chord=chord) + if task_id: + last_task.set(task_id=task_id) + result = last_task.type.AsyncResult(task_id) + # make sure we can do a link() and link_error() on a chain object. + if link: + tasks[-1].set(link=link) + tasks[0].apply_async(**options) + return result + + def prepare_steps(self, args, tasks, + root_id=None, link_error=None, app=None): + app = app or self.app + steps = deque(tasks) + next_step = prev_task = prev_res = None + tasks, results = [], [] + i = 0 + while steps: + task = steps.popleft() + if not i: # first task + # first task gets partial args from chain + task = task.clone(args) + res = task.freeze(root_id=root_id) + root_id = res.id if root_id is None else root_id + else: + task = task.clone() + res = task.freeze(root_id=root_id) + i += 1 + + if isinstance(task, group): + task = maybe_unroll_group(task) + + if isinstance(task, chain): + # splice the chain + steps.extendleft(reversed(task.tasks)) + continue + elif isinstance(task, group) and steps and \ + not isinstance(steps[0], group): + # automatically upgrade group(...) | s to chord(group, s) + try: + next_step = steps.popleft() + # for chords we freeze by pretending it's a normal + # signature instead of a group. + res = Signature.freeze(next_step) + task = chord( + task, body=next_step, + task_id=res.task_id, root_id=root_id, + ) + except IndexError: + pass # no callback, so keep as group. + + if prev_task: + # link previous task to this task. + prev_task.link(task) + # set AsyncResult.parent + if not res.parent: + res.parent = prev_res + + if link_error: + task.set(link_error=link_error) + + if not isinstance(prev_task, chord): + results.append(res) + tasks.append(task) + prev_task, prev_res = task, res + + return tasks, results + + def apply(self, args=(), kwargs={}, **options): + last, fargs = None, args + for task in self.tasks: + res = task.clone(fargs).apply( + last and (last.get(), ), **options + ) + res.parent, last, fargs = last, res, None + return last + @classmethod def from_dict(self, d, app=None): tasks = d['kwargs']['tasks'] @@ -357,11 +452,14 @@ def from_dict(self, d, app=None): return chain(*d['kwargs']['tasks'], app=app, **kwdict(d['options'])) @property - def type(self): - try: - return self._type or self.tasks[0].type.app.tasks['celery.chain'] - except KeyError: - return self.app.tasks['celery.chain'] + def app(self): + app = self._app + if app is None: + try: + app = self.tasks[0]._app + except (KeyError, IndexError): + pass + return app or current_app def __repr__(self): return ' | '.join(repr(t) for t in self.tasks) @@ -452,11 +550,6 @@ def _maybe_group(tasks): return tasks -def _maybe_clone(tasks, app): - return [s.clone() if isinstance(s, Signature) else signature(s, app=app) - for s in tasks] - - @Signature.register_type class group(Signature): @@ -477,14 +570,58 @@ def from_dict(self, d, app=None): task['args'] = task._merge(d['args'])[0] return group(tasks, app=app, **kwdict(d['options'])) - def apply_async(self, args=(), kwargs=None, add_to_parent=True, **options): - tasks = _maybe_clone(self.tasks, app=self._app) - if not tasks: + def _prepared(self, tasks, partial_args, group_id, root_id): + for task in tasks: + task = task.clone(partial_args) + yield task, task.freeze(group_id=group_id, root_id=root_id) + + def _apply_tasks(self, tasks, producer=None, app=None, **options): + app = app or self.app + with app.producer_or_acquire(producer) as producer: + for sig, res in tasks: + sig.apply_async(producer=producer, add_to_parent=False, + **options) + yield res + + def _freeze_gid(self, options): + # remove task_id and use that as the group_id, + # if we don't remove it then every task will have the same id... + options = dict(self.options, **options) + options['group_id'] = group_id = ( + options.pop('task_id', uuid())) + return options, group_id, options.get('root_id') + + def apply_async(self, args=(), kwargs=None, add_to_parent=True, + producer=None, **options): + app = self.app + if app.conf.CELERY_ALWAYS_EAGER: + return self.apply(args, kwargs, **options) + if not self.tasks: return self.freeze() - type = self.type + + options, group_id, root_id = self._freeze_gid(options) + tasks = self._prepared(self.tasks, args, group_id, root_id) + result = self.app.GroupResult( + group_id, list(self._apply_tasks(tasks, producer, app, **options)), + ) + parent_task = get_current_worker_task() + if add_to_parent and parent_task: + parent_task.add_trail(result) + return result + return type(*type.prepare(dict(self.options, **options), tasks, args), add_to_parent=add_to_parent) + def apply(self, args=(), kwargs={}, **options): + app = self.app + if not self.tasks: + return self.freeze() # empty group returns GroupResult + options, group_id, root_id = self._freeze_gid(options) + tasks = self._prepared(self.tasks, args, group_id, root_id) + return app.GroupResult(group_id, [ + sig.apply(**options) for sig, _ in tasks + ]) + def set_immutable(self, immutable): for task in self.tasks: task.set_immutable(immutable) @@ -498,15 +635,10 @@ def link_error(self, sig): sig = sig.clone().set(immutable=True) return self.tasks[0].link_error(sig) - def apply(self, *args, **kwargs): - if not self.tasks: - return self.freeze() # empty group returns GroupResult - return Signature.apply(self, *args, **kwargs) - def __call__(self, *partial_args, **options): return self.apply_async(partial_args, **options) - def freeze(self, _id=None, group_id=None, chord=None): + def freeze(self, _id=None, group_id=None, chord=None, root_id=None): opts = self.options try: gid = opts['task_id'] @@ -516,10 +648,13 @@ def freeze(self, _id=None, group_id=None, chord=None): opts['group_id'] = group_id if chord: opts['chord'] = group_id + root_id = opts.setdefault('root_id', root_id) new_tasks, results = [], [] for task in self.tasks: task = maybe_signature(task, app=self._app).clone() - results.append(task.freeze(group_id=group_id, chord=chord)) + results.append(task.freeze( + group_id=group_id, chord=chord, root_id=root_id, + )) new_tasks.append(task) self.tasks = self.kwargs['tasks'] = new_tasks return self.app.GroupResult(gid, results) @@ -538,14 +673,14 @@ def __repr__(self): return repr(self.tasks) @property - def type(self): - if self._type: - return self._type - # taking the app from the first task in the list, there may be a - # better solution for this, e.g. to consolidate tasks with the same - # app and apply them in batches. - app = self._app if self._app else self.tasks[0].type.app - return app.tasks[self['task']] + def app(self): + app = self._app + if app is None: + try: + app = self.tasks[0]._app + except (KeyError, IndexError): + pass + return app if app is not None else current_app @Signature.register_type @@ -560,8 +695,8 @@ def __init__(self, header, body=None, task='celery.chord', ) self.subtask_type = 'chord' - def freeze(self, _id=None, group_id=None, chord=None): - return self.body.freeze(_id, group_id=group_id, chord=chord) + def freeze(self, *args, **kwargs): + return self.body.freeze(*args, **kwargs) @classmethod def from_dict(self, d, app=None): @@ -574,20 +709,14 @@ def _unpack_args(header=None, body=None, **kwargs): # than manually popping things off. return (header, body), kwargs - @property - def type(self): - if self._type: - return self._type - # we will be able to fix this mess in 3.2 when we no longer - # require an actual task implementation for chord/group - if self._app: - app = self._app - else: - try: - app = self.tasks[0].type.app - except IndexError: - app = self.body.type.app - return app.tasks['celery.chord'] + @cached_property + def app(self): + app = self._app + if app is None: + app = self.tasks[0]._app + if app is None: + app = self.body._app + return app if app is not None else current_app def apply_async(self, args=(), kwargs={}, task_id=None, producer=None, publisher=None, connection=None, @@ -595,14 +724,41 @@ def apply_async(self, args=(), kwargs={}, task_id=None, body = kwargs.get('body') or self.kwargs['body'] kwargs = dict(self.kwargs, **kwargs) body = body.clone(**options) + app = self.app + tasks = (self.tasks.clone() if isinstance(self.tasks, group) + else group(self.tasks)) + if app.conf.CELERY_ALWAYS_EAGER: + return self.apply((), kwargs, + body=body, task_id=task_id, **options) + return self.run(tasks, body, args, task_id=task_id, **options) + + def apply(self, args=(), kwargs={}, propagate=True, body=None, **options): + body = self.body if body is None else body + tasks = (self.tasks.clone() if isinstance(self.tasks, group) + else group(self.tasks)) + return body.apply( + args=(tasks.apply().get(propagate=propagate), ), + ) - _chord = self.type - if _chord.app.conf.CELERY_ALWAYS_EAGER: - return self.apply((), kwargs, task_id=task_id, **options) - res = body.freeze(task_id) - parent = _chord(self.tasks, body, args, **options) - res.parent = parent - return res + def run(self, header, body, partial_args, app=None, interval=None, + countdown=1, max_retries=None, propagate=None, eager=False, + task_id=None, **options): + app = app or self.app + propagate = (app.conf.CELERY_CHORD_PROPAGATES + if propagate is None else propagate) + group_id = uuid() + root_id = body.options.get('root_id') + body.setdefault('chord_size', len(header.tasks)) + results = header.freeze( + group_id=group_id, chord=body, root_id=root_id).results + bodyres = body.freeze(task_id, root_id=root_id) + + parent = app.backend.apply_chord( + header, partial_args, group_id, body, + interval=interval, countdown=countdown, + max_retries=max_retries, propagate=propagate, result=results) + bodyres.parent = parent + return bodyres def __call__(self, body=None, **options): return self.apply_async((), {'body': body} if body else {}, **options) diff --git a/celery/tests/app/test_builtins.py b/celery/tests/app/test_builtins.py index 9b00c1a25..305877f47 100644 --- a/celery/tests/app/test_builtins.py +++ b/celery/tests/app/test_builtins.py @@ -136,18 +136,18 @@ def test_apply_async(self): def test_group_to_chord(self): c = ( - group(self.add.s(i, i) for i in range(5)) | + group([self.add.s(i, i) for i in range(5)], app=self.app) | self.add.s(10) | self.add.s(20) | self.add.s(30) ) - tasks, _ = c.type.prepare_steps((), c.tasks) + tasks, _ = c.prepare_steps((), c.tasks) self.assertIsInstance(tasks[0], chord) self.assertTrue(tasks[0].body.options['link']) self.assertTrue(tasks[0].body.options['link'][0].options['link']) c2 = self.add.s(2, 2) | group(self.add.s(i, i) for i in range(10)) - tasks2, _ = c2.type.prepare_steps((), c2.tasks) + tasks2, _ = c2.prepare_steps((), c2.tasks) self.assertIsInstance(tasks2[1], group) def test_apply_options(self): @@ -158,7 +158,7 @@ def clone(self, *args, **kwargs): return self def s(*args, **kwargs): - return static(self.add, args, kwargs, type=self.add) + return static(self.add, args, kwargs, type=self.add, app=self.app) c = s(2, 2) | s(4, 4) | s(8, 8) r1 = c.apply_async(task_id='some_id') @@ -196,18 +196,16 @@ def test_run_header_not_group(self): def test_forward_options(self): body = self.xsum.s() x = chord([self.add.s(i, i) for i in range(10)], body=body) - x._type = Mock() - x._type.app.conf.CELERY_ALWAYS_EAGER = False + x.run = Mock(name='chord.run(x)') x.apply_async(group_id='some_group_id') - self.assertTrue(x._type.called) - resbody = x._type.call_args[0][1] + self.assertTrue(x.run.called) + resbody = x.run.call_args[0][1] self.assertEqual(resbody.options['group_id'], 'some_group_id') x2 = chord([self.add.s(i, i) for i in range(10)], body=body) - x2._type = Mock() - x2._type.app.conf.CELERY_ALWAYS_EAGER = False + x2.run = Mock(name='chord.run(x2)') x2.apply_async(chord='some_chord_id') - self.assertTrue(x2._type.called) - resbody = x2._type.call_args[0][1] + self.assertTrue(x2.run.called) + resbody = x2.run.call_args[0][1] self.assertEqual(resbody.options['chord'], 'some_chord_id') def test_apply_eager(self): diff --git a/celery/tests/tasks/test_chord.py b/celery/tests/tasks/test_chord.py index 531d06467..27424a30a 100644 --- a/celery/tests/tasks/test_chord.py +++ b/celery/tests/tasks/test_chord.py @@ -205,7 +205,7 @@ def test_apply(self): m = Mock() m.app.conf.CELERY_ALWAYS_EAGER = False m.AsyncResult = AsyncResult - prev, chord._type = chord._type, m + prev, chord.run = chord.run, m try: x = chord(self.add.s(i, i) for i in range(10)) body = self.add.s(2) @@ -214,9 +214,9 @@ def test_apply(self): # does not modify original signature with self.assertRaises(KeyError): body.options['task_id'] - self.assertTrue(chord._type.called) + self.assertTrue(chord.run.called) finally: - chord._type = prev + chord.run = prev class test_Chord_task(ChordCase): @@ -227,7 +227,7 @@ def test_run(self): self.app.backend.cleanup.__name__ = 'cleanup' Chord = self.app.tasks['celery.chord'] - body = dict() + body = self.add.signature() Chord(group(self.add.signature((i, i)) for i in range(5)), body) Chord([self.add.signature((j, j)) for j in range(5)], body) self.assertEqual(self.app.backend.apply_chord.call_count, 2) diff --git a/docs/internals/protov2.rst b/docs/internals/protov2.rst index e0bb1ff89..d36936c2e 100644 --- a/docs/internals/protov2.rst +++ b/docs/internals/protov2.rst @@ -50,6 +50,7 @@ Notes - ``correlation_id`` replaces ``task_id`` field. +- ``root_id`` and ``parent_id`` fields helps keep track of workflows. - ``c_shadow`` lets you specify a different name for logs, monitors can be used for e.g. meta tasks that calls any function:: @@ -115,6 +116,8 @@ Definition 'chord': (uuid)chord_id, 'retries': (int)retries, 'timelimit': (tuple)(soft, hard), + 'root_id': (uuid)root_id, + 'parent_id': (uuid)parent_id, } body = (args, kwargs) From f786b85c14a5df1bf3719422c85739f77690d750 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 8 May 2014 16:50:17 +0100 Subject: [PATCH 0107/1103] docstrings --- celery/app/builtins.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/app/builtins.py b/celery/app/builtins.py index f08bf5054..3e5f111c5 100644 --- a/celery/app/builtins.py +++ b/celery/app/builtins.py @@ -137,6 +137,7 @@ def chunks(task, it, n): @connect_on_app_finalize def add_group_task(app): + """No longer used, but here for backwards compatibility.""" _app = app from celery.canvas import maybe_signature from celery.result import result_from_tuple @@ -166,6 +167,7 @@ def run(self, tasks, result, group_id, partial_args, @connect_on_app_finalize def add_chain_task(app): + """No longer used, but here for backwards compatibility.""" _app = app class Chain(app.Task): @@ -179,9 +181,7 @@ class Chain(app.Task): @connect_on_app_finalize def add_chord_task(app): - """Every chord is executed in a dedicated task, so that the chord - can be used as a signature, and this generates the task - responsible for that.""" + """No longer used, but here for backwards compatibility.""" from celery import group, chord as _chord from celery.canvas import maybe_signature _app = app From 1e9dd26592eb2b93f1cb16deb771cfc65ab79612 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 8 May 2014 19:13:42 +0100 Subject: [PATCH 0108/1103] Super refactor, merging everything for 3.2 --- celery/app/amqp.py | 1 + celery/app/builtins.py | 1 - celery/app/task.py | 13 +- celery/app/trace.py | 160 +++++++++++++-- celery/concurrency/asynpool.py | 9 - celery/concurrency/base.py | 3 +- celery/tests/tasks/test_trace.py | 8 +- celery/worker/consumer.py | 25 ++- celery/worker/control.py | 4 +- celery/worker/job.py | 296 ++++++++-------------------- celery/worker/loops.py | 4 +- celery/worker/strategy.py | 24 ++- docs/internals/protov2.rst | 5 +- funtests/stress/stress/templates.py | 1 + 14 files changed, 280 insertions(+), 274 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index e3f62b731..b70532cef 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -297,6 +297,7 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, headers={ 'lang': 'py', 'c_type': name, + 'task_id': task_id, 'eta': eta, 'expires': expires, 'callbacks': callbacks, diff --git a/celery/app/builtins.py b/celery/app/builtins.py index 3e5f111c5..81d5f074c 100644 --- a/celery/app/builtins.py +++ b/celery/app/builtins.py @@ -64,7 +64,6 @@ def unlock_chord(group_id, callback, interval=None, propagate=None, # check if the task group is ready, and if so apply the callback. callback = maybe_signature(callback, app) - root_id = callback.options.get('root_id') deps = GroupResult( group_id, [result_from_tuple(r, app=app) for r in result], diff --git a/celery/app/task.py b/celery/app/task.py index 4687f29d2..705c26269 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -695,7 +695,7 @@ def apply(self, args=None, kwargs=None, """ # trace imports Task, so need to import inline. - from celery.app.trace import eager_trace_task + from celery.app.trace import build_tracer app = self._get_app() args = args or () @@ -736,12 +736,15 @@ def apply(self, args=None, kwargs=None, kwargs.update(extend_with) tb = None - retval, info = eager_trace_task(task, task_id, args, kwargs, - app=self._get_app(), - request=request, propagate=throw) + tracer = build_tracer( + task.name, task, eager=True, + propagate=throw, app=self._get_app(), + ) + ret = tracer(task_id, args, kwargs, request) + retval = ret.retval if isinstance(retval, ExceptionInfo): retval, tb = retval.exception, retval.traceback - state = states.SUCCESS if info is None else info.state + state = states.SUCCESS if ret.info is None else ret.info.state return EagerResult(task_id, retval, state, traceback=tb) def AsyncResult(self, task_id, **kwargs): diff --git a/celery/app/trace.py b/celery/app/trace.py index 45e24c170..03e07423e 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -15,33 +15,68 @@ # but in the end it only resulted in bad performance and horrible tracebacks, # so instead we now use one closure per task class. +import logging import os import socket import sys +from collections import namedtuple from warnings import warn from billiard.einfo import ExceptionInfo from kombu.exceptions import EncodeError -from kombu.utils import kwdict +from kombu.serialization import decode as decode_message +from kombu.utils.encoding import safe_repr, safe_str from celery import current_app, group from celery import states, signals from celery._state import _task_stack from celery.app import set_default_app from celery.app.task import Task as BaseTask, Context -from celery.exceptions import Ignore, Reject, Retry +from celery.exceptions import Ignore, Reject, Retry, InvalidTaskError +from celery.five import monotonic from celery.utils.log import get_logger from celery.utils.objects import mro_lookup from celery.utils.serialization import ( - get_pickleable_exception, - get_pickleable_etype, + get_pickleable_exception, get_pickled_exception, get_pickleable_etype, ) +from celery.utils.text import truncate -__all__ = ['TraceInfo', 'build_tracer', 'trace_task', 'eager_trace_task', +__all__ = ['TraceInfo', 'build_tracer', 'trace_task', 'setup_worker_optimizations', 'reset_worker_optimizations'] -_logger = get_logger(__name__) +logger = get_logger(__name__) +info = logger.info + +#: Format string used to log task success. +LOG_SUCCESS = """\ +Task %(name)s[%(id)s] succeeded in %(runtime)ss: %(return_value)s\ +""" + +#: Format string used to log task failure. +LOG_FAILURE = """\ +Task %(name)s[%(id)s] %(description)s: %(exc)s\ +""" + +#: Format string used to log task internal error. +LOG_INTERNAL_ERROR = """\ +Task %(name)s[%(id)s] %(description)s: %(exc)s\ +""" + +#: Format string used to log task ignored. +LOG_IGNORED = """\ +Task %(name)s[%(id)s] %(description)s\ +""" + +#: Format string used to log task rejected. +LOG_REJECTED = """\ +Task %(name)s[%(id)s] %(exc)s\ +""" + +#: Format string used to log task retry. +LOG_RETRY = """\ +Task %(name)s[%(id)s] retry: %(exc)s\ +""" send_prerun = signals.task_prerun.send send_postrun = signals.task_postrun.send @@ -59,6 +94,8 @@ _tasks = None _patched = {} +trace_ok_t = namedtuple('trace_ok_t', ('retval', 'info', 'runtime', 'retstr')) + def task_has_custom(task, attr): """Return true if the task or one of its bases @@ -100,6 +137,10 @@ def handle_retry(self, task, store_errors=True): task.on_retry(reason.exc, req.id, req.args, req.kwargs, einfo) signals.task_retry.send(sender=task, request=req, reason=reason, einfo=einfo) + info(LOG_RETRY, { + 'id': req.id, 'name': task.name, + 'exc': safe_repr(reason.exc), + }) return einfo finally: del(tb) @@ -123,14 +164,71 @@ def handle_failure(self, task, store_errors=True): kwargs=req.kwargs, traceback=tb, einfo=einfo) + self._log_error(task, einfo) return einfo finally: del(tb) + def _log_error(self, task, einfo): + req = task.request + eobj = einfo.exception = get_pickled_exception(einfo.exception) + exception, traceback, exc_info, internal, sargs, skwargs = ( + safe_repr(eobj), + safe_str(einfo.traceback), + einfo.exc_info, + einfo.internal, + safe_repr(req.args), + safe_repr(req.kwargs), + ) + if task.throws and isinstance(eobj, task.throws): + do_send_mail, severity, exc_info, description = ( + False, logging.INFO, None, 'raised expected', + ) + else: + do_send_mail, severity, description = ( + True, logging.ERROR, 'raised unexpected', + ) + format = LOG_FAILURE + + if internal: + if isinstance(einfo.exception, Reject): + format = LOG_REJECTED + description = 'rejected' + severity = logging.WARN + exc_info = einfo + elif isinstance(einfo.exception, Ignore): + format = LOG_IGNORED + description = 'ignored' + severity = logging.INFO + exc_info = None + else: + format = LOG_INTERNAL_ERROR + description = 'INTERNAL ERROR' + severity = logging.CRITICAL + + context = { + 'hostname': req.hostname, + 'id': req.id, + 'name': task.name, + 'exc': exception, + 'traceback': traceback, + 'args': sargs, + 'kwargs': skwargs, + 'description': description, + 'internal': internal, + } + + logger.log(severity, format.strip(), context, + exc_info=exc_info, + extra={'data': context}) + + task.send_error_email(context, einfo.exception) + def build_tracer(name, task, loader=None, hostname=None, store_errors=True, Info=TraceInfo, eager=False, propagate=False, app=None, - IGNORE_STATES=IGNORE_STATES): + monotonic=monotonic, truncate=truncate, + trace_ok_t=trace_ok_t, IGNORE_STATES=IGNORE_STATES): """Return a function that traces task execution; catches all exceptions and updates result backend with the state and result @@ -186,6 +284,7 @@ def build_tracer(name, task, loader=None, hostname=None, store_errors=True, push_task = _task_stack.push pop_task = _task_stack.pop on_chord_part_return = backend.on_chord_part_return + _does_info = logger.isEnabledFor(logging.INFO) prerun_receivers = signals.task_prerun.receivers postrun_receivers = signals.task_postrun.receivers @@ -209,6 +308,8 @@ def on_error(request, exc, uuid, state=FAILURE, call_errbacks=True): def trace_task(uuid, args, kwargs, request=None): # R - is the possibly prepared return value. # I - is the Info object. + # T - runtime + # Rstr - textual representation of return value # retval - is the always unmodified return value. # state - is the resulting task state. @@ -216,9 +317,14 @@ def trace_task(uuid, args, kwargs, request=None): # for performance reasons, and because the function is so long # we want the main variables (I, and R) to stand out visually from the # the rest of the variables, so breaking PEP8 is worth it ;) - R = I = retval = state = None - kwargs = kwdict(kwargs) + R = I = T = Rstr = retval = state = None + time_start = monotonic() try: + try: + kwargs.items + except AttributeError: + raise InvalidTaskError( + 'Task keyword arguments is not a mapping') push_task(task) task_request = Context(request or {}, args=args, called_directly=False, kwargs=kwargs) @@ -289,6 +395,13 @@ def trace_task(uuid, args, kwargs, request=None): task_on_success(retval, uuid, args, kwargs) if success_receivers: send_success(sender=task, result=retval) + if _does_info: + T = monotonic() - time_start + Rstr = truncate(safe_repr(R), 256) + info(LOG_SUCCESS, { + 'id': uuid, 'name': name, + 'return_value': Rstr, 'runtime': T, + }) # -* POST *- if state not in IGNORE_STATES: @@ -314,15 +427,15 @@ def trace_task(uuid, args, kwargs, request=None): except (KeyboardInterrupt, SystemExit, MemoryError): raise except Exception as exc: - _logger.error('Process cleanup failed: %r', exc, - exc_info=True) + logger.error('Process cleanup failed: %r', exc, + exc_info=True) except MemoryError: raise except Exception as exc: if eager: raise R = report_internal_error(task, exc) - return R, I + return trace_ok_t(R, I, T, Rstr) return trace_task @@ -342,16 +455,23 @@ def _trace_task_ret(name, uuid, args, kwargs, request={}, app=None, **opts): trace_task_ret = _trace_task_ret -def _fast_trace_task(task, uuid, args, kwargs, request={}): +def _fast_trace_task_v1(task, uuid, args, kwargs, request={}): # setup_worker_optimizations will point trace_task_ret to here, # so this is the function used in the worker. - return _tasks[task].__trace__(uuid, args, kwargs, request)[0] - - -def eager_trace_task(task, uuid, args, kwargs, request=None, **opts): - opts.setdefault('eager', True) - return build_tracer(task.name, task, **opts)( - uuid, args, kwargs, request) + R, I, T, Rstr = _tasks[task].__trace__(uuid, args, kwargs, request)[0] + # exception instance if error, else result text + return (R if I else Rstr), T + + +def _fast_trace_task(task, uuid, request, body, content_type, + content_encoding, decode_message=decode_message, + **extra_request): + args, kwargs = decode_message(body, content_type, content_encoding) + request.update(args=args, kwargs=kwargs, **extra_request) + R, I, T, Rstr = _tasks[task].__trace__( + uuid, args, kwargs, request, + ) + return (R if I else Rstr), T def report_internal_error(task, exc): diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index a3906c492..c2dbb0241 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -37,7 +37,6 @@ from billiard.pool import RUN, TERMINATE, ACK, NACK, WorkersJoined from billiard import pool as _pool from billiard.compat import buf_t, setblocking, isblocking -from billiard.einfo import ExceptionInfo from billiard.queues import _SimpleQueue from kombu.async import READ, WRITE, ERR from kombu.serialization import pickle as _pickle @@ -46,7 +45,6 @@ from kombu.utils.eventio import SELECT_BAD_FD from celery.five import Counter, items, values from celery.utils.log import get_logger -from celery.utils.text import truncate from celery.worker import state as worker_state try: @@ -96,8 +94,6 @@ def unpack_from(fmt, iobuf, unpack=struct.unpack): # noqa 'fair': SCHED_STRATEGY_FAIR, } -RESULT_MAXLEN = 128 - Ack = namedtuple('Ack', ('id', 'fd', 'payload')) @@ -170,11 +166,6 @@ def on_loop_start(self, pid): # is writable. self.outq.put((WORKER_UP, (pid, ))) - def prepare_result(self, result, RESULT_MAXLEN=RESULT_MAXLEN): - if not isinstance(result, ExceptionInfo): - return truncate(repr(result), RESULT_MAXLEN) - return result - class ResultHandler(_pool.ResultHandler): """Handles messages from the pool processes.""" diff --git a/celery/concurrency/base.py b/celery/concurrency/base.py index 29c348d6a..24b39a1ee 100644 --- a/celery/concurrency/base.py +++ b/celery/concurrency/base.py @@ -66,6 +66,7 @@ class BasePool(object): _state = None _pool = None + _does_debug = True #: only used by multiprocessing pool uses_semaphore = False @@ -79,7 +80,6 @@ def __init__(self, limit=None, putlocks=True, self.options = options self.forking_enable = forking_enable self.callbacks_propagate = callbacks_propagate - self._does_debug = logger.isEnabledFor(logging.DEBUG) def on_start(self): pass @@ -128,6 +128,7 @@ def terminate(self): self.on_terminate() def start(self): + self._does_debug = logger.isEnabledFor(logging.DEBUG) self.on_start() self._state = self.RUN diff --git a/celery/tests/tasks/test_trace.py b/celery/tests/tasks/test_trace.py index 12c6280ef..0a7ec3345 100644 --- a/celery/tests/tasks/test_trace.py +++ b/celery/tests/tasks/test_trace.py @@ -6,7 +6,7 @@ from celery.exceptions import Ignore, Retry from celery.app.trace import ( TraceInfo, - eager_trace_task, + build_tracer, trace_task, setup_worker_optimizations, reset_worker_optimizations, @@ -15,8 +15,10 @@ def trace(app, task, args=(), kwargs={}, propagate=False, **opts): - return eager_trace_task(task, 'id-1', args, kwargs, - propagate=propagate, app=app, **opts) + t = build_tracer(task.name, task, + eager=True, propagate=propagate, app=app, **opts) + ret = t('id-1', args, kwargs, None) + return ret.retval, ret.info class TraceCase(AppCase): diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index c761d043a..33facabd1 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -127,6 +127,8 @@ class buffer_t(object): # noqa def dump_body(m, body): + # v2 protocol does not deserialize body + body = m.body if body is None else body if isinstance(body, buffer_t): body = bytes_t(body) return '{0} ({1}b)'.format(truncate(safe_repr(body), 1024), @@ -445,7 +447,7 @@ def create_task_handler(self): on_invalid_task = self.on_invalid_task callbacks = self.on_task_message - def on_task_received(body, message): + def on_v1_task_received(body, message): try: name = body['task'] except (KeyError, TypeError): @@ -461,6 +463,22 @@ def on_task_received(body, message): except InvalidTaskError as exc: on_invalid_task(body, message, exc) + def on_task_received(message): + headers = message.headers + try: + type_ = headers['c_type'] + except KeyError: + return on_v1_task_received(message.payload, message) + try: + strategies[type_]( + message, None, + message.ack_log_error, message.reject_log_error, callbacks, + ) + except KeyError as exc: + on_unknown_task(None, message, exc) + except InvalidTaskError as exc: + on_invalid_task(None, message, exc) + return on_task_received def __repr__(self): @@ -541,8 +559,9 @@ def __init__(self, c, without_heartbeat=False, heartbeat_interval=None, c.heart = None def start(self, c): - c.heart = heartbeat.Heart(c.timer, c.event_dispatcher, - self.heartbeat_interval) + c.heart = heartbeat.Heart( + c.timer, c.event_dispatcher, self.heartbeat_interval, + ) c.heart.start() def stop(self, c): diff --git a/celery/worker/control.py b/celery/worker/control.py index 6016543c7..2067d4043 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -364,7 +364,9 @@ def active_queues(state): def _wanted_config_key(key): - return isinstance(key, string_t) and key.isupper() and not key.startswith('__') + return (isinstance(key, string_t) and + key.isupper() and + not key.startswith('__')) @Panel.register diff --git a/celery/worker/job.py b/celery/worker/job.py index 8522d0091..74278cc1e 100644 --- a/celery/worker/job.py +++ b/celery/worker/job.py @@ -17,7 +17,6 @@ from datetime import datetime from weakref import ref -from kombu.utils import kwdict, reprcall from kombu.utils.encoding import safe_repr, safe_str from celery import signals @@ -27,14 +26,12 @@ SoftTimeLimitExceeded, TimeLimitExceeded, WorkerLostError, Terminated, Retry, Reject, ) -from celery.five import items, monotonic, string, string_t +from celery.five import string from celery.platforms import signals as _signals -from celery.utils import fun_takes_kwargs from celery.utils.functional import noop from celery.utils.log import get_logger -from celery.utils.serialization import get_pickled_exception -from celery.utils.text import truncate from celery.utils.timeutils import maybe_iso8601, timezone, maybe_make_aware +from celery.utils.serialization import get_pickled_exception from . import state @@ -69,8 +66,6 @@ def __optimize__(): task_ready = state.task_ready revoked_tasks = state.revoked -NEEDS_KWDICT = sys.version_info <= (2, 6) - #: Use when no message object passed to :class:`Request`. DEFAULT_FIELDS = { 'headers': None, @@ -85,63 +80,46 @@ def __optimize__(): } +class RequestV1(object): + if not IS_PYPY: + __slots__ = ( + 'app', 'name', 'id', 'root_id', 'parent_id', + 'on_ack', 'hostname', 'eventer', 'connection_errors', 'task', + 'eta', 'expires', 'request_dict', 'acknowledged', 'on_reject', + 'utc', 'time_start', 'worker_pid', '_already_revoked', + '_terminate_on_ack', '_apply_result', + '_tzlocal', '__weakref__', '__dict__', + ) + + class Request(object): """A request for task execution.""" if not IS_PYPY: # pragma: no cover __slots__ = ( - 'app', 'name', 'id', 'args', 'kwargs', 'on_ack', + 'app', 'name', 'id', 'on_ack', 'payload', 'hostname', 'eventer', 'connection_errors', 'task', 'eta', 'expires', 'request_dict', 'acknowledged', 'on_reject', - 'utc', 'time_start', 'worker_pid', '_already_revoked', - '_terminate_on_ack', '_apply_result', + 'utc', 'time_start', 'worker_pid', 'timeouts', + 'content_type', 'content_encoding', + '_already_revoked', '_terminate_on_ack', '_apply_result', '_tzlocal', '__weakref__', '__dict__', ) - #: Format string used to log task success. - success_msg = """\ - Task %(name)s[%(id)s] succeeded in %(runtime)ss: %(return_value)s - """ - - #: Format string used to log task failure. - error_msg = """\ - Task %(name)s[%(id)s] %(description)s: %(exc)s - """ - - #: Format string used to log internal error. - internal_error_msg = """\ - Task %(name)s[%(id)s] %(description)s: %(exc)s - """ - - ignored_msg = """\ - Task %(name)s[%(id)s] %(description)s - """ - - rejected_msg = """\ - Task %(name)s[%(id)s] %(exc)s - """ - - #: Format string used to log task retry. - retry_msg = """Task %(name)s[%(id)s] retry: %(exc)s""" - - def __init__(self, body, on_ack=noop, + def __init__(self, message, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, - message=None, task=None, on_reject=noop, **opts): + task=None, on_reject=noop, **opts): + headers = message.headers self.app = app - name = self.name = body['task'] - self.id = body['id'] - self.args = body.get('args', []) - self.kwargs = body.get('kwargs', {}) - try: - self.kwargs.items - except AttributeError: - raise InvalidTaskError( - 'Task keyword arguments is not a mapping') - if NEEDS_KWDICT: - self.kwargs = kwdict(self.kwargs) - eta = body.get('eta') - expires = body.get('expires') - utc = self.utc = body.get('utc', False) + name = self.name = headers['c_type'] + self.id = headers['task_id'] + self.payload = message.body + self.content_type = message.content_type + self.content_encoding = message.content_encoding + eta = headers.get('eta') + expires = headers.get('expires') + self.timeouts = (headers['timeouts'] if 'timeouts' in headers + else (None, None)) self.on_ack = on_ack self.on_reject = on_reject self.hostname = hostname or socket.gethostname() @@ -157,75 +135,42 @@ def __init__(self, body, on_ack=noop, # supported at this point is UTC. if eta is not None: try: - self.eta = maybe_iso8601(eta) + eta = maybe_iso8601(eta) except (AttributeError, ValueError, TypeError) as exc: raise InvalidTaskError( 'invalid eta value {0!r}: {1}'.format(eta, exc)) - if utc: - self.eta = maybe_make_aware(self.eta, self.tzlocal) + self.eta = maybe_make_aware(eta, self.tzlocal) else: self.eta = None if expires is not None: try: - self.expires = maybe_iso8601(expires) + expires = maybe_iso8601(expires) except (AttributeError, ValueError, TypeError) as exc: raise InvalidTaskError( 'invalid expires value {0!r}: {1}'.format(expires, exc)) - if utc: - self.expires = maybe_make_aware(self.expires, self.tzlocal) + self.expires = maybe_make_aware(expires, self.tzlocal) else: self.expires = None - if message: - delivery_info = message.delivery_info or {} - properties = message.properties or {} - body.update({ - 'headers': message.headers, - 'reply_to': properties.get('reply_to'), - 'correlation_id': properties.get('correlation_id'), - 'delivery_info': { - 'exchange': delivery_info.get('exchange'), - 'routing_key': delivery_info.get('routing_key'), - 'priority': delivery_info.get('priority'), - 'redelivered': delivery_info.get('redelivered'), - } - - }) - else: - body.update(DEFAULT_FIELDS) - self.request_dict = body + delivery_info = message.delivery_info or {} + properties = message.properties or {} + headers.update({ + 'reply_to': properties.get('reply_to'), + 'correlation_id': properties.get('correlation_id'), + 'delivery_info': { + 'exchange': delivery_info.get('exchange'), + 'routing_key': delivery_info.get('routing_key'), + 'priority': delivery_info.get('priority'), + 'redelivered': delivery_info.get('redelivered'), + } + + }) + self.request_dict = headers @property def delivery_info(self): return self.request_dict['delivery_info'] - def extend_with_default_kwargs(self): - """Extend the tasks keyword arguments with standard task arguments. - - Currently these are `logfile`, `loglevel`, `task_id`, - `task_name`, `task_retries`, and `delivery_info`. - - See :meth:`celery.task.base.Task.run` for more information. - - Magic keyword arguments are deprecated and will be removed - in version 4.0. - - """ - kwargs = dict(self.kwargs) - default_kwargs = {'logfile': None, # deprecated - 'loglevel': None, # deprecated - 'task_id': self.id, - 'task_name': self.name, - 'task_retries': self.request_dict.get('retries', 0), - 'task_is_eager': False, - 'delivery_info': self.delivery_info} - fun = self.task.run - supported_keys = fun_takes_kwargs(fun, default_kwargs) - extend_with = {key: val for key, val in items(default_kwargs) - if key in supported_keys} - kwargs.update(extend_with) - return kwargs - def execute_using_pool(self, pool, **kwargs): """Used by the worker to send this task to the pool. @@ -235,32 +180,28 @@ def execute_using_pool(self, pool, **kwargs): and ignored. """ - uuid = self.id + task_id = self.id task = self.task if self.revoked(): - raise TaskRevokedError(uuid) + raise TaskRevokedError(task_id) - hostname = self.hostname - kwargs = self.kwargs - if task.accept_magic_kwargs: - kwargs = self.extend_with_default_kwargs() - request = self.request_dict - request.update({'hostname': hostname, 'is_eager': False, - 'delivery_info': self.delivery_info, - 'group': self.request_dict.get('taskset')}) - timeout, soft_timeout = request.get('timelimit', (None, None)) + payload = self.payload + timeout, soft_timeout = self.timeouts timeout = timeout or task.time_limit soft_timeout = soft_timeout or task.soft_time_limit result = pool.apply_async( trace_task_ret, - args=(self.name, uuid, self.args, kwargs, request), + args=(self.name, task_id, self.request_dict, + bytes(payload) if isinstance(payload, buffer) else payload, + self.content_type, self.content_encoding), + kwargs={'hostname': self.hostname, 'is_eager': False}, accept_callback=self.on_accepted, timeout_callback=self.on_timeout, callback=self.on_success, error_callback=self.on_failure, - soft_timeout=soft_timeout, - timeout=timeout, - correlation_id=uuid, + soft_timeout=soft_timeout or task.soft_time_limit, + timeout=timeout or task.time_limit, + correlation_id=task_id, ) # cannot create weakref to None self._apply_result = ref(result) if result is not None else result @@ -281,8 +222,6 @@ def execute(self, loglevel=None, logfile=None): self.acknowledge() kwargs = self.kwargs - if self.task.accept_magic_kwargs: - kwargs = self.extend_with_default_kwargs() request = self.request_dict request.update({'loglevel': loglevel, 'logfile': logfile, 'hostname': self.hostname, 'is_eager': False, @@ -374,7 +313,7 @@ def on_timeout(self, soft, timeout): if self.task.acks_late: self.acknowledge() - def on_success(self, ret_value, now=None, nowfun=monotonic): + def on_success(self, ret_value, **kwargs): """Handler called if the task was successfully processed.""" if isinstance(ret_value, ExceptionInfo): if isinstance(ret_value.exception, ( @@ -387,18 +326,10 @@ def on_success(self, ret_value, now=None, nowfun=monotonic): self.acknowledge() if self.eventer and self.eventer.enabled: - now = nowfun() - runtime = self.time_start and (now - self.time_start) or 0 - self.send_event('task-succeeded', - result=safe_repr(ret_value), runtime=runtime) - - if _does_info: - now = now or nowfun() - runtime = self.time_start and (now - self.time_start) or 0 - info(self.success_msg.strip(), { - 'id': self.id, 'name': self.name, - 'return_value': self.repr_result(ret_value), - 'runtime': runtime}) + result, runtime = ret_value + self.send_event( + 'task-succeeded', result=ret_value, runtime=runtime, + ) def on_retry(self, exc_info): """Handler called if the task should be retried.""" @@ -409,17 +340,19 @@ def on_retry(self, exc_info): exception=safe_repr(exc_info.exception.exc), traceback=safe_str(exc_info.traceback)) - if _does_info: - info(self.retry_msg.strip(), - {'id': self.id, 'name': self.name, - 'exc': exc_info.exception}) - def on_failure(self, exc_info): """Handler called if the task raised an exception.""" task_ready(self) send_failed_event = True - if not exc_info.internal: + if exc_info.internal: + if isinstance(exc_info.exception, MemoryError): + raise MemoryError('Process got: %s' % (exc_info.exception, )) + elif isinstance(exc_info.exception, Reject): + self.reject(requeue=exc_info.exception.requeue) + elif isinstance(exc_info.exception, Ignore): + self.acknowledge() + else: exc = exc_info.exception if isinstance(exc, Retry): @@ -439,77 +372,14 @@ def on_failure(self, exc_info): # (acks_late) acknowledge after result stored. if self.task.acks_late: self.acknowledge() - self._log_error(exc_info, send_failed_event=send_failed_event) - - def _log_error(self, einfo, send_failed_event=True): - einfo.exception = get_pickled_exception(einfo.exception) - eobj = einfo.exception - exception, traceback, exc_info, internal, sargs, skwargs = ( - safe_repr(eobj), - safe_str(einfo.traceback), - einfo.exc_info, - einfo.internal, - safe_repr(self.args), - safe_repr(self.kwargs), - ) - task = self.task - if task.throws and isinstance(eobj, task.throws): - do_send_mail, severity, exc_info, description = ( - False, logging.INFO, None, 'raised expected', - ) - else: - do_send_mail, severity, description = ( - True, logging.ERROR, 'raised unexpected', - ) - format = self.error_msg + if send_failed_event: self.send_event( - 'task-failed', exception=exception, traceback=traceback, + 'task-failed', + exception=safe_repr(get_pickled_exception(exc_info.exception)), + traceback=exc_info.traceback, ) - if internal: - if isinstance(einfo.exception, MemoryError): - raise MemoryError('Process got: %s' % (einfo.exception, )) - elif isinstance(einfo.exception, Reject): - format = self.rejected_msg - description = 'rejected' - severity = logging.WARN - exc_info = einfo - self.reject(requeue=einfo.exception.requeue) - elif isinstance(einfo.exception, Ignore): - format = self.ignored_msg - description = 'ignored' - severity = logging.INFO - exc_info = None - self.acknowledge() - else: - format = self.internal_error_msg - description = 'INTERNAL ERROR' - severity = logging.CRITICAL - - context = { - 'hostname': self.hostname, - 'id': self.id, - 'name': self.name, - 'exc': exception, - 'traceback': traceback, - 'args': sargs, - 'kwargs': skwargs, - 'description': description, - } - - logger.log(severity, format.strip(), context, - exc_info=exc_info, - extra={'data': {'id': self.id, - 'name': self.name, - 'args': sargs, - 'kwargs': skwargs, - 'hostname': self.hostname, - 'internal': internal}}) - - if do_send_mail: - task.send_error_email(context, einfo.exception) - def acknowledge(self): """Acknowledge task.""" if not self.acknowledged: @@ -521,18 +391,10 @@ def reject(self, requeue=False): self.on_reject(logger, self.connection_errors, requeue) self.acknowledged = True - def repr_result(self, result, maxlen=RESULT_MAXLEN): - # 46 is the length needed to fit - # 'the quick brown fox jumps over the lazy dog' :) - if not isinstance(result, string_t): - result = safe_repr(result) - return truncate(result) if len(result) > maxlen else result - def info(self, safe=False): return {'id': self.id, 'name': self.name, - 'args': self.args if safe else safe_repr(self.args), - 'kwargs': self.kwargs if safe else safe_repr(self.kwargs), + 'body': self.body, 'hostname': self.hostname, 'time_start': self.time_start, 'acknowledged': self.acknowledged, @@ -546,9 +408,7 @@ def __str__(self): shortinfo = __str__ def __repr__(self): - return '<{0} {1}: {2}>'.format( - type(self).__name__, self.id, - reprcall(self.name, self.args, self.kwargs)) + return '<{0} {1}: {2}>'.format(type(self).__name__, self.id, self.name) @property def tzlocal(self): diff --git a/celery/worker/loops.py b/celery/worker/loops.py index 0891f51a6..406633e00 100644 --- a/celery/worker/loops.py +++ b/celery/worker/loops.py @@ -37,7 +37,7 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos, if heartbeat and connection.supports_heartbeats: hub.call_repeatedly(heartbeat / hbrate, hbtick, hbrate) - consumer.callbacks = [on_task_received] + consumer.on_message = on_task_received consumer.consume() obj.on_ready() obj.controller.register_with_event_loop(hub) @@ -86,7 +86,7 @@ def synloop(obj, connection, consumer, blueprint, hub, qos, """Fallback blocking event loop for transports that doesn't support AIO.""" on_task_received = obj.create_task_handler() - consumer.register_callback(on_task_received) + consumer.on_message = on_task_received consumer.consume() obj.on_ready() diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index 0b0d327c3..19a31ef90 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -11,12 +11,11 @@ import logging from kombu.async.timer import to_timestamp -from kombu.utils.encoding import safe_repr from celery.utils.log import get_logger from celery.utils.timeutils import timezone -from .job import Request +from .job import Request, RequestV1 from .state import task_reserved __all__ = ['default'] @@ -29,7 +28,8 @@ def default(task, app, consumer, to_system_tz=timezone.to_system): hostname = consumer.hostname eventer = consumer.event_dispatcher - Req = Request + ReqV2 = Request + ReqV1 = RequestV1 connection_errors = consumer.connection_errors _does_info = logger.isEnabledFor(logging.INFO) events = eventer and eventer.enabled @@ -43,11 +43,17 @@ def default(task, app, consumer, def task_message_handler(message, body, ack, reject, callbacks, to_timestamp=to_timestamp): - req = Req(body, on_ack=ack, on_reject=reject, - app=app, hostname=hostname, - eventer=eventer, task=task, - connection_errors=connection_errors, - message=message) + if body is None: + req = ReqV2(message, + on_ack=ack, on_reject=reject, app=app, + hostname=hostname, eventer=eventer, task=task, + connection_errors=connection_errors) + else: + req = ReqV1(body, + on_ack=ack, on_reject=reject, app=app, + hostname=hostname, eventer=eventer, task=task, + connection_errors=connection_errors, + message=message) if req.revoked(): return @@ -58,7 +64,7 @@ def task_message_handler(message, body, ack, reject, callbacks, send_event( 'task-received', uuid=req.id, name=req.name, - args=safe_repr(req.args), kwargs=safe_repr(req.kwargs), + args='', kwargs='', retries=req.request_dict.get('retries', 0), eta=req.eta and req.eta.isoformat(), expires=req.expires and req.expires.isoformat(), diff --git a/docs/internals/protov2.rst b/docs/internals/protov2.rst index d36936c2e..f3c8359ca 100644 --- a/docs/internals/protov2.rst +++ b/docs/internals/protov2.rst @@ -103,6 +103,9 @@ Definition headers = { 'lang': (string)'py' 'c_type': (string)task, + 'task_id': (uuid)task_id, + 'root_id': (uuid)root_id, + 'parent_id': (uuid)parent_id, # optional 'c_meth': (string)unused, @@ -116,8 +119,6 @@ Definition 'chord': (uuid)chord_id, 'retries': (int)retries, 'timelimit': (tuple)(soft, hard), - 'root_id': (uuid)root_id, - 'parent_id': (uuid)parent_id, } body = (args, kwargs) diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index 931269807..520481108 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -70,6 +70,7 @@ class default(object): 'interval_max': 2, 'interval_step': 0.1, } + CELERY_TASK_PROTOCOL = 2 @template() From e442df61b2ff1fe855881c1e2ff9acc970090f54 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 9 May 2014 17:42:40 +0100 Subject: [PATCH 0109/1103] Tests passing, more refactoring --- celery/app/amqp.py | 2 +- celery/app/trace.py | 109 +++++--- celery/events/state.py | 6 +- celery/tests/case.py | 42 ++- celery/tests/tasks/test_trace.py | 7 +- celery/tests/worker/test_control.py | 27 +- celery/tests/worker/test_loops.py | 43 ++- celery/tests/worker/test_request.py | 400 ++++++++++----------------- celery/tests/worker/test_strategy.py | 13 +- celery/tests/worker/test_worker.py | 138 +++++---- celery/worker/autoscale.py | 2 +- celery/worker/consumer.py | 35 +-- celery/worker/job.py | 96 ++++--- celery/worker/strategy.py | 2 +- docs/internals/protov2.rst | 4 +- 15 files changed, 430 insertions(+), 496 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index b70532cef..e8e801529 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -297,7 +297,7 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, headers={ 'lang': 'py', 'c_type': name, - 'task_id': task_id, + 'id': task_id, 'eta': eta, 'expires': expires, 'callbacks': callbacks, diff --git a/celery/app/trace.py b/celery/app/trace.py index 03e07423e..60776b9ff 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -25,7 +25,7 @@ from billiard.einfo import ExceptionInfo from kombu.exceptions import EncodeError -from kombu.serialization import decode as decode_message +from kombu.serialization import loads as loads_message, prepare_accept_content from kombu.utils.encoding import safe_repr, safe_str from celery import current_app, group @@ -78,6 +78,22 @@ Task %(name)s[%(id)s] retry: %(exc)s\ """ +log_policy_t = namedtuple( + 'log_policy_t', ('format', 'description', 'severity', 'traceback', 'mail'), +) + +log_policy_reject = log_policy_t(LOG_REJECTED, 'rejected', logging.WARN, 1, 1) +log_policy_ignore = log_policy_t(LOG_IGNORED, 'ignored', logging.INFO, 0, 0) +log_policy_internal = log_policy_t( + LOG_INTERNAL_ERROR, 'INTERNAL ERROR', logging.CRITICAL, 1, 1, +) +log_policy_expected = log_policy_t( + LOG_FAILURE, 'raised expected', logging.INFO, 0, 0, +) +log_policy_unexpected = log_policy_t( + LOG_FAILURE, 'raised unexpected', logging.ERROR, 1, 1, +) + send_prerun = signals.task_prerun.send send_postrun = signals.task_postrun.send send_success = signals.task_success.send @@ -91,7 +107,7 @@ IGNORE_STATES = frozenset([IGNORED, RETRY, REJECTED]) #: set by :func:`setup_worker_optimizations` -_tasks = None +_localized = [] _patched = {} trace_ok_t = namedtuple('trace_ok_t', ('retval', 'info', 'runtime', 'retstr')) @@ -104,6 +120,19 @@ def task_has_custom(task, attr): monkey_patched=['celery.app.task']) +def get_log_policy(task, einfo, exc): + if isinstance(exc, Reject): + return log_policy_reject + elif isinstance(exc, Ignore): + return log_policy_ignore + elif einfo.internal: + return log_policy_internal + else: + if task.throws and isinstance(exc, task.throws): + return log_policy_expected + return log_policy_unexpected + + class TraceInfo(object): __slots__ = ('state', 'retval') @@ -172,39 +201,14 @@ def handle_failure(self, task, store_errors=True): def _log_error(self, task, einfo): req = task.request eobj = einfo.exception = get_pickled_exception(einfo.exception) - exception, traceback, exc_info, internal, sargs, skwargs = ( + exception, traceback, exc_info, sargs, skwargs = ( safe_repr(eobj), safe_str(einfo.traceback), einfo.exc_info, - einfo.internal, safe_repr(req.args), safe_repr(req.kwargs), ) - if task.throws and isinstance(eobj, task.throws): - do_send_mail, severity, exc_info, description = ( - False, logging.INFO, None, 'raised expected', - ) - else: - do_send_mail, severity, description = ( - True, logging.ERROR, 'raised unexpected', - ) - format = LOG_FAILURE - - if internal: - if isinstance(einfo.exception, Reject): - format = LOG_REJECTED - description = 'rejected' - severity = logging.WARN - exc_info = einfo - elif isinstance(einfo.exception, Ignore): - format = LOG_IGNORED - description = 'ignored' - severity = logging.INFO - exc_info = None - else: - format = LOG_INTERNAL_ERROR - description = 'INTERNAL ERROR' - severity = logging.CRITICAL + policy = get_log_policy(task, einfo, eobj) context = { 'hostname': req.hostname, @@ -214,15 +218,16 @@ def _log_error(self, task, einfo): 'traceback': traceback, 'args': sargs, 'kwargs': skwargs, - 'description': description, - 'internal': internal, + 'description': policy.description, + 'internal': einfo.internal, } - logger.log(severity, format.strip(), context, - exc_info=exc_info, + logger.log(policy.severity, policy.format.strip(), context, + exc_info=exc_info if policy.traceback else None, extra={'data': context}) - task.send_error_email(context, einfo.exception) + if policy.mail: + task.send_error_email(context, einfo.exception) def build_tracer(name, task, loader=None, hostname=None, store_errors=True, @@ -444,14 +449,21 @@ def trace_task(task, uuid, args, kwargs, request={}, **opts): try: if task.__trace__ is None: task.__trace__ = build_tracer(task.name, task, **opts) - return task.__trace__(uuid, args, kwargs, request)[0] + return task.__trace__(uuid, args, kwargs, request) except Exception as exc: return report_internal_error(task, exc) -def _trace_task_ret(name, uuid, args, kwargs, request={}, app=None, **opts): - return trace_task((app or current_app).tasks[name], - uuid, args, kwargs, request, app=app, **opts) +def _trace_task_ret(name, uuid, request, body, content_type, + content_encoding, loads=loads_message, app=None, + **extra_request): + app = app or current_app._get_current_object() + accept = prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT) + args, kwargs = loads(body, content_type, content_encoding, accept=accept) + request.update(args=args, kwargs=kwargs, **extra_request) + R, I, T, Rstr = trace_task(app.tasks[name], + uuid, args, kwargs, request, app=app) + return (1, R, T) if I else (0, Rstr, T) trace_task_ret = _trace_task_ret @@ -460,18 +472,23 @@ def _fast_trace_task_v1(task, uuid, args, kwargs, request={}): # so this is the function used in the worker. R, I, T, Rstr = _tasks[task].__trace__(uuid, args, kwargs, request)[0] # exception instance if error, else result text - return (R if I else Rstr), T + return (1, R, T) if I else (0, Rstr, T) def _fast_trace_task(task, uuid, request, body, content_type, - content_encoding, decode_message=decode_message, + content_encoding, loads=loads_message, _loc=_localized, **extra_request): - args, kwargs = decode_message(body, content_type, content_encoding) + tasks, accept = _loc + try: + args, kwargs = loads(body, content_type, content_encoding, + accept=accept) + except Exception as exc: + print('OH NOEEES: %r' % (exc, )) request.update(args=args, kwargs=kwargs, **extra_request) - R, I, T, Rstr = _tasks[task].__trace__( + R, I, T, Rstr = tasks[task].__trace__( uuid, args, kwargs, request, ) - return (R if I else Rstr), T + return (1, R, T) if I else (0, Rstr, T) def report_internal_error(task, exc): @@ -488,7 +505,6 @@ def report_internal_error(task, exc): def setup_worker_optimizations(app): - global _tasks global trace_task_ret # make sure custom Task.__call__ methods that calls super @@ -508,7 +524,10 @@ def setup_worker_optimizations(app): app.finalize() # set fast shortcut to task registry - _tasks = app._tasks + _localized[:] = [ + app._tasks, + prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT), + ] trace_task_ret = _fast_trace_task from celery.worker import job as job_module diff --git a/celery/events/state.py b/celery/events/state.py index 541f72226..2a11891f7 100644 --- a/celery/events/state.py +++ b/celery/events/state.py @@ -30,7 +30,7 @@ from weakref import ref from kombu.clocks import timetuple -from kombu.utils import cached_property, kwdict +from kombu.utils import cached_property from celery import states from celery.five import class_property, items, values @@ -54,8 +54,6 @@ %s seconds. [orig: %s recv: %s] """ -CAN_KWDICT = sys.version_info >= (2, 6, 5) - logger = get_logger(__name__) warn = logger.warning @@ -86,7 +84,7 @@ def heartbeat_expires(timestamp, freq=60, def _depickle_task(cls, fields): - return cls(**(fields if CAN_KWDICT else kwdict(fields))) + return cls(**fields) def with_unique_field(attr): diff --git a/celery/tests/case.py b/celery/tests/case.py index 808347817..520e1f55b 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -48,7 +48,7 @@ from celery.utils.imports import qualname __all__ = [ - 'Case', 'AppCase', 'Mock', 'MagicMock', 'ANY', + 'Case', 'AppCase', 'Mock', 'MagicMock', 'ANY', 'TaskMessage', 'patch', 'call', 'sentinel', 'skip_unless_module', 'wrap_logger', 'with_environ', 'sleepdeprived', 'skip_if_environ', 'todo', 'skip', 'skip_if', @@ -56,7 +56,7 @@ 'replace_module_value', 'sys_platform', 'reset_modules', 'patch_modules', 'mock_context', 'mock_open', 'patch_many', 'assert_signal_called', 'skip_if_pypy', - 'skip_if_jython', 'body_from_sig', 'restore_logging', + 'skip_if_jython', 'task_message_from_sig', 'restore_logging', ] patch = mock.patch call = mock.call @@ -819,7 +819,7 @@ def _inner(*args, **kwargs): return _inner -def body_from_sig(app, sig, utc=True): +def task_message_from_sig(app, sig, utc=True): sig.freeze() callbacks = sig.options.pop('link', None) errbacks = sig.options.pop('link_error', None) @@ -835,17 +835,14 @@ def body_from_sig(app, sig, utc=True): expires = app.now() + timedelta(seconds=expires) if expires and isinstance(expires, datetime): expires = expires.isoformat() - return { - 'task': sig.task, - 'id': sig.id, - 'args': sig.args, - 'kwargs': sig.kwargs, - 'callbacks': [dict(s) for s in callbacks] if callbacks else None, - 'errbacks': [dict(s) for s in errbacks] if errbacks else None, - 'eta': eta, - 'utc': utc, - 'expires': expires, - } + return TaskMessage( + sig.task, id=sig.id, args=sig.args, + kwargs=sig.kwargs, + callbacks=[dict(s) for s in callbacks] if callbacks else None, + errbacks=[dict(s) for s in errbacks] if errbacks else None, + eta=eta, + expires=expires, + ) @contextmanager @@ -861,3 +858,20 @@ def restore_logging(): sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__ = outs root.level = level root.handlers[:] = handlers + + +def TaskMessage(name, id=None, args=(), kwargs={}, **options): + from celery import uuid + from kombu.serialization import dumps + id = id or uuid() + message = Mock(name='TaskMessage-{0}'.format(id)) + message.headers = { + 'id': id, + 'c_type': name, + } + message.headers.update(options) + message.content_type, message.content_encoding, message.body = dumps( + (args, kwargs), serializer='json', + ) + message.payload = (args, kwargs) + return message diff --git a/celery/tests/tasks/test_trace.py b/celery/tests/tasks/test_trace.py index 0a7ec3345..3d55d9041 100644 --- a/celery/tests/tasks/test_trace.py +++ b/celery/tests/tasks/test_trace.py @@ -14,10 +14,11 @@ from celery.tests.case import AppCase, Mock, patch -def trace(app, task, args=(), kwargs={}, propagate=False, **opts): +def trace(app, task, args=(), kwargs={}, + propagate=False, eager=True, request=None, **opts): t = build_tracer(task.name, task, - eager=True, propagate=propagate, app=app, **opts) - ret = t('id-1', args, kwargs, None) + eager=eager, propagate=propagate, app=app, **opts) + ret = t('id-1', args, kwargs, request) return ret.retval, ret.info diff --git a/celery/tests/worker/test_control.py b/celery/tests/worker/test_control.py index bb7df0daf..fc587f0d1 100644 --- a/celery/tests/worker/test_control.py +++ b/celery/tests/worker/test_control.py @@ -21,7 +21,7 @@ from celery.worker.control import Panel from celery.worker.pidbox import Pidbox, gPidbox -from celery.tests.case import AppCase, Mock, call, patch +from celery.tests.case import AppCase, Mock, TaskMessage, call, patch hostname = socket.gethostname() @@ -250,12 +250,7 @@ def test_report(self): self.panel.handle('report') def test_active(self): - r = Request({ - 'task': self.mytask.name, - 'id': 'do re mi', - 'args': (), - 'kwargs': {}, - }, app=self.app) + r = Request(TaskMessage(self.mytask.name, 'do re mi'), app=self.app) worker_state.active_requests.add(r) try: self.assertTrue(self.panel.handle('dump_active')) @@ -347,12 +342,7 @@ def test_dump_schedule(self): consumer = Consumer(self.app) panel = self.create_panel(consumer=consumer) self.assertFalse(panel.handle('dump_schedule')) - r = Request({ - 'task': self.mytask.name, - 'id': 'CAFEBABE', - 'args': (), - 'kwargs': {}, - }, app=self.app) + r = Request(TaskMessage(self.mytask.name, 'CAFEBABE'), app=self.app) consumer.timer.schedule.enter_at( consumer.timer.Entry(lambda x: x, (r, )), datetime.now() + timedelta(seconds=10)) @@ -363,19 +353,14 @@ def test_dump_schedule(self): def test_dump_reserved(self): consumer = Consumer(self.app) - worker_state.reserved_requests.add(Request({ - 'task': self.mytask.name, - 'id': uuid(), - 'args': (2, 2), - 'kwargs': {}, - }, app=self.app)) + worker_state.reserved_requests.add( + Request(TaskMessage(self.mytask.name, args=(2, 2)), app=self.app), + ) try: panel = self.create_panel(consumer=consumer) response = panel.handle('dump_reserved', {'safe': True}) self.assertDictContainsSubset( {'name': self.mytask.name, - 'args': (2, 2), - 'kwargs': {}, 'hostname': socket.gethostname()}, response[0], ) diff --git a/celery/tests/worker/test_loops.py b/celery/tests/worker/test_loops.py index 00c5d960f..80edd393a 100644 --- a/celery/tests/worker/test_loops.py +++ b/celery/tests/worker/test_loops.py @@ -11,7 +11,7 @@ from celery.worker.consumer import Consumer from celery.worker.loops import asynloop, synloop -from celery.tests.case import AppCase, Mock, body_from_sig +from celery.tests.case import AppCase, Mock, task_message_from_sig class X(object): @@ -107,7 +107,7 @@ def get_task_callback(*args, **kwargs): x = X(*args, **kwargs) x.blueprint.state = CLOSE asynloop(*x.args) - return x, x.consumer.callbacks[0] + return x, x.consumer.on_message class test_asynloop(AppCase): @@ -132,45 +132,44 @@ def test_setup_heartbeat(self): def task_context(self, sig, **kwargs): x, on_task = get_task_callback(self.app, **kwargs) - body = body_from_sig(self.app, sig) - message = Mock() - strategy = x.obj.strategies[sig.task] = Mock() - return x, on_task, body, message, strategy + message = task_message_from_sig(self.app, sig) + strategy = x.obj.strategies[sig.task] = Mock(name='strategy') + return x, on_task, message, strategy def test_on_task_received(self): - _, on_task, body, msg, strategy = self.task_context(self.add.s(2, 2)) - on_task(body, msg) + _, on_task, msg, strategy = self.task_context(self.add.s(2, 2)) + on_task(msg) strategy.assert_called_with( - msg, body, msg.ack_log_error, msg.reject_log_error, [], + msg, None, msg.ack_log_error, msg.reject_log_error, [], ) def test_on_task_received_executes_on_task_message(self): cbs = [Mock(), Mock(), Mock()] - _, on_task, body, msg, strategy = self.task_context( + _, on_task, msg, strategy = self.task_context( self.add.s(2, 2), on_task_message=cbs, ) - on_task(body, msg) + on_task(msg) strategy.assert_called_with( - msg, body, msg.ack_log_error, msg.reject_log_error, cbs, + msg, None, msg.ack_log_error, msg.reject_log_error, cbs, ) def test_on_task_message_missing_name(self): - x, on_task, body, msg, strategy = self.task_context(self.add.s(2, 2)) - body.pop('task') - on_task(body, msg) - x.on_unknown_message.assert_called_with(body, msg) + x, on_task, msg, strategy = self.task_context(self.add.s(2, 2)) + msg.headers.pop('c_type') + on_task(msg) + x.on_unknown_message.assert_called_with(((2, 2), {}), msg) def test_on_task_not_registered(self): - x, on_task, body, msg, strategy = self.task_context(self.add.s(2, 2)) + x, on_task, msg, strategy = self.task_context(self.add.s(2, 2)) exc = strategy.side_effect = KeyError(self.add.name) - on_task(body, msg) - x.on_unknown_task.assert_called_with(body, msg, exc) + on_task(msg) + x.on_unknown_task.assert_called_with(None, msg, exc) def test_on_task_InvalidTaskError(self): - x, on_task, body, msg, strategy = self.task_context(self.add.s(2, 2)) + x, on_task, msg, strategy = self.task_context(self.add.s(2, 2)) exc = strategy.side_effect = InvalidTaskError() - on_task(body, msg) - x.on_invalid_task.assert_called_with(body, msg, exc) + on_task(msg) + x.on_invalid_task.assert_called_with(None, msg, exc) def test_should_terminate(self): x = X(self.app) diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index 488ea72f4..280152475 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, unicode_literals -import anyjson +import numbers import os import signal import socket @@ -10,7 +10,6 @@ from datetime import datetime, timedelta from billiard.einfo import ExceptionInfo -from kombu.transport.base import Message from kombu.utils.encoding import from_utf8, default_encode from celery import states @@ -27,12 +26,13 @@ from celery.exceptions import ( Ignore, InvalidTaskError, + Reject, Retry, TaskRevokedError, Terminated, WorkerLostError, ) -from celery.five import keys, monotonic +from celery.five import monotonic from celery.signals import task_revoked from celery.utils import uuid from celery.worker import job as module @@ -44,8 +44,9 @@ Case, Mock, SkipTest, + TaskMessage, assert_signal_called, - body_from_sig, + task_message_from_sig, patch, ) @@ -85,7 +86,7 @@ def jail(app, task_id, name, args, kwargs): task.__trace__ = None # rebuild return trace_task( task, task_id, args, kwargs, request=request, eager=False, app=app, - ) + ).retval class test_default_encode(AppCase): @@ -138,7 +139,7 @@ def mytask_raising(i): raise KeyError(i) self.mytask_raising = mytask_raising - @patch('celery.app.trace._logger') + @patch('celery.app.trace.logger') def test_process_cleanup_fails(self, _logger): self.mytask.backend = Mock() self.mytask.backend.process_cleanup = Mock(side_effect=KeyError()) @@ -227,9 +228,10 @@ def mytask_raising(i): def get_request(self, sig, Request=Request, **kwargs): return Request( - body_from_sig(self.app, sig), - on_ack=Mock(), - eventer=Mock(), + task_message_from_sig(self.app, sig), + on_ack=Mock(name='on_ack'), + on_reject=Mock(name='on_reject'), + eventer=Mock(name='eventer'), app=self.app, connection_errors=(socket.error, ), task=sig.type, @@ -273,7 +275,7 @@ def test_on_failure_Termianted(self): uuid=req.id, terminated=True, signum='9', expired=False, ) - def test_log_error_propagates_MemoryError(self): + def test_on_failure_propagates_MemoryError(self): einfo = None try: raise MemoryError() @@ -282,9 +284,9 @@ def test_log_error_propagates_MemoryError(self): self.assertIsNotNone(einfo) req = self.get_request(self.add.s(2, 2)) with self.assertRaises(MemoryError): - req._log_error(einfo) + req.on_failure(einfo) - def test_log_error_when_Ignore(self): + def test_on_failure_Ignore_acknowledges(self): einfo = None try: raise Ignore() @@ -292,48 +294,55 @@ def test_log_error_when_Ignore(self): einfo = ExceptionInfo(internal=True) self.assertIsNotNone(einfo) req = self.get_request(self.add.s(2, 2)) - req._log_error(einfo) + req.on_failure(einfo) req.on_ack.assert_called_with(req_logger, req.connection_errors) + def test_on_failure_Reject_rejects(self): + einfo = None + try: + raise Reject() + except Reject: + einfo = ExceptionInfo(internal=True) + self.assertIsNotNone(einfo) + req = self.get_request(self.add.s(2, 2)) + req.on_failure(einfo) + req.on_reject.assert_called_with( + req_logger, req.connection_errors, False, + ) + + def test_on_failure_Reject_rejects_with_requeue(self): + einfo = None + try: + raise Reject(requeue=True) + except Reject: + einfo = ExceptionInfo(internal=True) + self.assertIsNotNone(einfo) + req = self.get_request(self.add.s(2, 2)) + req.on_failure(einfo) + req.on_reject.assert_called_with( + req_logger, req.connection_errors, True, + ) + def test_tzlocal_is_cached(self): req = self.get_request(self.add.s(2, 2)) req._tzlocal = 'foo' self.assertEqual(req.tzlocal, 'foo') - def test_execute_magic_kwargs(self): - task = self.add.s(2, 2) - task.freeze() - req = self.get_request(task) - self.add.accept_magic_kwargs = True - pool = Mock() - req.execute_using_pool(pool) - self.assertTrue(pool.apply_async.called) - args = pool.apply_async.call_args[1]['args'] - self.assertEqual(args[0], task.task) - self.assertEqual(args[1], task.id) - self.assertEqual(args[2], task.args) - kwargs = args[3] - self.assertEqual(kwargs.get('task_name'), task.task) - - def xRequest(self, body=None, **kwargs): - body = dict({'task': self.mytask.name, - 'id': uuid(), - 'args': [1], - 'kwargs': {'f': 'x'}}, **body or {}) - return Request(body, app=self.app, **kwargs) + def xRequest(self, name=None, id=None, args=None, kwargs=None, + on_ack=None, on_reject=None, **head): + args = [1] if args is None else args + kwargs = {'f': 'x'} if kwargs is None else kwargs + on_ack = on_ack or Mock(name='on_ack') + on_reject = on_reject or Mock(name='on_reject') + message = TaskMessage( + name or self.mytask.name, id, args=args, kwargs=kwargs, **head + ) + return Request(message, app=self.app, + on_ack=on_ack, on_reject=on_reject) def test_task_wrapper_repr(self): self.assertTrue(repr(self.xRequest())) - @patch('celery.worker.job.kwdict') - def test_kwdict(self, kwdict): - prev, module.NEEDS_KWDICT = module.NEEDS_KWDICT, True - try: - self.xRequest() - self.assertTrue(kwdict.called) - finally: - module.NEEDS_KWDICT = prev - def test_sets_store_errors(self): self.mytask.ignore_result = True job = self.xRequest() @@ -350,12 +359,7 @@ def test_send_event(self): self.assertIn('task-frobulated', job.eventer.sent) def test_on_retry(self): - job = Request({ - 'task': self.mytask.name, - 'id': uuid(), - 'args': [1], - 'kwargs': {'f': 'x'}, - }, app=self.app) + job = self.get_request(self.mytask.s(1, f='x')) job.eventer = MockEventDispatcher() try: raise Retry('foo', KeyError('moofoobar')) @@ -372,12 +376,7 @@ def test_on_retry(self): job.on_failure(einfo) def test_compat_properties(self): - job = Request({ - 'task': self.mytask.name, - 'id': uuid(), - 'args': [1], - 'kwargs': {'f': 'x'}, - }, app=self.app) + job = self.xRequest() self.assertEqual(job.task_id, job.id) self.assertEqual(job.task_name, job.name) job.task_id = 'ID' @@ -388,12 +387,7 @@ def test_compat_properties(self): def test_terminate__task_started(self): pool = Mock() signum = signal.SIGTERM - job = Request({ - 'task': self.mytask.name, - 'id': uuid(), - 'args': [1], - 'kwrgs': {'f': 'x'}, - }, app=self.app) + job = self.get_request(self.mytask.s(1, f='x')) with assert_signal_called( task_revoked, sender=job.task, request=job, terminated=True, expired=False, signum=signum): @@ -404,12 +398,7 @@ def test_terminate__task_started(self): def test_terminate__task_reserved(self): pool = Mock() - job = Request({ - 'task': self.mytask.name, - 'id': uuid(), - 'args': [1], - 'kwargs': {'f': 'x'}, - }, app=self.app) + job = self.get_request(self.mytask.s(1, f='x')) job.time_start = None job.terminate(pool, signal='TERM') self.assertFalse(pool.terminate_job.called) @@ -417,13 +406,9 @@ def test_terminate__task_reserved(self): job.terminate(pool, signal='TERM') def test_revoked_expires_expired(self): - job = Request({ - 'task': self.mytask.name, - 'id': uuid(), - 'args': [1], - 'kwargs': {'f': 'x'}, - 'expires': datetime.utcnow() - timedelta(days=1), - }, app=self.app) + job = self.get_request(self.mytask.s(1, f='x').set( + expires=datetime.utcnow() - timedelta(days=1) + )) with assert_signal_called( task_revoked, sender=job.task, request=job, terminated=False, expired=True, signum=None): @@ -435,9 +420,9 @@ def test_revoked_expires_expired(self): ) def test_revoked_expires_not_expired(self): - job = self.xRequest({ - 'expires': datetime.utcnow() + timedelta(days=1), - }) + job = self.xRequest( + expires=datetime.utcnow() + timedelta(days=1), + ) job.revoked() self.assertNotIn(job.id, revoked) self.assertNotEqual( @@ -447,47 +432,15 @@ def test_revoked_expires_not_expired(self): def test_revoked_expires_ignore_result(self): self.mytask.ignore_result = True - job = self.xRequest({ - 'expires': datetime.utcnow() - timedelta(days=1), - }) + job = self.xRequest( + expires=datetime.utcnow() - timedelta(days=1), + ) job.revoked() self.assertIn(job.id, revoked) self.assertNotEqual( self.mytask.backend.get_status(job.id), states.REVOKED, ) - def test_send_email(self): - app = self.app - mail_sent = [False] - - def mock_mail_admins(*args, **kwargs): - mail_sent[0] = True - - def get_ei(): - try: - raise KeyError('moofoobar') - except: - return ExceptionInfo() - - app.mail_admins = mock_mail_admins - self.mytask.send_error_emails = True - job = self.xRequest() - einfo = get_ei() - job.on_failure(einfo) - self.assertTrue(mail_sent[0]) - - einfo = get_ei() - mail_sent[0] = False - self.mytask.send_error_emails = False - job.on_failure(einfo) - self.assertFalse(mail_sent[0]) - - einfo = get_ei() - mail_sent[0] = False - self.mytask.send_error_emails = True - job.on_failure(einfo) - self.assertTrue(mail_sent[0]) - def test_already_revoked(self): job = self.xRequest() job._already_revoked = True @@ -510,10 +463,10 @@ def test_execute_does_not_execute_revoked(self): def test_execute_acks_late(self): self.mytask_raising.acks_late = True - job = self.xRequest({ - 'task': self.mytask_raising.name, - 'kwargs': {}, - }) + job = self.xRequest( + name=self.mytask_raising.name, + kwargs={}, + ) job.execute() self.assertTrue(job.acknowledged) job.execute() @@ -555,10 +508,10 @@ def test_on_accepted_terminates(self): def test_on_success_acks_early(self): job = self.xRequest() job.time_start = 1 - job.on_success(42) + job.on_success((0, 42, 0.001)) prev, module._does_info = module._does_info, False try: - job.on_success(42) + job.on_success((0, 42, 0.001)) self.assertFalse(job.acknowledged) finally: module._does_info = prev @@ -570,7 +523,7 @@ def test_on_success_BaseException(self): try: raise SystemExit() except SystemExit: - job.on_success(ExceptionInfo()) + job.on_success((1, ExceptionInfo(), 0.01)) else: assert False @@ -579,7 +532,7 @@ def test_on_success_eventer(self): job.time_start = 1 job.eventer = Mock() job.eventer.send = Mock() - job.on_success(42) + job.on_success((0, 42, 0.001)) self.assertTrue(job.eventer.send.called) def test_on_success_when_failure(self): @@ -589,14 +542,14 @@ def test_on_success_when_failure(self): try: raise KeyError('foo') except Exception: - job.on_success(ExceptionInfo()) + job.on_success((1, ExceptionInfo(), 0.001)) self.assertTrue(job.on_failure.called) def test_on_success_acks_late(self): job = self.xRequest() job.time_start = 1 self.mytask.acks_late = True - job.on_success(42) + job.on_success((0, 42, 0.001)) self.assertTrue(job.acknowledged) def test_on_failure_WorkerLostError(self): @@ -634,9 +587,10 @@ def test_on_failure_acks_late(self): self.assertTrue(job.acknowledged) def test_from_message_invalid_kwargs(self): - body = dict(task=self.mytask.name, id=1, args=(), kwargs='foo') + m = TaskMessage(self.mytask.name, args=(), kwargs='foo') + req = Request(m, app=self.app) with self.assertRaises(InvalidTaskError): - Request(body, message=None, app=self.app) + raise req.execute().exception @patch('celery.worker.job.error') @patch('celery.worker.job.warn') @@ -662,37 +616,60 @@ def test_fast_trace_task(self): from celery.app import trace setup_worker_optimizations(self.app) self.assertIs(trace.trace_task_ret, trace._fast_trace_task) + tid = uuid() + message = TaskMessage(self.mytask.name, tid, args=[4]) try: self.mytask.__trace__ = build_tracer( self.mytask.name, self.mytask, self.app.loader, 'test', app=self.app, ) - res = trace.trace_task_ret(self.mytask.name, uuid(), [4], {}) - self.assertEqual(res, 4 ** 4) + failed, res, runtime = trace.trace_task_ret( + self.mytask.name, tid, message.headers, message.body, + message.content_type, message.content_encoding) + self.assertFalse(failed) + self.assertEqual(res, repr(4 ** 4)) + self.assertTrue(runtime) + self.assertIsInstance(runtime, numbers.Real) finally: reset_worker_optimizations() self.assertIs(trace.trace_task_ret, trace._trace_task_ret) delattr(self.mytask, '__trace__') - res = trace.trace_task_ret( - self.mytask.name, uuid(), [4], {}, app=self.app, + failed, res, runtime = trace.trace_task_ret( + self.mytask.name, tid, message.headers, message.body, + message.content_type, message.content_encoding, app=self.app, ) - self.assertEqual(res, 4 ** 4) + self.assertFalse(failed) + self.assertEqual(res, repr(4 ** 4)) + self.assertTrue(runtime) + self.assertIsInstance(runtime, numbers.Real) def test_trace_task_ret(self): self.mytask.__trace__ = build_tracer( self.mytask.name, self.mytask, self.app.loader, 'test', app=self.app, ) - res = _trace_task_ret(self.mytask.name, uuid(), [4], {}, app=self.app) - self.assertEqual(res, 4 ** 4) + tid = uuid() + message = TaskMessage(self.mytask.name, tid, args=[4]) + _, R, _ = _trace_task_ret( + self.mytask.name, tid, message.headers, + message.body, message.content_type, + message.content_encoding, app=self.app, + ) + self.assertEqual(R, repr(4 ** 4)) def test_trace_task_ret__no_trace(self): try: delattr(self.mytask, '__trace__') except AttributeError: pass - res = _trace_task_ret(self.mytask.name, uuid(), [4], {}, app=self.app) - self.assertEqual(res, 4 ** 4) + tid = uuid() + message = TaskMessage(self.mytask.name, tid, args=[4]) + _, R, _ = _trace_task_ret( + self.mytask.name, tid, message.headers, + message.body, message.content_type, + message.content_encoding, app=self.app, + ) + self.assertEqual(R, repr(4 ** 4)) def test_trace_catches_exception(self): @@ -705,7 +682,7 @@ def raising(): with self.assertWarnsRegex(RuntimeWarning, r'Exception raised outside'): - res = trace_task(raising, uuid(), [], {}, app=self.app) + res = trace_task(raising, uuid(), [], {}, app=self.app)[0] self.assertIsInstance(res, ExceptionInfo) def test_worker_task_trace_handle_retry(self): @@ -749,71 +726,39 @@ def test_worker_task_trace_handle_failure(self): finally: self.mytask.pop_request() - def test_task_wrapper_mail_attrs(self): - job = self.xRequest({'args': [], 'kwargs': {}}) - x = job.success_msg % { - 'name': job.name, - 'id': job.id, - 'return_value': 10, - 'runtime': 0.3641, - } - self.assertTrue(x) - x = job.error_msg % { - 'name': job.name, - 'id': job.id, - 'exc': 'FOOBARBAZ', - 'description': 'raised unexpected', - 'traceback': 'foobarbaz', - } - self.assertTrue(x) - def test_from_message(self): us = 'æØåveéðƒeæ' - body = {'task': self.mytask.name, 'id': uuid(), - 'args': [2], 'kwargs': {us: 'bar'}} - m = Message(None, body=anyjson.dumps(body), backend='foo', - content_type='application/json', - content_encoding='utf-8') - job = Request(m.decode(), message=m, app=self.app) + tid = uuid() + m = TaskMessage(self.mytask.name, tid, args=[2], kwargs={us: 'bar'}) + job = Request(m, app=self.app) self.assertIsInstance(job, Request) - self.assertEqual(job.name, body['task']) - self.assertEqual(job.id, body['id']) - self.assertEqual(job.args, body['args']) - us = from_utf8(us) - if sys.version_info < (2, 6): - self.assertEqual(next(keys(job.kwargs)), us) - self.assertIsInstance(next(keys(job.kwargs)), str) + self.assertEqual(job.name, self.mytask.name) + self.assertEqual(job.id, tid) + self.assertIs(job.message, m) def test_from_message_empty_args(self): - body = {'task': self.mytask.name, 'id': uuid()} - m = Message(None, body=anyjson.dumps(body), backend='foo', - content_type='application/json', - content_encoding='utf-8') - job = Request(m.decode(), message=m, app=self.app) + tid = uuid() + m = TaskMessage(self.mytask.name, tid, args=[], kwargs={}) + job = Request(m, app=self.app) self.assertIsInstance(job, Request) - self.assertEqual(job.args, []) - self.assertEqual(job.kwargs, {}) def test_from_message_missing_required_fields(self): - body = {} - m = Message(None, body=anyjson.dumps(body), backend='foo', - content_type='application/json', - content_encoding='utf-8') + m = TaskMessage(self.mytask.name) + m.headers.clear() with self.assertRaises(KeyError): - Request(m.decode(), message=m, app=self.app) + Request(m, app=self.app) def test_from_message_nonexistant_task(self): - body = {'task': 'cu.mytask.doesnotexist', 'id': uuid(), - 'args': [2], 'kwargs': {'æØåveéðƒeæ': 'bar'}} - m = Message(None, body=anyjson.dumps(body), backend='foo', - content_type='application/json', - content_encoding='utf-8') + m = TaskMessage( + 'cu.mytask.doesnotexist', + args=[2], kwargs={'æØåveéðƒeæ': 'bar'}, + ) with self.assertRaises(KeyError): - Request(m.decode(), message=m, app=self.app) + Request(m, app=self.app) def test_execute(self): tid = uuid() - job = self.xRequest({'id': tid, 'args': [4], 'kwargs': {}}) + job = self.xRequest(id=tid, args=[4], kwargs={}) self.assertEqual(job.execute(), 256) meta = self.mytask.backend.get_task_meta(tid) self.assertEqual(meta['status'], states.SUCCESS) @@ -826,38 +771,17 @@ def mytask_no_kwargs(i): return i ** i tid = uuid() - job = self.xRequest({ - 'task': mytask_no_kwargs.name, - 'id': tid, - 'args': [4], - 'kwargs': {}, - }) + job = self.xRequest( + name=mytask_no_kwargs.name, + id=tid, + args=[4], + kwargs={}, + ) self.assertEqual(job.execute(), 256) meta = mytask_no_kwargs.backend.get_task_meta(tid) self.assertEqual(meta['result'], 256) self.assertEqual(meta['status'], states.SUCCESS) - def test_execute_success_some_kwargs(self): - scratch = {'task_id': None} - - @self.app.task(shared=False, accept_magic_kwargs=True) - def mytask_some_kwargs(i, task_id): - scratch['task_id'] = task_id - return i ** i - - tid = uuid() - job = self.xRequest({ - 'task': mytask_some_kwargs.name, - 'id': tid, - 'args': [4], - 'kwargs': {}, - }) - self.assertEqual(job.execute(), 256) - meta = mytask_some_kwargs.backend.get_task_meta(tid) - self.assertEqual(scratch.get('task_id'), tid) - self.assertEqual(meta['result'], 256) - self.assertEqual(meta['status'], states.SUCCESS) - def test_execute_ack(self): scratch = {'ACK': False} @@ -865,7 +789,7 @@ def on_ack(*args, **kwargs): scratch['ACK'] = True tid = uuid() - job = self.xRequest({'id': tid, 'args': [4]}, on_ack=on_ack) + job = self.xRequest(id=tid, args=[4], on_ack=on_ack) self.assertEqual(job.execute(), 256) meta = self.mytask.backend.get_task_meta(tid) self.assertTrue(scratch['ACK']) @@ -874,12 +798,13 @@ def on_ack(*args, **kwargs): def test_execute_fail(self): tid = uuid() - job = self.xRequest({ - 'task': self.mytask_raising.name, - 'id': tid, - 'args': [4], - 'kwargs': {}, - }) + job = self.xRequest( + name=self.mytask_raising.name, + id=tid, + args=[4], + kwargs={}, + ) + print(job.execute()) self.assertIsInstance(job.execute(), ExceptionInfo) meta = self.mytask_raising.backend.get_task_meta(tid) self.assertEqual(meta['status'], states.FAILURE) @@ -887,7 +812,7 @@ def test_execute_fail(self): def test_execute_using_pool(self): tid = uuid() - job = self.xRequest({'id': tid, 'args': [4]}) + job = self.xRequest(id=tid, args=[4]) class MockPool(BasePool): target = None @@ -908,48 +833,21 @@ def apply_async(self, target, args=None, kwargs=None, self.assertTrue(p.target) self.assertEqual(p.args[0], self.mytask.name) self.assertEqual(p.args[1], tid) - self.assertEqual(p.args[2], [4]) - self.assertIn('f', p.args[3]) - self.assertIn([4], p.args) + self.assertEqual(p.args[3], job.message.body) job.task.accept_magic_kwargs = False job.execute_using_pool(p) - def test_default_kwargs(self): - self.maxDiff = 3000 - tid = uuid() - job = self.xRequest({'id': tid, 'args': [4]}) - self.assertDictEqual( - job.extend_with_default_kwargs(), { - 'f': 'x', - 'logfile': None, - 'loglevel': None, - 'task_id': job.id, - 'task_retries': 0, - 'task_is_eager': False, - 'delivery_info': { - 'exchange': None, - 'routing_key': None, - 'priority': 0, - 'redelivered': False, - }, - 'task_name': job.name}) - - @patch('celery.worker.job.logger') - def _test_on_failure(self, exception, logger): - app = self.app + def _test_on_failure(self, exception): tid = uuid() - job = self.xRequest({'id': tid, 'args': [4]}) + job = self.xRequest(id=tid, args=[4]) + job.send_event = Mock(name='send_event') try: raise exception except Exception: exc_info = ExceptionInfo() - app.conf.CELERY_SEND_TASK_ERROR_EMAILS = True job.on_failure(exc_info) - self.assertTrue(logger.log.called) - context = logger.log.call_args[0][2] - self.assertEqual(self.mytask.name, context['name']) - self.assertIn(tid, context['id']) + self.assertTrue(job.send_event.called) def test_on_failure(self): self._test_on_failure(Exception('Inside unit tests')) diff --git a/celery/tests/worker/test_strategy.py b/celery/tests/worker/test_strategy.py index 7edf78bff..87ae65006 100644 --- a/celery/tests/worker/test_strategy.py +++ b/celery/tests/worker/test_strategy.py @@ -8,7 +8,7 @@ from celery.worker import state from celery.utils.timeutils import rate -from celery.tests.case import AppCase, Mock, patch, body_from_sig +from celery.tests.case import AppCase, Mock, patch, task_message_from_sig class test_default_strategy(AppCase): @@ -22,17 +22,16 @@ def add(x, y): class Context(object): - def __init__(self, sig, s, reserved, consumer, message, body): + def __init__(self, sig, s, reserved, consumer, message): self.sig = sig self.s = s self.reserved = reserved self.consumer = consumer self.message = message - self.body = body def __call__(self, **kwargs): return self.s( - self.message, self.body, + self.message, None, self.message.ack, self.message.reject, [], **kwargs ) @@ -76,10 +75,8 @@ def _context(self, sig, s = sig.type.start_strategy(self.app, consumer, task_reserved=reserved) self.assertTrue(s) - message = Mock() - body = body_from_sig(self.app, sig, utc=utc) - - yield self.Context(sig, s, reserved, consumer, message, body) + message = task_message_from_sig(self.app, sig, utc=utc) + yield self.Context(sig, s, reserved, consumer, message) def test_when_logging_disabled(self): with patch('celery.worker.strategy.logger') as logger: diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index b700a6ca6..e939a4e8b 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -17,7 +17,7 @@ from celery.concurrency.base import BasePool from celery.datastructures import AttributeDict from celery.exceptions import ( - WorkerShutdown, WorkerTerminate, TaskRevokedError, + WorkerShutdown, WorkerTerminate, TaskRevokedError, InvalidTaskError, ) from celery.five import Empty, range, Queue as FastQueue from celery.utils import uuid @@ -29,7 +29,9 @@ from celery.utils.serialization import pickle from celery.utils.timer2 import Timer -from celery.tests.case import AppCase, Mock, SkipTest, patch, restore_logging +from celery.tests.case import ( + AppCase, Mock, SkipTest, TaskMessage, patch, restore_logging, +) def MockStep(step=None): @@ -123,6 +125,13 @@ def create_message(channel, **data): return m +def create_task_message(channel, *args, **kwargs): + m = TaskMessage(*args, **kwargs) + m.channel = channel + m.delivery_info = {'consumer_tag': 'mock'} + return m + + class test_Consumer(AppCase): def setup(self): @@ -207,13 +216,13 @@ def test_receive_message_unknown(self, warn): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() - backend = Mock() - m = create_message(backend, unknown={'baz': '!!!'}) + channel = Mock() + m = create_message(channel, unknown={'baz': '!!!'}) l.event_dispatcher = mock_event_dispatcher() l.node = MockNode() callback = self._get_on_message(l) - callback(m.decode(), m) + callback(m) self.assertTrue(warn.call_count) @patch('celery.worker.strategy.to_timestamp') @@ -222,17 +231,18 @@ def test_receive_message_eta_OverflowError(self, to_timestamp): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() - m = create_message(Mock(), task=self.foo_task.name, - args=('2, 2'), - kwargs={}, - eta=datetime.now().isoformat()) + m = create_task_message( + Mock(), self.foo_task.name, + args=('2, 2'), kwargs={}, + eta=datetime.now().isoformat(), + ) l.event_dispatcher = mock_event_dispatcher() l.node = MockNode() l.update_strategies() l.qos = Mock() callback = self._get_on_message(l) - callback(m.decode(), m) + callback(m) self.assertTrue(m.acknowledged) @patch('celery.worker.consumer.error') @@ -241,13 +251,17 @@ def test_receive_message_InvalidTaskError(self, error): l.blueprint.state = RUN l.event_dispatcher = mock_event_dispatcher() l.steps.pop() - m = create_message(Mock(), task=self.foo_task.name, - args=(1, 2), kwargs='foobarbaz', id=1) + m = create_task_message( + Mock(), self.foo_task.name, + args=(1, 2), kwargs='foobarbaz', id=1) l.update_strategies() l.event_dispatcher = mock_event_dispatcher() + strat = l.strategies[self.foo_task.name] = Mock(name='strategy') + strat.side_effect = InvalidTaskError() callback = self._get_on_message(l) - callback(m.decode(), m) + callback(m) + self.assertTrue(error.called) self.assertIn('Received invalid task message', error.call_args[0][0]) @patch('celery.worker.consumer.crit') @@ -274,18 +288,20 @@ def _get_on_message(self, l): with self.assertRaises(WorkerShutdown): l.loop(*l.loop_args()) - self.assertTrue(l.task_consumer.register_callback.called) - return l.task_consumer.register_callback.call_args[0][0] + self.assertTrue(l.task_consumer.on_message) + return l.task_consumer.on_message def test_receieve_message(self): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.event_dispatcher = mock_event_dispatcher() - m = create_message(Mock(), task=self.foo_task.name, - args=[2, 4, 8], kwargs={}) + m = create_task_message( + Mock(), self.foo_task.name, + args=[2, 4, 8], kwargs={}, + ) l.update_strategies() callback = self._get_on_message(l) - callback(m.decode(), m) + callback(m) in_bucket = self.buffer.get_nowait() self.assertIsInstance(in_bucket, Request) @@ -419,8 +435,8 @@ def test_receieve_message_eta_isoformat(self): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() - m = create_message( - Mock(), task=self.foo_task.name, + m = create_task_message( + Mock(), self.foo_task.name, eta=(datetime.now() + timedelta(days=1)).isoformat(), args=[2, 4, 8], kwargs={}, ) @@ -432,7 +448,7 @@ def test_receieve_message_eta_isoformat(self): l.enabled = False l.update_strategies() callback = self._get_on_message(l) - callback(m.decode(), m) + callback(m) l.timer.stop() l.timer.join(1) @@ -469,27 +485,31 @@ def test_revoke(self): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() - backend = Mock() + channel = Mock() id = uuid() - t = create_message(backend, task=self.foo_task.name, args=[2, 4, 8], - kwargs={}, id=id) + t = create_task_message( + channel, self.foo_task.name, + args=[2, 4, 8], kwargs={}, id=id, + ) from celery.worker.state import revoked revoked.add(id) callback = self._get_on_message(l) - callback(t.decode(), t) + callback(t) self.assertTrue(self.buffer.empty()) def test_receieve_message_not_registered(self): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() - backend = Mock() - m = create_message(backend, task='x.X.31x', args=[2, 4, 8], kwargs={}) + channel = Mock(name='channel') + m = create_task_message( + channel, 'x.X.31x', args=[2, 4, 8], kwargs={}, + ) l.event_dispatcher = mock_event_dispatcher() callback = self._get_on_message(l) - self.assertFalse(callback(m.decode(), m)) + self.assertFalse(callback(m)) with self.assertRaises(Empty): self.buffer.get_nowait() self.assertTrue(self.timer.empty()) @@ -499,21 +519,25 @@ def test_receieve_message_not_registered(self): def test_receieve_message_ack_raises(self, logger, warn): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN - backend = Mock() - m = create_message(backend, args=[2, 4, 8], kwargs={}) + channel = Mock() + m = create_task_message( + channel, self.foo_task.name, + args=[2, 4, 8], kwargs={}, + ) + m.headers = None l.event_dispatcher = mock_event_dispatcher() + l.update_strategies() l.connection_errors = (socket.error, ) m.reject = Mock() m.reject.side_effect = socket.error('foo') callback = self._get_on_message(l) - self.assertFalse(callback(m.decode(), m)) + self.assertFalse(callback(m)) self.assertTrue(warn.call_count) with self.assertRaises(Empty): self.buffer.get_nowait() self.assertTrue(self.timer.empty()) - m.reject.assert_called_with(requeue=False) - self.assertTrue(logger.critical.call_count) + m.reject_log_error.assert_called_with(logger, l.connection_errors) def test_receive_message_eta(self): import sys @@ -529,10 +553,10 @@ def pp(*args, **kwargs): pp('-CREATE MYKOMBUCONSUMER') l.steps.pop() l.event_dispatcher = mock_event_dispatcher() - backend = Mock() + channel = Mock(name='channel') pp('+ CREATE MESSAGE') - m = create_message( - backend, task=self.foo_task.name, + m = create_task_message( + channel, self.foo_task.name, args=[2, 4, 8], kwargs={}, eta=(datetime.now() + timedelta(days=1)).isoformat(), ) @@ -556,7 +580,7 @@ def pp(*args, **kwargs): callback = self._get_on_message(l) pp('- GET ON MESSAGE') pp('+ CALLBACK') - callback(m.decode(), m) + callback(m) pp('- CALLBACK') finally: pp('+ STOP TIMER') @@ -925,10 +949,12 @@ def test_on_timer_tick(self): def test_process_task(self): worker = self.worker worker.pool = Mock() - backend = Mock() - m = create_message(backend, task=self.foo_task.name, args=[4, 8, 10], - kwargs={}) - task = Request(m.decode(), message=m, app=self.app) + channel = Mock() + m = create_task_message( + channel, self.foo_task.name, + args=[4, 8, 10], kwargs={}, + ) + task = Request(m, app=self.app) worker._process_task(task) self.assertEqual(worker.pool.apply_async.call_count, 1) worker.pool.stop() @@ -937,10 +963,12 @@ def test_process_task_raise_base(self): worker = self.worker worker.pool = Mock() worker.pool.apply_async.side_effect = KeyboardInterrupt('Ctrl+C') - backend = Mock() - m = create_message(backend, task=self.foo_task.name, args=[4, 8, 10], - kwargs={}) - task = Request(m.decode(), message=m, app=self.app) + channel = Mock() + m = create_task_message( + channel, self.foo_task.name, + args=[4, 8, 10], kwargs={}, + ) + task = Request(m, app=self.app) worker.steps = [] worker.blueprint.state = RUN with self.assertRaises(KeyboardInterrupt): @@ -950,10 +978,12 @@ def test_process_task_raise_WorkerTerminate(self): worker = self.worker worker.pool = Mock() worker.pool.apply_async.side_effect = WorkerTerminate() - backend = Mock() - m = create_message(backend, task=self.foo_task.name, args=[4, 8, 10], - kwargs={}) - task = Request(m.decode(), message=m, app=self.app) + channel = Mock() + m = create_task_message( + channel, self.foo_task.name, + args=[4, 8, 10], kwargs={}, + ) + task = Request(m, app=self.app) worker.steps = [] worker.blueprint.state = RUN with self.assertRaises(SystemExit): @@ -963,10 +993,12 @@ def test_process_task_raise_regular(self): worker = self.worker worker.pool = Mock() worker.pool.apply_async.side_effect = KeyError('some exception') - backend = Mock() - m = create_message(backend, task=self.foo_task.name, args=[4, 8, 10], - kwargs={}) - task = Request(m.decode(), message=m, app=self.app) + channel = Mock() + m = create_task_message( + channel, self.foo_task.name, + args=[4, 8, 10], kwargs={}, + ) + task = Request(m, app=self.app) worker._process_task(task) worker.pool.stop() diff --git a/celery/worker/autoscale.py b/celery/worker/autoscale.py index 14afc2e95..c8038b36d 100644 --- a/celery/worker/autoscale.py +++ b/celery/worker/autoscale.py @@ -81,7 +81,7 @@ def body(self): self.maybe_scale() sleep(1.0) - def _maybe_scale(self): + def _maybe_scale(self, req=None): procs = self.processes cur = min(self.qty, self.max_concurrency) if cur > procs: diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 33facabd1..3bb430c43 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -447,37 +447,30 @@ def create_task_handler(self): on_invalid_task = self.on_invalid_task callbacks = self.on_task_message - def on_v1_task_received(body, message): - try: - name = body['task'] - except (KeyError, TypeError): - return on_unknown_message(body, message) - - try: - strategies[name](message, body, - message.ack_log_error, - message.reject_log_error, - callbacks) - except KeyError as exc: - on_unknown_task(body, message, exc) - except InvalidTaskError as exc: - on_invalid_task(body, message, exc) - def on_task_received(message): - headers = message.headers + + # payload will only be set for v1 protocol, since v2 + # will defer deserializing the message body to the pool. + payload = None try: - type_ = headers['c_type'] + type_ = message.headers['c_type'] # protocol v2 + except TypeError: + return on_unknown_message(None, message) except KeyError: - return on_v1_task_received(message.payload, message) + payload = message.payload + try: + type_ = payload['task'] # protocol v1 + except (TypeError, KeyError): + return on_unknown_message(payload, message) try: strategies[type_]( message, None, message.ack_log_error, message.reject_log_error, callbacks, ) except KeyError as exc: - on_unknown_task(None, message, exc) + on_unknown_task(payload, message, exc) except InvalidTaskError as exc: - on_invalid_task(None, message, exc) + on_invalid_task(payload, message, exc) return on_task_received diff --git a/celery/worker/job.py b/celery/worker/job.py index 74278cc1e..0fd57f95a 100644 --- a/celery/worker/job.py +++ b/celery/worker/job.py @@ -13,7 +13,6 @@ import socket import sys -from billiard.einfo import ExceptionInfo from datetime import datetime from weakref import ref @@ -83,7 +82,7 @@ def __optimize__(): class RequestV1(object): if not IS_PYPY: __slots__ = ( - 'app', 'name', 'id', 'root_id', 'parent_id', + 'app', 'message', 'name', 'id', 'root_id', 'parent_id', 'on_ack', 'hostname', 'eventer', 'connection_errors', 'task', 'eta', 'expires', 'request_dict', 'acknowledged', 'on_reject', 'utc', 'time_start', 'worker_pid', '_already_revoked', @@ -94,9 +93,10 @@ class RequestV1(object): class Request(object): """A request for task execution.""" + utc = True if not IS_PYPY: # pragma: no cover __slots__ = ( - 'app', 'name', 'id', 'on_ack', 'payload', + 'app', 'name', 'id', 'on_ack', 'body', 'hostname', 'eventer', 'connection_errors', 'task', 'eta', 'expires', 'request_dict', 'acknowledged', 'on_reject', 'utc', 'time_start', 'worker_pid', 'timeouts', @@ -111,9 +111,10 @@ def __init__(self, message, on_ack=noop, task=None, on_reject=noop, **opts): headers = message.headers self.app = app + self.message = message name = self.name = headers['c_type'] - self.id = headers['task_id'] - self.payload = message.body + self.id = headers['id'] + self.body = message.body self.content_type = message.content_type self.content_encoding = message.content_encoding eta = headers.get('eta') @@ -185,14 +186,14 @@ def execute_using_pool(self, pool, **kwargs): if self.revoked(): raise TaskRevokedError(task_id) - payload = self.payload + body = self.body timeout, soft_timeout = self.timeouts timeout = timeout or task.time_limit soft_timeout = soft_timeout or task.soft_time_limit result = pool.apply_async( trace_task_ret, args=(self.name, task_id, self.request_dict, - bytes(payload) if isinstance(payload, buffer) else payload, + bytes(body) if isinstance(body, buffer) else body, self.content_type, self.content_encoding), kwargs={'hostname': self.hostname, 'is_eager': False}, accept_callback=self.on_accepted, @@ -221,14 +222,14 @@ def execute(self, loglevel=None, logfile=None): if not self.task.acks_late: self.acknowledge() - kwargs = self.kwargs request = self.request_dict + args, kwargs = self.message.payload request.update({'loglevel': loglevel, 'logfile': logfile, 'hostname': self.hostname, 'is_eager': False, - 'delivery_info': self.delivery_info}) - retval = trace_task(self.task, self.id, self.args, kwargs, request, + 'args': args, 'kwargs': kwargs}) + retval = trace_task(self.task, self.id, args, kwargs, request, hostname=self.hostname, loader=self.app.loader, - app=self.app) + app=self.app)[0] self.acknowledge() return retval @@ -313,22 +314,21 @@ def on_timeout(self, soft, timeout): if self.task.acks_late: self.acknowledge() - def on_success(self, ret_value, **kwargs): + def on_success(self, failed__retval__runtime, **kwargs): """Handler called if the task was successfully processed.""" - if isinstance(ret_value, ExceptionInfo): - if isinstance(ret_value.exception, ( - SystemExit, KeyboardInterrupt)): - raise ret_value.exception - return self.on_failure(ret_value) + failed, retval, runtime = failed__retval__runtime + if failed: + if isinstance(retval.exception, (SystemExit, KeyboardInterrupt)): + raise retval.exception + return self.on_failure(retval) task_ready(self) if self.task.acks_late: self.acknowledge() if self.eventer and self.eventer.enabled: - result, runtime = ret_value self.send_event( - 'task-succeeded', result=ret_value, runtime=runtime, + 'task-succeeded', result=retval, runtime=runtime, ) def on_retry(self, exc_info): @@ -340,38 +340,36 @@ def on_retry(self, exc_info): exception=safe_repr(exc_info.exception.exc), traceback=safe_str(exc_info.traceback)) - def on_failure(self, exc_info): + def on_failure(self, exc_info, send_failed_event=True): """Handler called if the task raised an exception.""" task_ready(self) - send_failed_event = True - - if exc_info.internal: - if isinstance(exc_info.exception, MemoryError): - raise MemoryError('Process got: %s' % (exc_info.exception, )) - elif isinstance(exc_info.exception, Reject): - self.reject(requeue=exc_info.exception.requeue) - elif isinstance(exc_info.exception, Ignore): - self.acknowledge() - else: - exc = exc_info.exception - - if isinstance(exc, Retry): - return self.on_retry(exc_info) - - # These are special cases where the process would not have had - # time to write the result. - if self.store_errors: - if isinstance(exc, WorkerLostError): - self.task.backend.mark_as_failure( - self.id, exc, request=self, - ) - elif isinstance(exc, Terminated): - self._announce_revoked( - 'terminated', True, string(exc), False) - send_failed_event = False # already sent revoked event - # (acks_late) acknowledge after result stored. - if self.task.acks_late: - self.acknowledge() + + if isinstance(exc_info.exception, MemoryError): + raise MemoryError('Process got: %s' % (exc_info.exception, )) + elif isinstance(exc_info.exception, Reject): + return self.reject(requeue=exc_info.exception.requeue) + elif isinstance(exc_info.exception, Ignore): + return self.acknowledge() + + exc = exc_info.exception + + if isinstance(exc, Retry): + return self.on_retry(exc_info) + + # These are special cases where the process would not have had + # time to write the result. + if self.store_errors: + if isinstance(exc, WorkerLostError): + self.task.backend.mark_as_failure( + self.id, exc, request=self, + ) + elif isinstance(exc, Terminated): + self._announce_revoked( + 'terminated', True, string(exc), False) + send_failed_event = False # already sent revoked event + # (acks_late) acknowledge after result stored. + if self.task.acks_late: + self.acknowledge() if send_failed_event: self.send_event( diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index 19a31ef90..4f12741ee 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -89,7 +89,7 @@ def task_message_handler(message, body, ack, reject, callbacks, return limit_task(req, bucket, 1) task_reserved(req) if callbacks: - [callback() for callback in callbacks] + [callback(req) for callback in callbacks] handle(req) return task_message_handler diff --git a/docs/internals/protov2.rst b/docs/internals/protov2.rst index f3c8359ca..8bbdcc4ee 100644 --- a/docs/internals/protov2.rst +++ b/docs/internals/protov2.rst @@ -102,8 +102,8 @@ Definition } headers = { 'lang': (string)'py' - 'c_type': (string)task, - 'task_id': (uuid)task_id, + 'task': (string)task, + 'id': (uuid)task_id, 'root_id': (uuid)root_id, 'parent_id': (uuid)parent_id, From dbb074858e3c312c1833a53490a25c313720e471 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 9 May 2014 17:49:56 +0100 Subject: [PATCH 0110/1103] Renames celery.worker.job -> celery.worker.request --- celery/app/trace.py | 10 +++++----- celery/contrib/batches.py | 2 +- celery/tests/worker/test_control.py | 2 +- celery/tests/worker/test_request.py | 10 +++++----- celery/tests/worker/test_worker.py | 2 +- celery/worker/control.py | 2 +- celery/worker/{job.py => request.py} | 4 ++-- celery/worker/state.py | 4 ++-- celery/worker/strategy.py | 2 +- docs/internals/app-overview.rst | 2 +- docs/internals/worker.rst | 2 +- ...celery.worker.job.rst => celery.worker.request.rst} | 6 +++--- docs/reference/index.rst | 2 +- docs/userguide/extending.rst | 2 +- docs/userguide/signals.rst | 2 +- docs/whatsnew-3.1.rst | 7 ++++--- 16 files changed, 31 insertions(+), 30 deletions(-) rename celery/worker/{job.py => request.py} (99%) rename docs/reference/{celery.worker.job.rst => celery.worker.request.rst} (57%) diff --git a/celery/app/trace.py b/celery/app/trace.py index 60776b9ff..c0523c3f2 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -530,9 +530,9 @@ def setup_worker_optimizations(app): ] trace_task_ret = _fast_trace_task - from celery.worker import job as job_module - job_module.trace_task_ret = _fast_trace_task - job_module.__optimize__() + from celery.worker import request as request_module + request_module.trace_task_ret = _fast_trace_task + request_module.__optimize__() def reset_worker_optimizations(): @@ -546,8 +546,8 @@ def reset_worker_optimizations(): BaseTask.__call__ = _patched.pop('BaseTask.__call__') except KeyError: pass - from celery.worker import job as job_module - job_module.trace_task_ret = _trace_task_ret + from celery.worker import request as request_module + request_module.trace_task_ret = _trace_task_ret def _install_stack_protection(): diff --git a/celery/contrib/batches.py b/celery/contrib/batches.py index 8cabc6f61..5bfa3a902 100644 --- a/celery/contrib/batches.py +++ b/celery/contrib/batches.py @@ -88,7 +88,7 @@ def wot_api_real(urls): from celery.task import Task from celery.five import Empty, Queue from celery.utils.log import get_logger -from celery.worker.job import Request +from celery.worker.request import Request from celery.utils import noop __all__ = ['Batches'] diff --git a/celery/tests/worker/test_control.py b/celery/tests/worker/test_control.py index fc587f0d1..829bd9c9d 100644 --- a/celery/tests/worker/test_control.py +++ b/celery/tests/worker/test_control.py @@ -16,7 +16,7 @@ from celery.worker import consumer from celery.worker import control from celery.worker import state as worker_state -from celery.worker.job import Request +from celery.worker.request import Request from celery.worker.state import revoked from celery.worker.control import Panel from celery.worker.pidbox import Pidbox, gPidbox diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index 280152475..e4cbddf6c 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -35,8 +35,8 @@ from celery.five import monotonic from celery.signals import task_revoked from celery.utils import uuid -from celery.worker import job as module -from celery.worker.job import Request, logger as req_logger +from celery.worker import request as module +from celery.worker.request import Request, logger as req_logger from celery.worker.state import revoked from celery.tests.case import ( @@ -247,7 +247,7 @@ def test_invalid_expires_raises_InvalidTaskError(self): self.get_request(self.add.s(2, 2).set(expires='12345')) def test_valid_expires_with_utc_makes_aware(self): - with patch('celery.worker.job.maybe_make_aware') as mma: + with patch('celery.worker.request.maybe_make_aware') as mma: self.get_request(self.add.s(2, 2).set(expires=10)) self.assertTrue(mma.called) @@ -592,8 +592,8 @@ def test_from_message_invalid_kwargs(self): with self.assertRaises(InvalidTaskError): raise req.execute().exception - @patch('celery.worker.job.error') - @patch('celery.worker.job.warn') + @patch('celery.worker.request.error') + @patch('celery.worker.request.warn') def test_on_timeout(self, warn, error): job = self.xRequest() diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index e939a4e8b..1596c4616 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -24,7 +24,7 @@ from celery.worker import components from celery.worker import consumer from celery.worker.consumer import Consumer as __Consumer -from celery.worker.job import Request +from celery.worker.request import Request from celery.utils import worker_direct from celery.utils.serialization import pickle from celery.utils.timer2 import Timer diff --git a/celery/worker/control.py b/celery/worker/control.py index 2067d4043..b0bb93fde 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -22,8 +22,8 @@ from celery.utils import jsonify from . import state as worker_state +from .request import Request from .state import revoked -from .job import Request __all__ = ['Panel'] DEFAULT_TASK_INFO_ITEMS = ('exchange', 'routing_key', 'rate_limit') diff --git a/celery/worker/job.py b/celery/worker/request.py similarity index 99% rename from celery/worker/job.py rename to celery/worker/request.py index 0fd57f95a..949cc95a3 100644 --- a/celery/worker/job.py +++ b/celery/worker/request.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ - celery.worker.job - ~~~~~~~~~~~~~~~~~ + celery.worker.request + ~~~~~~~~~~~~~~~~~~~~~ This module defines the :class:`Request` class, which specifies how tasks are executed. diff --git a/celery/worker/state.py b/celery/worker/state.py index 8abaa5d73..f2f7a79fa 100644 --- a/celery/worker/state.py +++ b/celery/worker/state.py @@ -42,10 +42,10 @@ #: being expired when the max limit has been exceeded. REVOKE_EXPIRES = 10800 -#: set of all reserved :class:`~celery.worker.job.Request`'s. +#: set of all reserved :class:`~celery.worker.request.Request`'s. reserved_requests = set() -#: set of currently active :class:`~celery.worker.job.Request`'s. +#: set of currently active :class:`~celery.worker.request.Request`'s. active_requests = set() #: count of tasks accepted by the worker, sorted by type. diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index 4f12741ee..397aa4f24 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -15,7 +15,7 @@ from celery.utils.log import get_logger from celery.utils.timeutils import timezone -from .job import Request, RequestV1 +from .request import Request, RequestV1 from .state import task_reserved __all__ = ['default'] diff --git a/docs/internals/app-overview.rst b/docs/internals/app-overview.rst index 33dd4e815..602f33d29 100644 --- a/docs/internals/app-overview.rst +++ b/docs/internals/app-overview.rst @@ -226,7 +226,7 @@ App Dependency Tree * celery.apps.worker.Worker * celery.worker.WorkerController * celery.worker.consumer.Consumer - * celery.worker.job.TaskRequest + * celery.worker.request.Request * celery.events.EventDispatcher * celery.worker.control.ControlDispatch * celery.woker.control.registry.Panel diff --git a/docs/internals/worker.rst b/docs/internals/worker.rst index 30eb64185..c1695cb48 100644 --- a/docs/internals/worker.rst +++ b/docs/internals/worker.rst @@ -35,7 +35,7 @@ Receives messages from the broker using `Kombu`_. .. _`Kombu`: http://pypi.python.org/pypi/kombu When a message is received it's converted into a -:class:`celery.worker.job.TaskRequest` object. +:class:`celery.worker.request.Request` object. Tasks with an ETA, or rate-limit are entered into the `timer`, messages that can be immediately processed are sent to the execution pool. diff --git a/docs/reference/celery.worker.job.rst b/docs/reference/celery.worker.request.rst similarity index 57% rename from docs/reference/celery.worker.job.rst rename to docs/reference/celery.worker.request.rst index 36fc1a7b3..8821d6bef 100644 --- a/docs/reference/celery.worker.job.rst +++ b/docs/reference/celery.worker.request.rst @@ -1,11 +1,11 @@ ===================================== - celery.worker.job + celery.worker.request ===================================== .. contents:: :local: -.. currentmodule:: celery.worker.job +.. currentmodule:: celery.worker.request -.. automodule:: celery.worker.job +.. automodule:: celery.worker.request :members: :undoc-members: diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 5f1c72a08..a8ae3ea02 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -47,7 +47,7 @@ celery.apps.beat celery.worker celery.worker.consumer - celery.worker.job + celery.worker.request celery.worker.state celery.worker.strategy celery.bin.base diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index 41878034f..a4417c78a 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -463,7 +463,7 @@ Methods .. method:: apply_eta_task(request) Schedule eta task to execute based on the ``request.eta`` attribute. - (:class:`~celery.worker.job.Request`) + (:class:`~celery.worker.request.Request`) diff --git a/docs/userguide/signals.rst b/docs/userguide/signals.rst index 4d6d72e69..7b927472b 100644 --- a/docs/userguide/signals.rst +++ b/docs/userguide/signals.rst @@ -271,7 +271,7 @@ Provides arguments: * request - This is a :class:`~celery.worker.job.Request` instance, and not + This is a :class:`~celery.worker.request.Request` instance, and not ``task.request``. When using the prefork pool this signal is dispatched in the parent process, so ``task.request`` is not available and should not be used. Use this object instead, which should have many diff --git a/docs/whatsnew-3.1.rst b/docs/whatsnew-3.1.rst index 6ac166166..99b6c3ad3 100644 --- a/docs/whatsnew-3.1.rst +++ b/docs/whatsnew-3.1.rst @@ -1072,8 +1072,9 @@ In Other News (Issue #1555). The revoked signal is dispatched after the task request is removed from - the stack, so it must instead use the :class:`~celery.worker.job.Request` - object to get information about the task. + the stack, so it must instead use the + :class:`~celery.worker.request.Request` object to get information + about the task. - Worker: New :option:`-X` command line argument to exclude queues (Issue #1399). @@ -1235,7 +1236,7 @@ Internal changes - Result backends (:class:`celery.backends.base.BaseBackend`) - :class:`celery.worker.WorkController` - :class:`celery.worker.Consumer` - - :class:`celery.worker.job.Request` + - :class:`celery.worker.request.Request` This means that you have to pass a specific app when instantiating these classes. From 67e0dec844269d364d275e1c1c21546c249fcd3b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 9 May 2014 17:50:54 +0100 Subject: [PATCH 0111/1103] Removes module celery.task.trace as scheduled for 3.2 --- celery/task/trace.py | 12 ------------ extra/release/doc4allmods | 1 - 2 files changed, 13 deletions(-) delete mode 100644 celery/task/trace.py diff --git a/celery/task/trace.py b/celery/task/trace.py deleted file mode 100644 index 5e5f5a8e9..000000000 --- a/celery/task/trace.py +++ /dev/null @@ -1,12 +0,0 @@ -"""This module has moved to celery.app.trace.""" -from __future__ import absolute_import - -import sys - -from celery.utils import warn_deprecated - -warn_deprecated('celery.task.trace', removal='3.2', - alternative='Please use celery.app.trace instead.') - -from celery.app import trace -sys.modules[__name__] = trace diff --git a/extra/release/doc4allmods b/extra/release/doc4allmods index b08b769eb..c36cb6273 100755 --- a/extra/release/doc4allmods +++ b/extra/release/doc4allmods @@ -7,7 +7,6 @@ SKIP_FILES="celery.five.rst celery.task.rst celery.task.base.rst celery.task.sets.rst - celery.task.trace.rst celery.bin.rst celery.bin.celeryd_detach.rst celery.contrib.rst From 2e1cad9e046feb79c0954157e05f3386e3f649d4 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 9 May 2014 18:01:37 +0100 Subject: [PATCH 0112/1103] Removes magic keyword arguments support --- celery/_state.py | 4 +-- celery/app/base.py | 19 +++++--------- celery/app/builtins.py | 3 --- celery/app/task.py | 26 +++---------------- celery/app/trace.py | 12 ++++----- celery/app/utils.py | 1 - celery/five.py | 9 +------ celery/task/base.py | 4 +-- celery/task/http.py | 3 +-- celery/tests/app/test_app.py | 4 +-- celery/tests/compat_modules/test_compat.py | 23 ---------------- .../tests/compat_modules/test_compat_utils.py | 4 --- .../tests/compat_modules/test_decorators.py | 1 - celery/tests/tasks/test_tasks.py | 4 --- celery/tests/worker/test_request.py | 3 --- celery/utils/__init__.py | 9 ------- 16 files changed, 21 insertions(+), 108 deletions(-) diff --git a/celery/_state.py b/celery/_state.py index a76b3f4b7..7592ca242 100644 --- a/celery/_state.py +++ b/celery/_state.py @@ -77,10 +77,8 @@ def _get_current_app(): #: creates the global fallback app instance. from celery.app import Celery set_default_app(Celery( - 'default', + 'default', fixups=[], set_as_current=False, loader=os.environ.get('CELERY_LOADER') or 'default', - fixups=[], - set_as_current=False, accept_magic_kwargs=True, )) return _tls.current_app or default_app diff --git a/celery/app/base.py b/celery/app/base.py index 02590025a..04b1749a2 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -127,11 +127,13 @@ class Celery(object): #: Signal sent after app has been finalized. on_after_finalize = None + #: ignored + accept_magic_kwargs = False + def __init__(self, main=None, loader=None, backend=None, amqp=None, events=None, log=None, control=None, - set_as_current=True, accept_magic_kwargs=False, - tasks=None, broker=None, include=None, changes=None, - config_source=None, fixups=None, task_cls=None, + set_as_current=True, tasks=None, broker=None, include=None, + changes=None, config_source=None, fixups=None, task_cls=None, autofinalize=True, **kwargs): self.clock = LamportClock() self.main = main @@ -144,7 +146,6 @@ def __init__(self, main=None, loader=None, backend=None, self.task_cls = task_cls or self.task_cls self.set_as_current = set_as_current self.registry_cls = symbol_by_name(self.registry_cls) - self.accept_magic_kwargs = accept_magic_kwargs self.user_options = defaultdict(set) self.steps = defaultdict(set) self.autofinalize = autofinalize @@ -239,12 +240,6 @@ def _create_task_cls(fun): cons = lambda app: app._task_from_fun(fun, **opts) cons.__name__ = fun.__name__ connect_on_app_finalize(cons) - if self.accept_magic_kwargs: # compat mode - task = self._task_from_fun(fun, **opts) - if filter: - task = filter(task) - return task - if self.finalized or opts.get('_force_evaluate'): ret = self._task_from_fun(fun, **opts) else: @@ -276,7 +271,6 @@ def _task_from_fun(self, fun, **options): T = type(fun.__name__, (base, ), dict({ 'app': self, - 'accept_magic_kwargs': False, 'run': fun if bind else staticmethod(fun), '_decorated': True, '__doc__': fun.__doc__, @@ -581,7 +575,6 @@ def __reduce_keys__(self): 'events': self.events_cls, 'log': self.log_cls, 'control': self.control_cls, - 'accept_magic_kwargs': self.accept_magic_kwargs, 'fixups': self.fixups, 'config_source': self._config_source, 'task_cls': self.task_cls, @@ -592,7 +585,7 @@ def __reduce_args__(self): return (self.main, self.conf.changes, self.loader_cls, self.backend_cls, self.amqp_cls, self.events_cls, self.log_cls, self.control_cls, - self.accept_magic_kwargs, self._config_source) + False, self._config_source) @cached_property def Worker(self): diff --git a/celery/app/builtins.py b/celery/app/builtins.py index 81d5f074c..0100cb86c 100644 --- a/celery/app/builtins.py +++ b/celery/app/builtins.py @@ -144,7 +144,6 @@ def add_group_task(app): class Group(app.Task): app = _app name = 'celery.group' - accept_magic_kwargs = False _decorated = True def run(self, tasks, result, group_id, partial_args, @@ -172,7 +171,6 @@ def add_chain_task(app): class Chain(app.Task): app = _app name = 'celery.chain' - accept_magic_kwargs = False _decorated = True return Chain @@ -188,7 +186,6 @@ def add_chord_task(app): class Chord(app.Task): app = _app name = 'celery.chord' - accept_magic_kwargs = False ignore_result = False _decorated = True diff --git a/celery/app/task.py b/celery/app/task.py index 705c26269..1fa6b3381 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -20,7 +20,7 @@ from celery.five import class_property, items, with_metaclass from celery.local import Proxy from celery.result import EagerResult -from celery.utils import gen_task_name, fun_takes_kwargs, uuid, maybe_reraise +from celery.utils import gen_task_name, uuid, maybe_reraise from celery.utils.functional import mattrgetter, maybe_list from celery.utils.imports import instantiate from celery.utils.mail import ErrorMail @@ -237,10 +237,6 @@ class Task(object): #: If :const:`True` the task is an abstract base class. abstract = True - #: If disabled the worker will not forward magic keyword arguments. - #: Deprecated and scheduled for removal in v4.0. - accept_magic_kwargs = False - #: Maximum number of retries before giving up. If set to :const:`None`, #: it will **never** stop retrying. max_retries = 3 @@ -345,6 +341,9 @@ class Task(object): 'CELERY_STORE_ERRORS_EVEN_IF_IGNORED'), ) + #: ignored + accept_magic_kwargs = False + _backend = None # set by backend property. __bound__ = False @@ -362,8 +361,6 @@ def bind(self, app): for attr_name, config_name in self.from_config: if getattr(self, attr_name, None) is None: setattr(self, attr_name, conf[config_name]) - if self.accept_magic_kwargs is None: - self.accept_magic_kwargs = app.accept_magic_kwargs # decorate with annotations from config. if not was_bound: @@ -720,21 +717,6 @@ def apply(self, args=None, kwargs=None, 'errbacks': maybe_list(link_error), 'headers': options.get('headers'), 'delivery_info': {'is_eager': True}} - if self.accept_magic_kwargs: - default_kwargs = {'task_name': task.name, - 'task_id': task_id, - 'task_retries': retries, - 'task_is_eager': True, - 'logfile': options.get('logfile'), - 'loglevel': options.get('loglevel', 0), - 'delivery_info': {'is_eager': True}} - supported_keys = fun_takes_kwargs(task.run, default_kwargs) - extend_with = { - key: val for key, val in items(default_kwargs) - if key in supported_keys - } - kwargs.update(extend_with) - tb = None tracer = build_tracer( task.name, task, eager=True, diff --git a/celery/app/trace.py b/celery/app/trace.py index c0523c3f2..c21ef1ee2 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -467,10 +467,11 @@ def _trace_task_ret(name, uuid, request, body, content_type, trace_task_ret = _trace_task_ret -def _fast_trace_task_v1(task, uuid, args, kwargs, request={}): +def _fast_trace_task_v1(task, uuid, args, kwargs, request={}, _loc=_localized): # setup_worker_optimizations will point trace_task_ret to here, # so this is the function used in the worker. - R, I, T, Rstr = _tasks[task].__trace__(uuid, args, kwargs, request)[0] + tasks, _ = _loc + R, I, T, Rstr = tasks[task].__trace__(uuid, args, kwargs, request)[0] # exception instance if error, else result text return (1, R, T) if I else (0, Rstr, T) @@ -479,11 +480,8 @@ def _fast_trace_task(task, uuid, request, body, content_type, content_encoding, loads=loads_message, _loc=_localized, **extra_request): tasks, accept = _loc - try: - args, kwargs = loads(body, content_type, content_encoding, - accept=accept) - except Exception as exc: - print('OH NOEEES: %r' % (exc, )) + args, kwargs = loads(body, content_type, content_encoding, + accept=accept) request.update(args=args, kwargs=kwargs, **extra_request) R, I, T, Rstr = tasks[task].__trace__( uuid, args, kwargs, request, diff --git a/celery/app/utils.py b/celery/app/utils.py index a409d8fac..d017de2a3 100644 --- a/celery/app/utils.py +++ b/celery/app/utils.py @@ -152,7 +152,6 @@ def build_standard_kwargs(self, main, changes, loader, backend, amqp, return dict(main=main, loader=loader, backend=backend, amqp=amqp, changes=changes, events=events, log=log, control=control, set_as_current=False, - accept_magic_kwargs=accept_magic_kwargs, config_source=config_source) def construct(self, cls, **kwargs): diff --git a/celery/five.py b/celery/five.py index 56c640ac8..b7fe25eac 100644 --- a/celery/five.py +++ b/celery/five.py @@ -210,15 +210,8 @@ def getappattr(path): return current_app._rgetattr(path) -def _compat_task_decorator(*args, **kwargs): - from celery import current_app - kwargs.setdefault('accept_magic_kwargs', True) - return current_app.task(*args, **kwargs) - - def _compat_periodic_task_decorator(*args, **kwargs): from celery.task import periodic_task - kwargs.setdefault('accept_magic_kwargs', True) return periodic_task(*args, **kwargs) @@ -228,7 +221,7 @@ def _compat_periodic_task_decorator(*args, **kwargs): 'send_task': 'send_task', }, 'decorators': { - 'task': _compat_task_decorator, + 'task': 'task', 'periodic_task': _compat_periodic_task_decorator, }, 'log': { diff --git a/celery/task/base.py b/celery/task/base.py index 6feffc48d..a47885aeb 100644 --- a/celery/task/base.py +++ b/celery/task/base.py @@ -51,7 +51,6 @@ class Task(BaseTask): priority = None type = 'regular' disable_error_emails = False - accept_magic_kwargs = False from_config = BaseTask.from_config + ( ('exchange_type', 'CELERY_DEFAULT_EXCHANGE_TYPE'), @@ -178,8 +177,7 @@ def on_bound(cls, app): def task(*args, **kwargs): """Deprecated decorator, please use :func:`celery.task`.""" - return current_app.task(*args, **dict({'accept_magic_kwargs': False, - 'base': Task}, **kwargs)) + return current_app.task(*args, **dict({'base': Task}, **kwargs)) def periodic_task(*args, **options): diff --git a/celery/task/http.py b/celery/task/http.py index 2c9d8604b..62b89b896 100644 --- a/celery/task/http.py +++ b/celery/task/http.py @@ -162,8 +162,7 @@ def http_headers(self): return headers -@shared_task(name='celery.http_dispatch', bind=True, - url=None, method=None, accept_magic_kwargs=False) +@shared_task(name='celery.http_dispatch', bind=True, url=None, method=None) def dispatch(self, url=None, method='GET', **kwargs): """Task dispatching to an URL. diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index 5bb1ef61e..69187d0a0 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -258,7 +258,7 @@ def foo(): self.assertFalse(sh.called) def test_task_compat_with_filter(self): - with self.Celery(accept_magic_kwargs=True) as app: + with self.Celery() as app: check = Mock() def filter(task): @@ -271,7 +271,7 @@ def foo(): check.assert_called_with(foo) def test_task_with_filter(self): - with self.Celery(accept_magic_kwargs=False) as app: + with self.Celery() as app: check = Mock() def filter(task): diff --git a/celery/tests/compat_modules/test_compat.py b/celery/tests/compat_modules/test_compat.py index d285188e0..58f0cea0c 100644 --- a/celery/tests/compat_modules/test_compat.py +++ b/celery/tests/compat_modules/test_compat.py @@ -15,29 +15,6 @@ from celery.tests.case import AppCase, depends_on_current_app -class test_Task(AppCase): - - def test_base_task_inherits_magic_kwargs_from_app(self): - from celery.task import Task as OldTask - - class timkX(OldTask): - abstract = True - - with self.Celery(set_as_current=False, - accept_magic_kwargs=True) as app: - timkX.bind(app) - # see #918 - self.assertFalse(timkX.accept_magic_kwargs) - - from celery import Task as NewTask - - class timkY(NewTask): - abstract = True - - timkY.bind(app) - self.assertFalse(timkY.accept_magic_kwargs) - - @depends_on_current_app class test_periodic_tasks(AppCase): diff --git a/celery/tests/compat_modules/test_compat_utils.py b/celery/tests/compat_modules/test_compat_utils.py index b041a0b3e..d1ef81a98 100644 --- a/celery/tests/compat_modules/test_compat_utils.py +++ b/celery/tests/compat_modules/test_compat_utils.py @@ -40,11 +40,7 @@ def test_decorators_task(self): def _test_decorators_task(): pass - self.assertTrue(_test_decorators_task.accept_magic_kwargs) - def test_decorators_periodic_task(self): @celery.decorators.periodic_task(run_every=3600) def _test_decorators_ptask(): pass - - self.assertTrue(_test_decorators_ptask.accept_magic_kwargs) diff --git a/celery/tests/compat_modules/test_decorators.py b/celery/tests/compat_modules/test_decorators.py index 9f5dff947..df95916ae 100644 --- a/celery/tests/compat_modules/test_decorators.py +++ b/celery/tests/compat_modules/test_decorators.py @@ -27,7 +27,6 @@ def setup(self): def assertCompatDecorator(self, decorator, type, **opts): task = decorator(**opts)(add) self.assertEqual(task(8, 8), 16) - self.assertTrue(task.accept_magic_kwargs) self.assertIsInstance(task, type) def test_task(self): diff --git a/celery/tests/tasks/test_tasks.py b/celery/tests/tasks/test_tasks.py index 5607c255d..6f11ba1be 100644 --- a/celery/tests/tasks/test_tasks.py +++ b/celery/tests/tasks/test_tasks.py @@ -363,10 +363,6 @@ def test_task_class_repr(self): self.mytask.app.Task._app = None self.assertIn('unbound', repr(self.mytask.app.Task, )) - def test_bind_no_magic_kwargs(self): - self.mytask.accept_magic_kwargs = None - self.mytask.bind(self.mytask.app) - def test_annotate(self): with patch('celery.app.task.resolve_all_annotations') as anno: anno.return_value = [{'FOO': 'BAR'}] diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index e4cbddf6c..02f065df1 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -835,9 +835,6 @@ def apply_async(self, target, args=None, kwargs=None, self.assertEqual(p.args[1], tid) self.assertEqual(p.args[3], job.message.body) - job.task.accept_magic_kwargs = False - job.execute_using_pool(p) - def _test_on_failure(self, exception): tid = uuid() job = self.xRequest(id=tid, args=[4]) diff --git a/celery/utils/__init__.py b/celery/utils/__init__.py index 7bf6c3137..a3264b8d6 100644 --- a/celery/utils/__init__.py +++ b/celery/utils/__init__.py @@ -19,7 +19,6 @@ from collections import Callable from functools import partial, wraps -from inspect import getargspec from pprint import pprint from kombu.entity import Exchange, Queue @@ -189,14 +188,6 @@ def is_iterable(obj): return True -def fun_takes_kwargs(fun, kwlist=[]): - # deprecated - S = getattr(fun, 'argspec', getargspec(fun)) - if S.keywords is not None: - return kwlist - return [kw for kw in kwlist if kw in S.args] - - def isatty(fh): try: return fh.isatty() From 469fd49d619131cd620262e304a7e07984f47a01 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Mon, 12 May 2014 14:38:49 +0200 Subject: [PATCH 0113/1103] Clarify behavior of rate limits in docs --- docs/userguide/tasks.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 8e7cb0739..24df1cdb9 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -514,10 +514,14 @@ General If it is an integer or float, it is interpreted as "tasks per second". The rate limits can be specified in seconds, minutes or hours - by appending `"/s"`, `"/m"` or `"/h"` to the value. - Example: `"100/m"` (hundred tasks a minute). Default is the - :setting:`CELERY_DEFAULT_RATE_LIMIT` setting, which if not specified means - rate limiting for tasks is disabled by default. + by appending `"/s"`, `"/m"` or `"/h"` to the value. Tasks will be evenly + distributed over the specified time frame. + + Example: `"100/m"` (hundred tasks a minute). This will enforce a minimum + delay of 10ms between starting two tasks. + + Default is the :setting:`CELERY_DEFAULT_RATE_LIMIT` setting, + which if not specified means rate limiting for tasks is disabled by default. .. attribute:: Task.time_limit From 924f3076f6fe6ad133217123a357f9c98e967366 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 13 May 2014 12:46:36 +0100 Subject: [PATCH 0114/1103] PromiseProxy: Only remove original object if evaluation succeeded --- celery/local.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/celery/local.py b/celery/local.py index ada6e9381..e042fa0a0 100644 --- a/celery/local.py +++ b/celery/local.py @@ -251,7 +251,9 @@ def __evaluate__(self, thing = Proxy._get_current_object(self) object.__setattr__(self, '__thing', thing) return thing - finally: + except: + raise + else: for attr in _clean: try: object.__delattr__(self, attr) From 2d68f8ddace82c4e490465faba865cf27826603a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 14 May 2014 16:53:37 +0100 Subject: [PATCH 0115/1103] Fixes weird traceback issues with connection_or_acquire + producer_or_acquire --- celery/app/base.py | 32 ++++++++++++----------------- celery/utils/objects.py | 45 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 19 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 04b1749a2..18c280a18 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -9,6 +9,7 @@ from __future__ import absolute_import import os +import sys import threading import warnings @@ -37,7 +38,7 @@ from celery.utils.dispatch import Signal from celery.utils.functional import first, maybe_list from celery.utils.imports import instantiate, symbol_by_name -from celery.utils.objects import mro_lookup +from celery.utils.objects import FallbackContext, mro_lookup from .annotations import prepare as prepare_annotations from .defaults import DEFAULTS, find_deprecated_settings @@ -406,27 +407,20 @@ def connection(self, hostname=None, userid=None, password=None, ) broker_connection = connection - @contextmanager - def connection_or_acquire(self, connection=None, pool=True, - *args, **kwargs): - if connection: - yield connection - else: - if pool: - with self.pool.acquire(block=True) as connection: - yield connection - else: - with self.connection() as connection: - yield connection + def _acquire_connection(self, pool=True): + """Helper for :meth:`connection_or_acquire`.""" + if pool: + return self.pool.acquire(block=True) + return self.connection() + + def connection_or_acquire(self, connection=None, pool=True, *_, **__): + return FallbackContext(connection, self._acquire_connection, pool=pool) default_connection = connection_or_acquire # XXX compat - @contextmanager def producer_or_acquire(self, producer=None): - if producer: - yield producer - else: - with self.amqp.producer_pool.acquire(block=True) as producer: - yield producer + return FallbackContext( + producer, self.amqp.producer_pool.acquire, block=True, + ) default_producer = producer_or_acquire # XXX compat def prepare_config(self, c): diff --git a/celery/utils/objects.py b/celery/utils/objects.py index b2ad646b3..c81550af2 100644 --- a/celery/utils/objects.py +++ b/celery/utils/objects.py @@ -35,3 +35,48 @@ def mro_lookup(cls, attr, stop=(), monkey_patched=[]): return if attr in node.__dict__: return node + + +class FallbackContext(object): + """The built-in ``@contextmanager`` utility does not work well + when wrapping other contexts, as the traceback is wrong when + the wrapped context raises. + + This solves this problem and can be used instead of ``@contextmanager`` + in this example:: + + @contextmanager + def connection_or_default_connection(connection=None): + if connection: + # user already has a connection, should not close + # after use + yield connection + else: + # must have new connection, and also close the connection + # after the block returns + with create_new_connection() as connection: + yield connection + + This wrapper can be used instead for the above like this:: + + def connection_or_default_connection(connection=None): + return FallbackContext(connection, create_new_connection) + + """ + + def __init__(self, provided, fallback, *fb_args, **fb_kwargs): + self.provided = provided + self.fallback = fallback + self.fb_args = fb_args + self.fb_kwargs = fb_kwargs + self._context = None + + def __enter__(self): + if self.provided is not None: + return self.provided + context = self._context = self.fallback(*self.fb_args, **self.fb_kwargs).__enter__() + return context + + def __exit__(self, *exc_info): + if self._context is not None: + return self._context.__exit__(*exc_info) From 6c08d2acdf2cf1b4a583f594d11384f51e0c3aba Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 14 May 2014 16:53:37 +0100 Subject: [PATCH 0116/1103] Fixes weird traceback issues with connection_or_acquire + producer_or_acquire --- celery/app/base.py | 32 ++++++++++++----------------- celery/utils/objects.py | 45 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 19 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index b0079f7db..66367e6f7 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -9,6 +9,7 @@ from __future__ import absolute_import import os +import sys import threading import warnings @@ -37,7 +38,7 @@ from celery.utils.dispatch import Signal from celery.utils.functional import first, maybe_list from celery.utils.imports import instantiate, symbol_by_name -from celery.utils.objects import mro_lookup +from celery.utils.objects import FallbackContext, mro_lookup from .annotations import prepare as prepare_annotations from .defaults import DEFAULTS, find_deprecated_settings @@ -411,27 +412,20 @@ def connection(self, hostname=None, userid=None, password=None, ) broker_connection = connection - @contextmanager - def connection_or_acquire(self, connection=None, pool=True, - *args, **kwargs): - if connection: - yield connection - else: - if pool: - with self.pool.acquire(block=True) as connection: - yield connection - else: - with self.connection() as connection: - yield connection + def _acquire_connection(self, pool=True): + """Helper for :meth:`connection_or_acquire`.""" + if pool: + return self.pool.acquire(block=True) + return self.connection() + + def connection_or_acquire(self, connection=None, pool=True, *_, **__): + return FallbackContext(connection, self._acquire_connection, pool=pool) default_connection = connection_or_acquire # XXX compat - @contextmanager def producer_or_acquire(self, producer=None): - if producer: - yield producer - else: - with self.amqp.producer_pool.acquire(block=True) as producer: - yield producer + return FallbackContext( + producer, self.amqp.producer_pool.acquire, block=True, + ) default_producer = producer_or_acquire # XXX compat def prepare_config(self, c): diff --git a/celery/utils/objects.py b/celery/utils/objects.py index b2ad646b3..c81550af2 100644 --- a/celery/utils/objects.py +++ b/celery/utils/objects.py @@ -35,3 +35,48 @@ def mro_lookup(cls, attr, stop=(), monkey_patched=[]): return if attr in node.__dict__: return node + + +class FallbackContext(object): + """The built-in ``@contextmanager`` utility does not work well + when wrapping other contexts, as the traceback is wrong when + the wrapped context raises. + + This solves this problem and can be used instead of ``@contextmanager`` + in this example:: + + @contextmanager + def connection_or_default_connection(connection=None): + if connection: + # user already has a connection, should not close + # after use + yield connection + else: + # must have new connection, and also close the connection + # after the block returns + with create_new_connection() as connection: + yield connection + + This wrapper can be used instead for the above like this:: + + def connection_or_default_connection(connection=None): + return FallbackContext(connection, create_new_connection) + + """ + + def __init__(self, provided, fallback, *fb_args, **fb_kwargs): + self.provided = provided + self.fallback = fallback + self.fb_args = fb_args + self.fb_kwargs = fb_kwargs + self._context = None + + def __enter__(self): + if self.provided is not None: + return self.provided + context = self._context = self.fallback(*self.fb_args, **self.fb_kwargs).__enter__() + return context + + def __exit__(self, *exc_info): + if self._context is not None: + return self._context.__exit__(*exc_info) From ca8b519222a13b11e3e4e5d05011b230ae548ad1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 15 May 2014 12:55:17 +0100 Subject: [PATCH 0117/1103] Time limit in message is now a list --- celery/app/amqp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index e8e801529..5f9813e73 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -306,7 +306,7 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, 'group': group_id, 'chord': chord, 'retries': retries, - 'timelimit': (time_limit, soft_time_limit), + 'timelimit': [time_limit, soft_time_limit], 'root_id': root_id, 'parent_id': parent_id, }, From c7c5fc4d2f9e3596d79dbc64e6c624edfddda373 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 15 May 2014 13:23:59 +0100 Subject: [PATCH 0118/1103] Removes dead code --- celery/canvas.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 16924eeba..2c2883bcf 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -609,9 +609,6 @@ def apply_async(self, args=(), kwargs=None, add_to_parent=True, parent_task.add_trail(result) return result - return type(*type.prepare(dict(self.options, **options), tasks, args), - add_to_parent=add_to_parent) - def apply(self, args=(), kwargs={}, **options): app = self.app if not self.tasks: From 904e2af39b232c8b0358ba9e08248c2989bc4669 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 15 May 2014 14:04:42 +0100 Subject: [PATCH 0119/1103] Exception sent by JSON serialized worker can be read by pickle configured worker --- celery/backends/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 41ce1ef17..aec6dd3f4 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -165,11 +165,11 @@ def prepare_exception(self, exc, serializer=None): def exception_to_python(self, exc): """Convert serialized exception to Python exception.""" - if self.serializer in EXCEPTION_ABLE_CODECS: - return get_pickled_exception(exc) - elif not isinstance(exc, BaseException): - return create_exception_cls( + if not isinstance(exc, BaseException): + exc = create_exception_cls( from_utf8(exc['exc_type']), __name__)(exc['exc_message']) + if self.serializer in EXCEPTION_ABLE_CODECS: + exc = get_pickled_exception(exc) return exc def prepare_value(self, result): From acee6680ce009688b5d497b245466e5f02d7afb2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 15 May 2014 14:06:52 +0100 Subject: [PATCH 0120/1103] Needs to copy buffer into bytes early so that librabbitmq does not release the buffer --- celery/concurrency/base.py | 1 + celery/concurrency/solo.py | 1 + celery/five.py | 8 ++++++++ celery/worker/consumer.py | 10 +--------- celery/worker/request.py | 7 +++---- celery/worker/strategy.py | 10 ++++++++-- 6 files changed, 22 insertions(+), 15 deletions(-) diff --git a/celery/concurrency/base.py b/celery/concurrency/base.py index 24b39a1ee..4913ffb27 100644 --- a/celery/concurrency/base.py +++ b/celery/concurrency/base.py @@ -72,6 +72,7 @@ class BasePool(object): uses_semaphore = False task_join_will_block = True + body_can_be_buffer = False def __init__(self, limit=None, putlocks=True, forking_enable=True, callbacks_propagate=(), **options): diff --git a/celery/concurrency/solo.py b/celery/concurrency/solo.py index a2dc19970..a83f46219 100644 --- a/celery/concurrency/solo.py +++ b/celery/concurrency/solo.py @@ -17,6 +17,7 @@ class TaskPool(BasePool): """Solo task pool (blocking, inline, fast).""" + body_can_be_buffer = True def __init__(self, *args, **kwargs): super(TaskPool, self).__init__(*args, **kwargs) diff --git a/celery/five.py b/celery/five.py index b7fe25eac..bfe9ff0cd 100644 --- a/celery/five.py +++ b/celery/five.py @@ -28,6 +28,14 @@ def Counter(): # noqa return defaultdict(int) +try: + buffer_t = buffer +except NameError: # pragma: no cover + # Py3 does not have buffer, but we only need isinstance. + + class buffer_t(object): # noqa + pass + ############## py3k ######################################################### import sys PY3 = sys.version_info[0] == 3 diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 3bb430c43..e4d741422 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -35,7 +35,7 @@ from celery.app.trace import build_tracer from celery.canvas import signature from celery.exceptions import InvalidTaskError -from celery.five import items, values +from celery.five import buffer_t, items, values from celery.utils.functional import noop from celery.utils.log import get_logger from celery.utils.text import truncate @@ -44,14 +44,6 @@ from . import heartbeat, loops, pidbox from .state import task_reserved, maybe_shutdown, revoked, reserved_requests -try: - buffer_t = buffer -except NameError: # pragma: no cover - # Py3 does not have buffer, but we only need isinstance. - - class buffer_t(object): # noqa - pass - __all__ = [ 'Consumer', 'Connection', 'Events', 'Heart', 'Control', 'Tasks', 'Evloop', 'Agent', 'Mingle', 'Gossip', 'dump_body', diff --git a/celery/worker/request.py b/celery/worker/request.py index 949cc95a3..83aa4c36e 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -108,13 +108,13 @@ class Request(object): def __init__(self, message, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, - task=None, on_reject=noop, **opts): + task=None, on_reject=noop, body=None, **opts): headers = message.headers self.app = app self.message = message name = self.name = headers['c_type'] self.id = headers['id'] - self.body = message.body + self.body = message.body if body is None else body self.content_type = message.content_type self.content_encoding = message.content_encoding eta = headers.get('eta') @@ -192,8 +192,7 @@ def execute_using_pool(self, pool, **kwargs): soft_timeout = soft_timeout or task.soft_time_limit result = pool.apply_async( trace_task_ret, - args=(self.name, task_id, self.request_dict, - bytes(body) if isinstance(body, buffer) else body, + args=(self.name, task_id, self.request_dict, self.body, self.content_type, self.content_encoding), kwargs={'hostname': self.hostname, 'is_eager': False}, accept_callback=self.on_accepted, diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index 397aa4f24..31c5050a8 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -12,6 +12,7 @@ from kombu.async.timer import to_timestamp +from celery.five import buffer_t from celery.utils.log import get_logger from celery.utils.timeutils import timezone @@ -25,7 +26,7 @@ def default(task, app, consumer, info=logger.info, error=logger.error, task_reserved=task_reserved, - to_system_tz=timezone.to_system): + to_system_tz=timezone.to_system, bytes=bytes, buffer_t=buffer_t): hostname = consumer.hostname eventer = consumer.event_dispatcher ReqV2 = Request @@ -40,14 +41,19 @@ def default(task, app, consumer, bucket = consumer.task_buckets[task.name] handle = consumer.on_task_request limit_task = consumer._limit_task + body_can_be_buffer = consumer.pool.body_can_be_buffer def task_message_handler(message, body, ack, reject, callbacks, to_timestamp=to_timestamp): if body is None: + body = message.body + if not body_can_be_buffer: + body = bytes(body) if isinstance(body, buffer_t) else body req = ReqV2(message, on_ack=ack, on_reject=reject, app=app, hostname=hostname, eventer=eventer, task=task, - connection_errors=connection_errors) + connection_errors=connection_errors, + body=body) else: req = ReqV1(body, on_ack=ack, on_reject=reject, app=app, From 411b7f0919b11d0e95697ba3877b1bf2bbf70747 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 15 May 2014 14:07:24 +0100 Subject: [PATCH 0121/1103] Worker now stores result for internal errors, including ConrtentDisallowed --- celery/worker/request.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/celery/worker/request.py b/celery/worker/request.py index 83aa4c36e..bac39bb45 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -319,7 +319,7 @@ def on_success(self, failed__retval__runtime, **kwargs): if failed: if isinstance(retval.exception, (SystemExit, KeyboardInterrupt)): raise retval.exception - return self.on_failure(retval) + return self.on_failure(retval, return_ok=True) task_ready(self) if self.task.acks_late: @@ -339,7 +339,7 @@ def on_retry(self, exc_info): exception=safe_repr(exc_info.exception.exc), traceback=safe_str(exc_info.traceback)) - def on_failure(self, exc_info, send_failed_event=True): + def on_failure(self, exc_info, send_failed_event=True, return_ok=False): """Handler called if the task raised an exception.""" task_ready(self) @@ -358,14 +358,14 @@ def on_failure(self, exc_info, send_failed_event=True): # These are special cases where the process would not have had # time to write the result. if self.store_errors: - if isinstance(exc, WorkerLostError): - self.task.backend.mark_as_failure( - self.id, exc, request=self, - ) - elif isinstance(exc, Terminated): + if isinstance(exc, Terminated): self._announce_revoked( 'terminated', True, string(exc), False) send_failed_event = False # already sent revoked event + elif isinstance(exc, WorkerLostError) or not return_ok: + self.task.backend.mark_as_failure( + self.id, exc, request=self, + ) # (acks_late) acknowledge after result stored. if self.task.acks_late: self.acknowledge() @@ -377,6 +377,10 @@ def on_failure(self, exc_info, send_failed_event=True): traceback=exc_info.traceback, ) + if not return_ok: + error('Task handler raised error: %r', exc, + exc_info=exc_info.exc_info) + def acknowledge(self): """Acknowledge task.""" if not self.acknowledged: From a665094b0a25249ed1f7f2612acb48578dcf8bd7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 15 May 2014 15:14:36 +0100 Subject: [PATCH 0122/1103] Tests passing --- celery/local.py | 4 ++-- celery/tests/utils/test_local.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/celery/local.py b/celery/local.py index e042fa0a0..1a10c2d8c 100644 --- a/celery/local.py +++ b/celery/local.py @@ -249,11 +249,10 @@ def __evaluate__(self, '_Proxy__kwargs')): try: thing = Proxy._get_current_object(self) - object.__setattr__(self, '__thing', thing) - return thing except: raise else: + object.__setattr__(self, '__thing', thing) for attr in _clean: try: object.__delattr__(self, attr) @@ -274,6 +273,7 @@ def __evaluate__(self, object.__delattr__(self, '__pending__') except AttributeError: pass + return thing def maybe_evaluate(obj): diff --git a/celery/tests/utils/test_local.py b/celery/tests/utils/test_local.py index 2b50efcda..67b44b221 100644 --- a/celery/tests/utils/test_local.py +++ b/celery/tests/utils/test_local.py @@ -341,6 +341,7 @@ def test_callbacks(self): self.assertTrue(object.__getattribute__(p, '__pending__')) self.assertTrue(repr(p)) + self.assertTrue(p.__evaluated__()) with self.assertRaises(AttributeError): object.__getattribute__(p, '__pending__') cbA.assert_called_with(p) From f838ace597fa335f30bf24a27c39e9759a791d1f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 15 May 2014 16:41:35 +0100 Subject: [PATCH 0123/1103] Now supports task message protocol 1.0 again --- celery/app/amqp.py | 2 +- celery/app/base.py | 2 - celery/app/trace.py | 13 ++- celery/tests/worker/test_loops.py | 2 +- celery/utils/objects.py | 4 +- celery/worker/consumer.py | 26 +++-- celery/worker/request.py | 143 ++++++++++++++++++---------- celery/worker/strategy.py | 53 +++++++---- docs/internals/protov2.rst | 12 +-- funtests/stress/stress/templates.py | 4 + 10 files changed, 171 insertions(+), 90 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 5f9813e73..84ced793c 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -296,7 +296,7 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, return task_message( headers={ 'lang': 'py', - 'c_type': name, + 'task': name, 'id': task_id, 'eta': eta, 'expires': expires, diff --git a/celery/app/base.py b/celery/app/base.py index 18c280a18..c0174df93 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -9,12 +9,10 @@ from __future__ import absolute_import import os -import sys import threading import warnings from collections import defaultdict, deque -from contextlib import contextmanager from copy import deepcopy from operator import attrgetter diff --git a/celery/app/trace.py b/celery/app/trace.py index c21ef1ee2..ec63de83f 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -478,11 +478,16 @@ def _fast_trace_task_v1(task, uuid, args, kwargs, request={}, _loc=_localized): def _fast_trace_task(task, uuid, request, body, content_type, content_encoding, loads=loads_message, _loc=_localized, - **extra_request): + hostname=None, **_): tasks, accept = _loc - args, kwargs = loads(body, content_type, content_encoding, - accept=accept) - request.update(args=args, kwargs=kwargs, **extra_request) + if content_type: + args, kwargs = loads(body, content_type, content_encoding, + accept=accept) + else: + args, kwargs = body + request.update({ + 'args': args, 'kwargs': kwargs, 'hostname': hostname, + }) R, I, T, Rstr = tasks[task].__trace__( uuid, args, kwargs, request, ) diff --git a/celery/tests/worker/test_loops.py b/celery/tests/worker/test_loops.py index 80edd393a..9673c5f7c 100644 --- a/celery/tests/worker/test_loops.py +++ b/celery/tests/worker/test_loops.py @@ -155,7 +155,7 @@ def test_on_task_received_executes_on_task_message(self): def test_on_task_message_missing_name(self): x, on_task, msg, strategy = self.task_context(self.add.s(2, 2)) - msg.headers.pop('c_type') + msg.headers.pop('task') on_task(msg) x.on_unknown_message.assert_called_with(((2, 2), {}), msg) diff --git a/celery/utils/objects.py b/celery/utils/objects.py index c81550af2..1555f9caf 100644 --- a/celery/utils/objects.py +++ b/celery/utils/objects.py @@ -74,7 +74,9 @@ def __init__(self, provided, fallback, *fb_args, **fb_kwargs): def __enter__(self): if self.provided is not None: return self.provided - context = self._context = self.fallback(*self.fb_args, **self.fb_kwargs).__enter__() + context = self._context = self.fallback( + *self.fb_args, **self.fb_kwargs + ).__enter__() return context def __exit__(self, *exc_info): diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index e4d741422..71cf7c635 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -445,24 +445,32 @@ def on_task_received(message): # will defer deserializing the message body to the pool. payload = None try: - type_ = message.headers['c_type'] # protocol v2 + type_ = message.headers['task'] # protocol v2 except TypeError: return on_unknown_message(None, message) except KeyError: payload = message.payload try: - type_ = payload['task'] # protocol v1 + type_, payload = payload['task'], payload # protocol v1 except (TypeError, KeyError): return on_unknown_message(payload, message) try: - strategies[type_]( - message, None, - message.ack_log_error, message.reject_log_error, callbacks, - ) + strategy = strategies[type_] except KeyError as exc: - on_unknown_task(payload, message, exc) - except InvalidTaskError as exc: - on_invalid_task(payload, message, exc) + return on_unknown_task(payload, message, exc) + else: + try: + strategy( + message, payload, message.ack_log_error, + message.reject_log_error, callbacks, + ) + except InvalidTaskError as exc: + return on_invalid_task(payload, message, exc) + except MemoryError: + raise + except Exception as exc: + # XXX handle as internal error? + return on_invalid_task(payload, message, exc) return on_task_received diff --git a/celery/worker/request.py b/celery/worker/request.py index bac39bb45..41b1d765e 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -44,9 +44,6 @@ _does_info = False _does_debug = False -#: Max length of result representation -RESULT_MAXLEN = 128 - def __optimize__(): # this is also called by celery.app.trace.setup_worker_optimizations @@ -65,75 +62,63 @@ def __optimize__(): task_ready = state.task_ready revoked_tasks = state.revoked -#: Use when no message object passed to :class:`Request`. -DEFAULT_FIELDS = { - 'headers': None, - 'reply_to': None, - 'correlation_id': None, - 'delivery_info': { - 'exchange': None, - 'routing_key': None, - 'priority': 0, - 'redelivered': False, - }, -} - - -class RequestV1(object): - if not IS_PYPY: - __slots__ = ( - 'app', 'message', 'name', 'id', 'root_id', 'parent_id', - 'on_ack', 'hostname', 'eventer', 'connection_errors', 'task', - 'eta', 'expires', 'request_dict', 'acknowledged', 'on_reject', - 'utc', 'time_start', 'worker_pid', '_already_revoked', - '_terminate_on_ack', '_apply_result', - '_tzlocal', '__weakref__', '__dict__', - ) - class Request(object): """A request for task execution.""" - utc = True + acknowledged = False + time_start = None + worker_pid = None + timeouts = (None, None) + _already_revoked = False + _terminate_on_ack = None + _apply_result = None + _tzlocal = None + if not IS_PYPY: # pragma: no cover __slots__ = ( 'app', 'name', 'id', 'on_ack', 'body', 'hostname', 'eventer', 'connection_errors', 'task', 'eta', - 'expires', 'request_dict', 'acknowledged', 'on_reject', - 'utc', 'time_start', 'worker_pid', 'timeouts', + 'expires', 'request_dict', 'on_reject', 'utc', 'content_type', 'content_encoding', - '_already_revoked', '_terminate_on_ack', '_apply_result', - '_tzlocal', '__weakref__', '__dict__', + '__weakref__', '__dict__', ) def __init__(self, message, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, - task=None, on_reject=noop, body=None, **opts): - headers = message.headers + task=None, on_reject=noop, body=None, + headers=None, decoded=False, utc=True, + maybe_make_aware=maybe_make_aware, + maybe_iso8601=maybe_iso8601, **opts): + if headers is None: + headers = message.headers + if body is None: + body = message.body self.app = app self.message = message - name = self.name = headers['c_type'] + self.body = body + self.utc = utc + if decoded: + self.content_type = self.content_encoding = None + else: + self.content_type, self.content_encoding = ( + message.content_type, message.content_encoding, + ) + + name = self.name = headers['task'] self.id = headers['id'] - self.body = message.body if body is None else body - self.content_type = message.content_type - self.content_encoding = message.content_encoding - eta = headers.get('eta') - expires = headers.get('expires') - self.timeouts = (headers['timeouts'] if 'timeouts' in headers - else (None, None)) + if 'timeouts' in headers: + self.timeouts = headers['timeouts'] self.on_ack = on_ack self.on_reject = on_reject self.hostname = hostname or socket.gethostname() self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[name] - self.acknowledged = self._already_revoked = False - self.time_start = self.worker_pid = self._terminate_on_ack = None - self._apply_result = None - self._tzlocal = None # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. + eta = headers.get('eta') if eta is not None: try: eta = maybe_iso8601(eta) @@ -143,6 +128,8 @@ def __init__(self, message, on_ack=noop, self.eta = maybe_make_aware(eta, self.tzlocal) else: self.eta = None + + expires = headers.get('expires') if expires is not None: try: expires = maybe_iso8601(expires) @@ -186,15 +173,13 @@ def execute_using_pool(self, pool, **kwargs): if self.revoked(): raise TaskRevokedError(task_id) - body = self.body timeout, soft_timeout = self.timeouts timeout = timeout or task.time_limit soft_timeout = soft_timeout or task.soft_time_limit result = pool.apply_async( trace_task_ret, args=(self.name, task_id, self.request_dict, self.body, - self.content_type, self.content_encoding), - kwargs={'hostname': self.hostname, 'is_eager': False}, + self.content_type, self.content_encoding, self.hostname), accept_callback=self.on_accepted, timeout_callback=self.on_timeout, callback=self.on_success, @@ -449,3 +434,61 @@ def reply_to(self): def correlation_id(self): # used similarly to reply_to return self.request_dict['correlation_id'] + + +def create_request_cls(base, task, pool, hostname, eventer, + ref=ref, revoked_tasks=revoked_tasks, + task_ready=task_ready): + from celery.app.trace import trace_task_ret as trace + default_time_limit = task.time_limit + default_soft_time_limit = task.soft_time_limit + apply_async = pool.apply_async + acks_late = task.acks_late + std_kwargs = {'hostname': hostname, 'is_eager': False} + events = eventer and eventer.enabled + + class Request(base): + + def execute_using_pool(self, pool, **kwargs): + task_id = self.id + if (self.expires or task_id in revoked_tasks) and self.revoked(): + raise TaskRevokedError(task_id) + + timeout, soft_timeout = self.timeouts + timeout = timeout or default_time_limit + soft_timeout = soft_timeout or default_soft_time_limit + result = apply_async( + trace, + args=(self.name, task_id, self.request_dict, self.body, + self.content_type, self.content_encoding), + kwargs=std_kwargs, + accept_callback=self.on_accepted, + timeout_callback=self.on_timeout, + callback=self.on_success, + error_callback=self.on_failure, + soft_timeout=soft_timeout, + timeout=timeout, + correlation_id=task_id, + ) + # cannot create weakref to None + self._apply_result = ref(result) if result is not None else result + return result + + def on_success(self, failed__retval__runtime, **kwargs): + failed, retval, runtime = failed__retval__runtime + if failed: + if isinstance(retval.exception, ( + SystemExit, KeyboardInterrupt)): + raise retval.exception + return self.on_failure(retval, return_ok=True) + task_ready(self) + + if acks_late: + self.acknowledge() + + if events: + self.send_event( + 'task-succeeded', result=retval, runtime=runtime, + ) + + return Request diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index 31c5050a8..a4ed1cac2 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -12,11 +12,12 @@ from kombu.async.timer import to_timestamp +from celery.exceptions import InvalidTaskError from celery.five import buffer_t from celery.utils.log import get_logger from celery.utils.timeutils import timezone -from .request import Request, RequestV1 +from .request import Request, create_request_cls from .state import task_reserved __all__ = ['default'] @@ -24,13 +25,31 @@ logger = get_logger(__name__) +def proto1_to_proto2(message, body): + """Converts Task message protocol 1 arguments to protocol 2. + + Returns tuple of ``(body, headers, already_decoded_status, utc)`` + + """ + try: + args, kwargs = body['args'], body['kwargs'] + kwargs.items + except KeyError: + raise InvalidTaskError('Message does not have args/kwargs') + except AttributeError: + raise InvalidTaskError( + 'Task keyword arguments must be a mapping', + ) + body['headers'] = message.headers + return (args, kwargs), body, True, body.get('utc', True) + + def default(task, app, consumer, info=logger.info, error=logger.error, task_reserved=task_reserved, - to_system_tz=timezone.to_system, bytes=bytes, buffer_t=buffer_t): + to_system_tz=timezone.to_system, bytes=bytes, buffer_t=buffer_t, + proto1_to_proto2=proto1_to_proto2): hostname = consumer.hostname eventer = consumer.event_dispatcher - ReqV2 = Request - ReqV1 = RequestV1 connection_errors = consumer.connection_errors _does_info = logger.isEnabledFor(logging.INFO) events = eventer and eventer.enabled @@ -42,25 +61,27 @@ def default(task, app, consumer, handle = consumer.on_task_request limit_task = consumer._limit_task body_can_be_buffer = consumer.pool.body_can_be_buffer + Req = create_request_cls(Request, task, consumer.pool, hostname, eventer) + + revoked_tasks = consumer.controller.state.revoked def task_message_handler(message, body, ack, reject, callbacks, to_timestamp=to_timestamp): if body is None: - body = message.body + body, headers, decoded, utc = ( + message.body, message.headers, False, True, + ) if not body_can_be_buffer: body = bytes(body) if isinstance(body, buffer_t) else body - req = ReqV2(message, - on_ack=ack, on_reject=reject, app=app, - hostname=hostname, eventer=eventer, task=task, - connection_errors=connection_errors, - body=body) else: - req = ReqV1(body, - on_ack=ack, on_reject=reject, app=app, - hostname=hostname, eventer=eventer, task=task, - connection_errors=connection_errors, - message=message) - if req.revoked(): + body, headers, decoded, utc = proto1_to_proto2(message, body) + req = Req( + message, + on_ack=ack, on_reject=reject, app=app, hostname=hostname, + eventer=eventer, task=task, connection_errors=connection_errors, + body=body, headers=headers, decoded=decoded, utc=utc, + ) + if (req.expires or req.id in revoked_tasks) and req.revoked(): return if _does_info: diff --git a/docs/internals/protov2.rst b/docs/internals/protov2.rst index 8bbdcc4ee..01b2e1325 100644 --- a/docs/internals/protov2.rst +++ b/docs/internals/protov2.rst @@ -28,9 +28,9 @@ Notes - Java/C, etc. can use a thrift/protobuf document as the body -- Dispatches to actor based on ``c_type``, ``c_meth`` headers +- Dispatches to actor based on ``task``, ``meth`` headers - ``c_meth`` is unused by python, but may be used in the future + ``meth`` is unused by python, but may be used in the future to specify class+method pairs. - Chain gains a dedicated field. @@ -52,7 +52,7 @@ Notes - ``root_id`` and ``parent_id`` fields helps keep track of workflows. -- ``c_shadow`` lets you specify a different name for logs, monitors +- ``shadow`` lets you specify a different name for logs, monitors can be used for e.g. meta tasks that calls any function:: from celery.utils.imports import qualname @@ -108,8 +108,8 @@ Definition 'parent_id': (uuid)parent_id, # optional - 'c_meth': (string)unused, - 'c_shadow': (string)replace_name, + 'meth': (string)unused, + 'shadow': (string)replace_name, 'eta': (iso8601)eta, 'expires'; (iso8601)expires, 'callbacks': (list)Signature, @@ -135,7 +135,7 @@ Example message=json.dumps([[2, 2], {}]), application_headers={ 'lang': 'py', - 'c_type': 'proj.tasks.add', + 'task': 'proj.tasks.add', 'chain': [ # reversed chain list {'task': 'proj.tasks.add', 'args': (8, )}, diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index 520481108..977cfec4d 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -125,3 +125,7 @@ class sqs(default): BROKER_TRANSPORT_OPTIONS = { 'region': os.environ.get('AWS_REGION', 'us-east-1'), } + +@template() +class proto1(default): + CELERY_TASK_PROTOCOL = 1 From 7c372aaf8b191728aec2ae24fff8a0c097df24ff Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 15 May 2014 15:14:36 +0100 Subject: [PATCH 0124/1103] Tests passing --- celery/local.py | 4 ++-- celery/tests/utils/test_local.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/celery/local.py b/celery/local.py index e042fa0a0..1a10c2d8c 100644 --- a/celery/local.py +++ b/celery/local.py @@ -249,11 +249,10 @@ def __evaluate__(self, '_Proxy__kwargs')): try: thing = Proxy._get_current_object(self) - object.__setattr__(self, '__thing', thing) - return thing except: raise else: + object.__setattr__(self, '__thing', thing) for attr in _clean: try: object.__delattr__(self, attr) @@ -274,6 +273,7 @@ def __evaluate__(self, object.__delattr__(self, '__pending__') except AttributeError: pass + return thing def maybe_evaluate(obj): diff --git a/celery/tests/utils/test_local.py b/celery/tests/utils/test_local.py index 2b50efcda..67b44b221 100644 --- a/celery/tests/utils/test_local.py +++ b/celery/tests/utils/test_local.py @@ -341,6 +341,7 @@ def test_callbacks(self): self.assertTrue(object.__getattribute__(p, '__pending__')) self.assertTrue(repr(p)) + self.assertTrue(p.__evaluated__()) with self.assertRaises(AttributeError): object.__getattribute__(p, '__pending__') cbA.assert_called_with(p) From dc8e04a4d1fbe8549d8b4f002cbf96220167a90b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 15 May 2014 17:55:52 +0100 Subject: [PATCH 0125/1103] Tests passing --- celery/tests/case.py | 2 +- celery/tests/worker/test_loops.py | 2 +- celery/tests/worker/test_request.py | 3 ++- celery/tests/worker/test_strategy.py | 4 +++- celery/tests/worker/test_worker.py | 29 ++++++++++++++++++++++++++-- 5 files changed, 34 insertions(+), 6 deletions(-) diff --git a/celery/tests/case.py b/celery/tests/case.py index 520e1f55b..77a2dbc5d 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -867,7 +867,7 @@ def TaskMessage(name, id=None, args=(), kwargs={}, **options): message = Mock(name='TaskMessage-{0}'.format(id)) message.headers = { 'id': id, - 'c_type': name, + 'task': name, } message.headers.update(options) message.content_type, message.content_encoding, message.body = dumps( diff --git a/celery/tests/worker/test_loops.py b/celery/tests/worker/test_loops.py index 9673c5f7c..fd6c8046f 100644 --- a/celery/tests/worker/test_loops.py +++ b/celery/tests/worker/test_loops.py @@ -163,7 +163,7 @@ def test_on_task_not_registered(self): x, on_task, msg, strategy = self.task_context(self.add.s(2, 2)) exc = strategy.side_effect = KeyError(self.add.name) on_task(msg) - x.on_unknown_task.assert_called_with(None, msg, exc) + x.on_invalid_task.assert_called_with(None, msg, exc) def test_on_task_InvalidTaskError(self): x, on_task, msg, strategy = self.task_context(self.add.s(2, 2)) diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index 02f065df1..05aef9301 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -248,7 +248,8 @@ def test_invalid_expires_raises_InvalidTaskError(self): def test_valid_expires_with_utc_makes_aware(self): with patch('celery.worker.request.maybe_make_aware') as mma: - self.get_request(self.add.s(2, 2).set(expires=10)) + self.get_request(self.add.s(2, 2).set(expires=10), + maybe_make_aware=mma) self.assertTrue(mma.called) def test_maybe_expire_when_expires_is_None(self): diff --git a/celery/tests/worker/test_strategy.py b/celery/tests/worker/test_strategy.py index 87ae65006..6e34f3841 100644 --- a/celery/tests/worker/test_strategy.py +++ b/celery/tests/worker/test_strategy.py @@ -70,6 +70,7 @@ def _context(self, sig, if limit: bucket = TokenBucket(rate(limit), capacity=1) consumer.task_buckets[sig.task] = bucket + consumer.controller.state.revoked = set() consumer.disable_rate_limits = not rate_limits consumer.event_dispatcher.enabled = events s = sig.type.start_strategy(self.app, consumer, task_reserved=reserved) @@ -126,9 +127,10 @@ def test_when_rate_limited__limits_disabled(self): def test_when_revoked(self): task = self.add.s(2, 2) task.freeze() - state.revoked.add(task.id) try: with self._context(task) as C: + C.consumer.controller.state.revoked.add(task.id) + state.revoked.add(task.id) C() with self.assertRaises(ValueError): C.get_request() diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index 1596c4616..f3a3e1494 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -153,7 +153,7 @@ def test_info(self): l.connection = Mock() l.connection.info.return_value = {'foo': 'bar'} l.controller = l.app.WorkController() - l.controller.pool = Mock() + l.pool = l.controller.pool = Mock() l.controller.pool.info.return_value = [Mock(), Mock()] l.controller.consumer = l info = l.controller.stats() @@ -167,6 +167,8 @@ def test_start_when_closed(self): def test_connection(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) + l.controller = l.app.WorkController() + l.pool = l.controller.pool = Mock() l.blueprint.start(l) self.assertIsInstance(l.connection, Connection) @@ -229,6 +231,8 @@ def test_receive_message_unknown(self, warn): def test_receive_message_eta_OverflowError(self, to_timestamp): to_timestamp.side_effect = OverflowError() l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) + l.controller = l.app.WorkController() + l.pool = l.controller.pool = Mock() l.blueprint.state = RUN l.steps.pop() m = create_task_message( @@ -251,6 +255,8 @@ def test_receive_message_InvalidTaskError(self, error): l.blueprint.state = RUN l.event_dispatcher = mock_event_dispatcher() l.steps.pop() + l.controller = l.app.WorkController() + l.pool = l.controller.pool = Mock() m = create_task_message( Mock(), self.foo_task.name, args=(1, 2), kwargs='foobarbaz', id=1) @@ -293,6 +299,8 @@ def _get_on_message(self, l): def test_receieve_message(self): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) + l.controller = l.app.WorkController() + l.pool = l.controller.pool = Mock() l.blueprint.state = RUN l.event_dispatcher = mock_event_dispatcher() m = create_task_message( @@ -322,6 +330,8 @@ def loop(self, *args, **kwargs): l = MockConsumer(self.buffer.put, timer=self.timer, send_events=False, pool=BasePool(), app=self.app) + l.controller = l.app.WorkController() + l.pool = l.controller.pool = Mock() l.channel_errors = (KeyError, ) with self.assertRaises(KeyError): l.start() @@ -340,6 +350,8 @@ def loop(self, *args, **kwargs): l = MockConsumer(self.buffer.put, timer=self.timer, send_events=False, pool=BasePool(), app=self.app) + l.controller = l.app.WorkController() + l.pool = l.controller.pool = Mock() l.connection_errors = (KeyError, ) self.assertRaises(SyntaxError, l.start) @@ -422,6 +434,8 @@ def test_ignore_errors(self): def test_apply_eta_task(self): from celery.worker import state l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) + l.controller = l.app.WorkController() + l.pool = l.controller.pool = Mock() l.qos = QoS(None, 10) task = object() @@ -433,6 +447,8 @@ def test_apply_eta_task(self): def test_receieve_message_eta_isoformat(self): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) + l.controller = l.app.WorkController() + l.pool = l.controller.pool = Mock() l.blueprint.state = RUN l.steps.pop() m = create_task_message( @@ -518,6 +534,8 @@ def test_receieve_message_not_registered(self): @patch('celery.worker.consumer.logger') def test_receieve_message_ack_raises(self, logger, warn): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) + l.controller = l.app.WorkController() + l.pool = l.controller.pool = Mock() l.blueprint.state = RUN channel = Mock() m = create_task_message( @@ -550,6 +568,8 @@ def pp(*args, **kwargs): pp('TEST RECEIVE MESSAGE ETA') pp('+CREATE MYKOMBUCONSUMER') l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) + l.controller = l.app.WorkController() + l.pool = l.controller.pool = Mock() pp('-CREATE MYKOMBUCONSUMER') l.steps.pop() l.event_dispatcher = mock_event_dispatcher() @@ -732,6 +752,8 @@ def reset_connection(self): init_callback = Mock() l = _Consumer(self.buffer.put, timer=self.timer, init_callback=init_callback, app=self.app) + l.controller = l.app.WorkController() + l.pool = l.controller.pool = Mock() l.task_consumer = Mock() l.broadcast_consumer = Mock() l.qos = _QoS() @@ -754,6 +776,8 @@ def raises_KeyError(*args, **kwargs): init_callback.reset_mock() l = _Consumer(self.buffer.put, timer=self.timer, app=self.app, send_events=False, init_callback=init_callback) + l.controller = l.app.WorkController() + l.pool = l.controller.pool = Mock() l.qos = _QoS() l.task_consumer = Mock() l.broadcast_consumer = Mock() @@ -765,8 +789,9 @@ def raises_KeyError(*args, **kwargs): def test_reset_connection_with_no_node(self): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) + l.controller = l.app.WorkController() + l.pool = l.controller.pool = Mock() l.steps.pop() - self.assertEqual(None, l.pool) l.blueprint.start(l) From e71f86d66e105914feb881708cd317a28abf7113 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 15 May 2014 18:01:15 +0100 Subject: [PATCH 0126/1103] 3.2.0a2 --- celery/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/__init__.py b/celery/__init__.py index 86a3e450f..da44bf4ea 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -14,7 +14,7 @@ ) SERIES = 'DEV' -VERSION = version_info_t(3, 2, 0, 'a1', '') +VERSION = version_info_t(3, 2, 0, 'a2', '') __version__ = '{0.major}.{0.minor}.{0.micro}{0.releaselevel}'.format(VERSION) __author__ = 'Ask Solem' __contact__ = 'ask@celeryproject.org' From 5f8cfd78305b3056aaa6b42f068b60d8024a5dfe Mon Sep 17 00:00:00 2001 From: Alexey Kotlyarov Date: Fri, 16 May 2014 10:17:14 +1000 Subject: [PATCH 0127/1103] Make empty ResultSet support get() --- celery/result.py | 2 +- celery/tests/tasks/test_result.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/celery/result.py b/celery/result.py index eb7364a84..34943a76d 100644 --- a/celery/result.py +++ b/celery/result.py @@ -718,7 +718,7 @@ def subtasks(self): @property def supports_native_join(self): - return self.results[0].supports_native_join + return len(self.results) > 0 and self.results[0].supports_native_join @property def backend(self): diff --git a/celery/tests/tasks/test_result.py b/celery/tests/tasks/test_result.py index ee3c9bb1a..37692d00e 100644 --- a/celery/tests/tasks/test_result.py +++ b/celery/tests/tasks/test_result.py @@ -276,6 +276,12 @@ def test_get(self): x.get() self.assertTrue(x.join_native.called) + def test_get_empty(self): + x = self.app.ResultSet([]) + x.join = Mock() + x.get() + self.assertTrue(x.join.called) + def test_add(self): x = self.app.ResultSet([1]) x.add(2) From bd4dc6fb35d580b2a8da797beefbcf7e34a28b89 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 16 May 2014 14:07:17 +0100 Subject: [PATCH 0128/1103] Cosmetics for #2041 --- celery/result.py | 5 ++++- celery/tests/tasks/test_result.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/celery/result.py b/celery/result.py index 34943a76d..9bdd7d4f7 100644 --- a/celery/result.py +++ b/celery/result.py @@ -718,7 +718,10 @@ def subtasks(self): @property def supports_native_join(self): - return len(self.results) > 0 and self.results[0].supports_native_join + try: + return self.results[0].supports_native_join + except IndexError: + pass @property def backend(self): diff --git a/celery/tests/tasks/test_result.py b/celery/tests/tasks/test_result.py index 37692d00e..61bf09dfe 100644 --- a/celery/tests/tasks/test_result.py +++ b/celery/tests/tasks/test_result.py @@ -278,7 +278,8 @@ def test_get(self): def test_get_empty(self): x = self.app.ResultSet([]) - x.join = Mock() + self.assertIsNone(x.supports_native_join) + x.join = Mock(name='join') x.get() self.assertTrue(x.join.called) From 5a508638faf5dded9c9bb261090a3de6dbaf8bbd Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 May 2014 13:54:15 +0100 Subject: [PATCH 0129/1103] Fixes duplicate nodename warning. Closes #2046 --- celery/app/control.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/celery/app/control.py b/celery/app/control.py index 34076df0e..284537493 100644 --- a/celery/app/control.py +++ b/celery/app/control.py @@ -15,26 +15,27 @@ from kombu.utils import cached_property from celery.exceptions import DuplicateNodenameWarning +from celery.utils.text import pluralize __all__ = ['Inspect', 'Control', 'flatten_reply'] W_DUPNODE = """\ -Received multiple replies from node name {0!r}. +Received multiple replies from node name: {0!r}. Please make sure you give each node a unique nodename using the `-n` option.\ """ def flatten_reply(reply): - nodes = {} - seen = set() + nodes, dupes = {}, set() for item in reply: - dup = next((nodename in seen for nodename in item), None) - if dup: - warnings.warn(DuplicateNodenameWarning( - W_DUPNODE.format(dup), - )) - seen.update(item) + [dupes.add(name) for name in item if name in nodes] nodes.update(item) + if dupes: + warnings.warn(DuplicateNodenameWarning( + W_DUPNODE.format( + pluralize(len(dupes), 'name'), ', '.join(sorted(dupes)), + ), + )) return nodes From e08012f3ae00f3e94887390d651081cd1ea3a088 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 May 2014 14:15:10 +0100 Subject: [PATCH 0130/1103] Worker --detach default workdir is now CWD --- celery/bin/celeryd_detach.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/bin/celeryd_detach.py b/celery/bin/celeryd_detach.py index 12e1f6497..0e88d6052 100644 --- a/celery/bin/celeryd_detach.py +++ b/celery/bin/celeryd_detach.py @@ -30,7 +30,7 @@ C_FAKEFORK = os.environ.get('C_FAKEFORK') OPTION_LIST = daemon_options(default_pidfile='celeryd.pid') + ( - Option('--workdir', default='/', dest='working_directory'), + Option('--workdir', default=None, dest='working_directory'), Option('--fake', default=False, action='store_true', dest='fake', help="Don't fork (for debugging purposes)"), From b345094151dc85bdf35d2e8cf65214be9246a07e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 May 2014 14:15:47 +0100 Subject: [PATCH 0131/1103] Worker --detach with C_FAKEFORK no longer closes open fds. Closes #2044 --- celery/platforms.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/celery/platforms.py b/celery/platforms.py index c4013b578..11efd7bcd 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -317,9 +317,10 @@ def open(self): if self.after_chdir: self.after_chdir() - close_open_fds(self.stdfds) - for fd in self.stdfds: - self.redirect_to_null(maybe_fileno(fd)) + if not self.fake: + close_open_fds(self.stdfds) + for fd in self.stdfds: + self.redirect_to_null(maybe_fileno(fd)) self._is_open = True __enter__ = open From 5a4cf8b7d02618ae16e22812cd44ca1f5c576e2c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 May 2014 14:29:24 +0100 Subject: [PATCH 0132/1103] Did not log for Ignore and Reject --- celery/app/trace.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/celery/app/trace.py b/celery/app/trace.py index ec63de83f..d06ed9f48 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -150,6 +150,12 @@ def handle_error_state(self, task, eager=False): FAILURE: self.handle_failure, }[self.state](task, store_errors=store_errors) + def handle_reject(self, task, **kwargs): + self._log_error(task, ExceptionInfo()) + + def handle_ignore(self, task, **kwargs): + self._log_error(task, ExceptionInfo()) + def handle_retry(self, task, store_errors=True): """Handle retry exception.""" # the exception raised is the Retry semi-predicate, @@ -353,9 +359,11 @@ def trace_task(uuid, args, kwargs, request=None): except Reject as exc: I, R = Info(REJECTED, exc), ExceptionInfo(internal=True) state, retval = I.state, I.retval + I.handle_reject(task) except Ignore as exc: I, R = Info(IGNORED, exc), ExceptionInfo(internal=True) state, retval = I.state, I.retval + I.handle_ignore(task) except Retry as exc: I, R, state, retval = on_error( task_request, exc, uuid, RETRY, call_errbacks=False, From 8c56d527857fb05e8517251a3d1eeb4ecbd56fc4 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 May 2014 14:45:01 +0100 Subject: [PATCH 0133/1103] Events are now transient by default --- celery/events/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/celery/events/__init__.py b/celery/events/__init__.py index b4ca9045c..a4142f76a 100644 --- a/celery/events/__init__.py +++ b/celery/events/__init__.py @@ -124,7 +124,7 @@ class EventDispatcher(object): def __init__(self, connection=None, hostname=None, enabled=True, channel=None, buffer_while_offline=True, app=None, - serializer=None, groups=None): + serializer=None, groups=None, delivery_mode=1): self.app = app_or_default(app or self.app) self.connection = connection self.channel = channel @@ -139,6 +139,7 @@ def __init__(self, connection=None, hostname=None, enabled=True, self.groups = set(groups or []) self.tzoffset = [-time.timezone, -time.altzone] self.clock = self.app.clock + self.delivery_mode = delivery_mode if not connection and channel: self.connection = channel.connection.client self.enabled = enabled @@ -213,6 +214,7 @@ def publish(self, type, fields, producer, retry=False, declare=[exchange], serializer=self.serializer, headers=self.headers, + delivery_mode=self.delivery_mode, ) def send(self, type, blind=False, **fields): From ee0016bf1f024dc880ea2eb108ea25855010e2db Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 May 2014 14:50:25 +0100 Subject: [PATCH 0134/1103] Stupid zsh is messing with our beloved ksh heritage :chicken: Closes #2038 --- docs/includes/installation.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/includes/installation.txt b/docs/includes/installation.txt index 54ec954b0..ffc9edef0 100644 --- a/docs/includes/installation.txt +++ b/docs/includes/installation.txt @@ -28,9 +28,9 @@ commas. .. code-block:: bash - $ pip install celery[librabbitmq] + $ pip install "celery[librabbitmq]" - $ pip install celery[librabbitmq,redis,auth,msgpack] + $ pip install "celery[librabbitmq,redis,auth,msgpack]" The following bundles are available: From 961429539e620180a7c457db160735fcd6d6dad7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 May 2014 16:23:02 +0100 Subject: [PATCH 0135/1103] Task proto2: Custom headers were ignored --- celery/app/amqp.py | 10 ++++++---- docs/userguide/signals.rst | 27 ++++++++++++++++++++------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 84ced793c..4a4fbc15c 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -417,7 +417,9 @@ def publish_task(producer, name, message, compression=None, declare=None, headers=None, **kwargs): retry = default_retry if retry is None else retry - headers, properties, body, sent_event = message + headers2, properties, body, sent_event = message + if headers: + headers2.update(headers) if kwargs: properties.update(kwargs) @@ -448,7 +450,7 @@ def publish_task(producer, name, message, send_before_publish( sender=name, body=body, exchange=exchange, routing_key=routing_key, - declare=declare, headers=headers, + declare=declare, headers=headers2, properties=kwargs, retry_policy=retry_policy, ) ret = producer.publish( @@ -459,11 +461,11 @@ def publish_task(producer, name, message, compression=compression or default_compressor, retry=retry, retry_policy=_rp, delivery_mode=delivery_mode, declare=declare, - headers=headers, + headers=headers2, **properties ) if after_receivers: - send_after_publish(sender=name, body=body, + send_after_publish(sender=name, body=body, headers=headers2, exchange=exchange, routing_key=routing_key) if sent_receivers: # XXX deprecated send_task_sent(sender=name, task_id=body['id'], task=name, diff --git a/docs/userguide/signals.rst b/docs/userguide/signals.rst index 7b927472b..00dab2dd9 100644 --- a/docs/userguide/signals.rst +++ b/docs/userguide/signals.rst @@ -28,9 +28,12 @@ Example connecting to the :signal:`after_task_publish` signal: from celery.signals import after_task_publish @after_task_publish.connect - def task_sent_handler(sender=None, body=None, **kwargs): - print('after_task_publish for task id {body[id]}'.format( - body=body, + def task_sent_handler(sender=None, headers=None, body=None, **kwargs): + # information about task are located in headers for task messages + # using the task protocol version 2. + info = headers if 'task' in headers else body + print('after_task_publish for task id {info[id]}'.format( + info=info, )) @@ -44,9 +47,12 @@ is published: .. code-block:: python @after_task_publish.connect(sender='proj.tasks.add') - def task_sent_handler(sender=None, body=None, **kwargs): - print('after_task_publish for task id {body[id]}'.format( - body=body, + def task_sent_handler(sender=None, headers=None, body=None, **kwargs): + # information about task are located in headers for task messages + # using the task protocol version 2. + info = headers if 'task' in headers else body + print('after_task_publish for task id {info[id]}'.format( + info=info, )) Signals use the same implementation as django.core.dispatch. As a result other @@ -123,9 +129,16 @@ Sender is the name of the task being sent. Provides arguments: +* headers + + The task message headers, see :ref:`task-message-protocol-v2` + and :ref:`task-message-protocol-v1`. + for a reference of possible fields that can be defined. + * body - The task message body, see :ref:`task-message-protocol-v1` + The task message body, see :ref:`task-message-protocol-v2` + and :ref:`task-message-protocol-v1`. for a reference of possible fields that can be defined. * exchange From fbbef2723ad4481f4a80e6bd72a03d3836ebf2b8 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 May 2014 16:24:08 +0100 Subject: [PATCH 0136/1103] Canvas: Makes sure group() in a workflow is not applied as a "celery.group" task --- celery/canvas.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 2c2883bcf..ae98fa4e1 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -570,9 +570,19 @@ def from_dict(self, d, app=None): task['args'] = task._merge(d['args'])[0] return group(tasks, app=app, **kwdict(d['options'])) - def _prepared(self, tasks, partial_args, group_id, root_id): + def _prepared(self, tasks, partial_args, group_id, root_id, + dict=dict, Signature=Signature, from_dict=Signature.from_dict): for task in tasks: - task = task.clone(partial_args) + if isinstance(task, dict): + if isinstance(task, Signature): + # local sigs are always of type Signature, and we + # clone them to make sure we do not modify the originals. + task = task.clone() + else: + # serialized sigs must be converted to Signature. + task = from_dict(task) + if partial_args and not task.immutable: + task.args = tuple(partial_args) + tuple(task.args) yield task, task.freeze(group_id=group_id, root_id=root_id) def _apply_tasks(self, tasks, producer=None, app=None, **options): @@ -792,7 +802,7 @@ def __repr__(self): def signature(varies, *args, **kwargs): - if not (args or kwargs) and isinstance(varies, dict): + if isinstance(varies, dict): if isinstance(varies, Signature): return varies.clone() return Signature.from_dict(varies) @@ -804,9 +814,10 @@ def maybe_signature(d, app=None): if d is not None: if isinstance(d, dict): if not isinstance(d, Signature): - return signature(d, app=app) + d = signature(d) elif isinstance(d, list): return [maybe_signature(s, app=app) for s in d] + if app is not None: d._app = app return d From 6ed09aaccf6332e90251228638683ce6a92f5026 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 May 2014 18:53:37 +0100 Subject: [PATCH 0137/1103] Import print_function in every module using print --- celery/__init__.py | 2 +- celery/__main__.py | 2 +- celery/app/__init__.py | 2 +- celery/apps/beat.py | 2 +- celery/events/snapshot.py | 2 +- celery/tests/worker/test_request.py | 1 - celery/utils/timer2.py | 2 +- celery/utils/timeutils.py | 2 +- celery/worker/control.py | 2 +- celery/worker/state.py | 2 +- 10 files changed, 9 insertions(+), 10 deletions(-) diff --git a/celery/__init__.py b/celery/__init__.py index da44bf4ea..6ec3a4e77 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -5,7 +5,7 @@ # :copyright: (c) 2012-2013 GoPivotal, Inc., All rights reserved. # :license: BSD (3 Clause), see LICENSE for more details. -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals from collections import namedtuple diff --git a/celery/__main__.py b/celery/__main__.py index 04448e239..ba98e0a8a 100644 --- a/celery/__main__.py +++ b/celery/__main__.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function import sys diff --git a/celery/app/__init__.py b/celery/app/__init__.py index 952a8746d..3c690fb41 100644 --- a/celery/app/__init__.py +++ b/celery/app/__init__.py @@ -6,7 +6,7 @@ Celery Application. """ -from __future__ import absolute_import +from __future__ import absolute_import, print_function import os diff --git a/celery/apps/beat.py b/celery/apps/beat.py index 46cef9b8b..a3c278c86 100644 --- a/celery/apps/beat.py +++ b/celery/apps/beat.py @@ -10,7 +10,7 @@ and so on. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_statement, unicode_literals import numbers import socket diff --git a/celery/events/snapshot.py b/celery/events/snapshot.py index 0dd41554c..a202a70f3 100644 --- a/celery/events/snapshot.py +++ b/celery/events/snapshot.py @@ -10,7 +10,7 @@ in :mod:`djcelery.snapshots` in the `django-celery` distribution. """ -from __future__ import absolute_import +from __future__ import absolute_import, print_function from kombu.utils.limits import TokenBucket diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index 05aef9301..2700d26ef 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -805,7 +805,6 @@ def test_execute_fail(self): args=[4], kwargs={}, ) - print(job.execute()) self.assertIsInstance(job.execute(), ExceptionInfo) meta = self.mytask_raising.backend.get_task_meta(tid) self.assertEqual(meta['status'], states.FAILURE) diff --git a/celery/utils/timer2.py b/celery/utils/timer2.py index e42660c23..fdac90803 100644 --- a/celery/utils/timer2.py +++ b/celery/utils/timer2.py @@ -6,7 +6,7 @@ Scheduler for Python functions. """ -from __future__ import absolute_import +from __future__ import absolute_import, print_function import os import sys diff --git a/celery/utils/timeutils.py b/celery/utils/timeutils.py index 5b75b83a8..bd0b066e6 100644 --- a/celery/utils/timeutils.py +++ b/celery/utils/timeutils.py @@ -6,7 +6,7 @@ This module contains various utilities related to dates and times. """ -from __future__ import absolute_import +from __future__ import absolute_import, print_function import numbers import os diff --git a/celery/worker/control.py b/celery/worker/control.py index b0bb93fde..d0b119d85 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -227,7 +227,7 @@ def objgraph(state, num=200, max_depth=10, type='Request'): # pragma: no cover import objgraph except ImportError: raise ImportError('Requires the objgraph library') - print('Dumping graph for type %r' % (type, )) + logger.info('Dumping graph for type %r', type) with tempfile.NamedTemporaryFile(prefix='cobjg', suffix='.png', delete=False) as fh: objects = objgraph.by_type(type)[:num] diff --git a/celery/worker/state.py b/celery/worker/state.py index f2f7a79fa..3e1ab95c5 100644 --- a/celery/worker/state.py +++ b/celery/worker/state.py @@ -9,7 +9,7 @@ statistics, and revoked tasks. """ -from __future__ import absolute_import +from __future__ import absolute_import, print_function import os import sys From 90c0ee87456aa5cc7381775e6a3f6f3ed7f39c2b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 May 2014 19:15:08 +0100 Subject: [PATCH 0138/1103] flakes --- celery/canvas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index ae98fa4e1..f419cabb3 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -570,8 +570,8 @@ def from_dict(self, d, app=None): task['args'] = task._merge(d['args'])[0] return group(tasks, app=app, **kwdict(d['options'])) - def _prepared(self, tasks, partial_args, group_id, root_id, - dict=dict, Signature=Signature, from_dict=Signature.from_dict): + def _prepared(self, tasks, partial_args, group_id, root_id, dict=dict, + Signature=Signature, from_dict=Signature.from_dict): for task in tasks: if isinstance(task, dict): if isinstance(task, Signature): From d6ae1a2be401a3bfd53cd1e68ee0a1226f8848e3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 May 2014 20:04:22 +0100 Subject: [PATCH 0139/1103] Fix test for working_directory --- celery/tests/bin/test_celeryd_detach.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/tests/bin/test_celeryd_detach.py b/celery/tests/bin/test_celeryd_detach.py index 000d2f633..964aa0b77 100644 --- a/celery/tests/bin/test_celeryd_detach.py +++ b/celery/tests/bin/test_celeryd_detach.py @@ -85,7 +85,7 @@ def test_execute_from_commandline(self, detach, exit): detach.assert_called_with( path=x.execv_path, uid=None, gid=None, umask=0, fake=False, logfile='/var/log', pidfile='celeryd.pid', - working_directory='/', + working_directory=None, argv=x.execv_argv + [ '-c', '1', '-lDEBUG', '--logfile=/var/log', '--pidfile=celeryd.pid', From b6e49175ce4df301bcb2c97b1bfe5e2d3a535245 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 May 2014 20:23:37 +0100 Subject: [PATCH 0140/1103] beat --detached now runs after forkers --- celery/bin/celeryd_detach.py | 3 ++- celery/platforms.py | 7 ++++++- celery/tests/bin/test_celeryd_detach.py | 6 ++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/celery/bin/celeryd_detach.py b/celery/bin/celeryd_detach.py index 0e88d6052..8a6d339d4 100644 --- a/celery/bin/celeryd_detach.py +++ b/celery/bin/celeryd_detach.py @@ -40,7 +40,8 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, gid=None, umask=0, working_directory=None, fake=False, app=None): fake = 1 if C_FAKEFORK else fake - with detached(logfile, pidfile, uid, gid, umask, working_directory, fake): + with detached(logfile, pidfile, uid, gid, umask, working_directory, fake, + after_forkers=False): try: os.execv(path, [path] + argv) except Exception: diff --git a/celery/platforms.py b/celery/platforms.py index 11efd7bcd..651b8f5fa 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -36,6 +36,7 @@ resource = try_import('resource') pwd = try_import('pwd') grp = try_import('grp') +mputil = try_import('multiprocessing.util') __all__ = ['EX_OK', 'EX_FAILURE', 'EX_UNAVAILABLE', 'EX_USAGE', 'SYSTEM', 'IS_OSX', 'IS_WINDOWS', 'pyimplementation', 'LockFailed', @@ -294,11 +295,13 @@ class DaemonContext(object): _is_open = False def __init__(self, pidfile=None, workdir=None, umask=None, - fake=False, after_chdir=None, **kwargs): + fake=False, after_chdir=None, after_forkers=True, + **kwargs): self.workdir = workdir or DAEMON_WORKDIR self.umask = DAEMON_UMASK if umask is None else umask self.fake = fake self.after_chdir = after_chdir + self.after_forkers = after_forkers self.stdfds = (sys.stdin, sys.stdout, sys.stderr) def redirect_to_null(self, fd): @@ -321,6 +324,8 @@ def open(self): close_open_fds(self.stdfds) for fd in self.stdfds: self.redirect_to_null(maybe_fileno(fd)) + if self.after_forkers and mputil is not None: + mputil._run_after_forkers() self._is_open = True __enter__ = open diff --git a/celery/tests/bin/test_celeryd_detach.py b/celery/tests/bin/test_celeryd_detach.py index 964aa0b77..9e1b4a04d 100644 --- a/celery/tests/bin/test_celeryd_detach.py +++ b/celery/tests/bin/test_celeryd_detach.py @@ -24,8 +24,10 @@ def test_execs(self, setup_logs, logger, execv, detached): detach('/bin/boo', ['a', 'b', 'c'], logfile='/var/log', pidfile='/var/pid') - detached.assert_called_with('/var/log', '/var/pid', None, None, 0, - None, False) + detached.assert_called_with( + '/var/log', '/var/pid', None, None, 0, None, False, + after_forkers=False, + ) execv.assert_called_with('/bin/boo', ['/bin/boo', 'a', 'b', 'c']) execv.side_effect = Exception('foo') From 9b8e2cbd50086020abce1cb7157e8825f6778a79 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 May 2014 20:47:30 +0100 Subject: [PATCH 0141/1103] Fixes typo --- celery/apps/beat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/apps/beat.py b/celery/apps/beat.py index a3c278c86..97fa4829a 100644 --- a/celery/apps/beat.py +++ b/celery/apps/beat.py @@ -10,7 +10,7 @@ and so on. """ -from __future__ import absolute_import, print_statement, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import numbers import socket From 3ff387099dac98bc78d20f79b168ecc03f78327a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 May 2014 21:46:12 +0100 Subject: [PATCH 0142/1103] Remove use of kwdict workaround fixed in py2.7 --- celery/canvas.py | 12 ++++++------ celery/utils/__init__.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index f419cabb3..f7ba6d1cd 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -18,7 +18,7 @@ from operator import itemgetter from itertools import chain as _chain -from kombu.utils import cached_property, fxrange, kwdict, reprcall, uuid +from kombu.utils import cached_property, fxrange, reprcall, uuid from celery._state import current_app, get_current_worker_task from celery.utils.functional import ( @@ -133,7 +133,7 @@ def register_type(cls, subclass, name=None): def from_dict(self, d, app=None): typ = d.get('subtask_type') if typ: - return self.TYPES[typ].from_dict(kwdict(d), app=app) + return self.TYPES[typ].from_dict(d, app=app) return Signature(d, app=app) def __init__(self, task=None, args=None, kwargs=None, options=None, @@ -449,7 +449,7 @@ def from_dict(self, d, app=None): if d['args'] and tasks: # partial args passed on to first task in chain (Issue #1057). tasks[0]['args'] = tasks[0]._merge(d['args'])[0] - return chain(*d['kwargs']['tasks'], app=app, **kwdict(d['options'])) + return chain(*d['kwargs']['tasks'], app=app, **d['options']) @property def app(self): @@ -568,7 +568,7 @@ def from_dict(self, d, app=None): # partial args passed on to all tasks in the group (Issue #1057). for task in tasks: task['args'] = task._merge(d['args'])[0] - return group(tasks, app=app, **kwdict(d['options'])) + return group(tasks, app=app, **d['options']) def _prepared(self, tasks, partial_args, group_id, root_id, dict=dict, Signature=Signature, from_dict=Signature.from_dict): @@ -707,8 +707,8 @@ def freeze(self, *args, **kwargs): @classmethod def from_dict(self, d, app=None): - args, d['kwargs'] = self._unpack_args(**kwdict(d['kwargs'])) - return self(*args, app=app, **kwdict(d)) + args, d['kwargs'] = self._unpack_args(**d['kwargs']) + return self(*args, app=app, **d) @staticmethod def _unpack_args(header=None, body=None, **kwargs): diff --git a/celery/utils/__init__.py b/celery/utils/__init__.py index a3264b8d6..2e31c9f68 100644 --- a/celery/utils/__init__.py +++ b/celery/utils/__init__.py @@ -386,5 +386,5 @@ def resolve(match): instantiate, import_from_cwd ) from .functional import chunks, noop # noqa -from kombu.utils import cached_property, kwdict, uuid # noqa +from kombu.utils import cached_property, uuid # noqa gen_unique_id = uuid From 6ac362660a5ada7374ef2c8eda8e104b307a386f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 May 2014 22:04:29 +0100 Subject: [PATCH 0143/1103] Now imports OrderedDict directly from collections --- celery/bin/multi.py | 3 +-- celery/five.py | 9 ++------- celery/result.py | 3 +-- celery/utils/functional.py | 2 +- funtests/stress/stress/suite.py | 4 +--- 5 files changed, 6 insertions(+), 15 deletions(-) diff --git a/celery/bin/multi.py b/celery/bin/multi.py index 2f2055ece..7bb90cf92 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -103,13 +103,12 @@ import socket import sys -from collections import defaultdict, namedtuple +from collections import OrderedDict, defaultdict, namedtuple from functools import partial from subprocess import Popen from time import sleep from kombu.utils import cached_property -from kombu.utils.compat import OrderedDict from kombu.utils.encoding import from_utf8 from celery import VERSION_BANNER diff --git a/celery/five.py b/celery/five.py index bfe9ff0cd..5a272c9d6 100644 --- a/celery/five.py +++ b/celery/five.py @@ -14,9 +14,8 @@ 'zip_longest', 'map', 'string', 'string_t', 'long_t', 'text_t', 'range', 'int_types', 'items', 'keys', 'values', 'nextfun', 'reraise', 'WhateverIO', 'with_metaclass', - 'OrderedDict', 'THREAD_TIMEOUT_MAX', 'format_d', - 'class_property', 'reclassmethod', 'create_module', - 'recreate_module', 'monotonic'] + 'THREAD_TIMEOUT_MAX', 'format_d', 'class_property', 'reclassmethod', + 'create_module', 'recreate_module', 'monotonic'] import io @@ -152,10 +151,6 @@ def _clone_with_metaclass(Class): return _clone_with_metaclass -############## collections.OrderedDict ###################################### -# was moved to kombu -from kombu.utils.compat import OrderedDict # noqa - ############## threading.TIMEOUT_MAX ####################################### try: from threading import TIMEOUT_MAX as THREAD_TIMEOUT_MAX diff --git a/celery/result.py b/celery/result.py index 9bdd7d4f7..cab76bab4 100644 --- a/celery/result.py +++ b/celery/result.py @@ -11,12 +11,11 @@ import time import warnings -from collections import deque +from collections import OrderedDict, deque from contextlib import contextmanager from copy import copy from kombu.utils import cached_property -from kombu.utils.compat import OrderedDict from . import current_app from . import states diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 8903ff08d..be7a2289c 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -11,12 +11,12 @@ import sys import threading +from collections import OrderedDict from functools import wraps from itertools import islice from kombu.utils import cached_property from kombu.utils.functional import lazy, maybe_evaluate, is_list, maybe_list -from kombu.utils.compat import OrderedDict from celery.five import UserDict, UserList, items, keys diff --git a/funtests/stress/stress/suite.py b/funtests/stress/stress/suite.py index 10d56f4cf..134850409 100755 --- a/funtests/stress/stress/suite.py +++ b/funtests/stress/stress/suite.py @@ -6,12 +6,10 @@ import socket import sys -from collections import defaultdict, namedtuple +from collections import OrderedDict, defaultdict, namedtuple from itertools import count from time import sleep -from kombu.utils.compat import OrderedDict - from celery import group, VERSION_BANNER from celery.exceptions import TimeoutError from celery.five import items, monotonic, range, values From 43ef0321058f318310cb0abd994b82047a25751e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 May 2014 15:59:52 +0100 Subject: [PATCH 0144/1103] unicode literals --- celery/__main__.py | 2 +- celery/_state.py | 2 +- celery/app/__init__.py | 2 +- celery/backends/base.py | 3 +-- celery/backends/cassandra.py | 9 ++++----- celery/contrib/rdb.py | 2 +- celery/events/dumper.py | 2 +- celery/schedules.py | 12 ++++++------ celery/tests/backends/test_redis.py | 7 ++++--- celery/tests/compat_modules/test_compat.py | 6 +++--- celery/tests/utils/test_timeutils.py | 14 -------------- celery/utils/timeutils.py | 5 ++--- docs/internals/app-overview.rst | 20 +------------------- extra/release/verify_config_reference.py | 5 +++-- funtests/benchmarks/bench_worker.py | 2 +- funtests/stress/stress/__main__.py | 2 +- funtests/stress/stress/fbi.py | 2 +- funtests/suite/test_leak.py | 4 ++-- pavement.py | 2 +- 19 files changed, 35 insertions(+), 68 deletions(-) diff --git a/celery/__main__.py b/celery/__main__.py index ba98e0a8a..572f7c3c9 100644 --- a/celery/__main__.py +++ b/celery/__main__.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import sys diff --git a/celery/_state.py b/celery/_state.py index 7592ca242..9ed62b89d 100644 --- a/celery/_state.py +++ b/celery/_state.py @@ -9,7 +9,7 @@ This module shouldn't be used directly. """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import os import sys diff --git a/celery/app/__init__.py b/celery/app/__init__.py index 3c690fb41..22a9700ae 100644 --- a/celery/app/__init__.py +++ b/celery/app/__init__.py @@ -6,7 +6,7 @@ Celery Application. """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import os diff --git a/celery/backends/base.py b/celery/backends/base.py index aec6dd3f4..996171abf 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -33,7 +33,6 @@ from celery.result import ( GroupResult, ResultBase, allow_join_result, result_from_tuple, ) -from celery.utils import timeutils from celery.utils.functional import LRUCache from celery.utils.log import get_logger from celery.utils.serialization import ( @@ -226,7 +225,7 @@ def prepare_expires(self, value, type=None): if value is None: value = self.app.conf.CELERY_TASK_RESULT_EXPIRES if isinstance(value, timedelta): - value = timeutils.timedelta_seconds(value) + value = value.total_seconds() if value is not None and type: return type(value) return value diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index 774e6b792..adb70afc7 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -22,7 +22,7 @@ from celery.exceptions import ImproperlyConfigured from celery.five import monotonic from celery.utils.log import get_logger -from celery.utils.timeutils import maybe_timedelta, timedelta_seconds +from celery.utils.timeutils import maybe_timedelta from .base import BaseBackend @@ -148,14 +148,13 @@ def _do_store(): 'children': self.encode( self.current_task_children(request), )} + ttl = self.expires and max(self.expires.total_seconds(), 0) if self.detailed_mode: meta['result'] = result - cf.insert(task_id, {date_done: self.encode(meta)}, - ttl=self.expires and timedelta_seconds(self.expires)) + cf.insert(task_id, {date_done: self.encode(meta)}, ttl=ttl) else: meta['result'] = self.encode(result) - cf.insert(task_id, meta, - ttl=self.expires and timedelta_seconds(self.expires)) + cf.insert(task_id, meta, ttl=ttl) return self._retry_on_error(_do_store) diff --git a/celery/contrib/rdb.py b/celery/contrib/rdb.py index 3e9f55bba..8435ec31b 100644 --- a/celery/contrib/rdb.py +++ b/celery/contrib/rdb.py @@ -34,7 +34,7 @@ def add(x, y): base port. The selected port will be logged by the worker. """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import errno import os diff --git a/celery/events/dumper.py b/celery/events/dumper.py index 323afc4e1..3c20186e6 100644 --- a/celery/events/dumper.py +++ b/celery/events/dumper.py @@ -7,7 +7,7 @@ as they happen. Think of it like a `tcpdump` for Celery events. """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import sys diff --git a/celery/schedules.py b/celery/schedules.py index 18cf48190..be6832151 100644 --- a/celery/schedules.py +++ b/celery/schedules.py @@ -21,8 +21,8 @@ from .five import range, string_t from .utils import is_iterable from .utils.timeutils import ( - timedelta_seconds, weekday, maybe_timedelta, remaining, - humanize_seconds, timezone, maybe_make_aware, ffwd + weekday, maybe_timedelta, remaining, humanize_seconds, + timezone, maybe_make_aware, ffwd ) from .datastructures import AttributeDict @@ -116,7 +116,7 @@ def is_due(self, last_run_at): """ last_run_at = self.maybe_make_aware(last_run_at) rem_delta = self.remaining_estimate(last_run_at) - remaining_s = timedelta_seconds(rem_delta) + remaining_s = max(rem_delta.total_seconds(), 0) if remaining_s == 0: return schedstate(is_due=True, next=self.seconds) return schedstate(is_due=False, next=remaining_s) @@ -142,7 +142,7 @@ def __reduce__(self): @property def seconds(self): - return timedelta_seconds(self.run_every) + return max(self.run_every.total_seconds(), 0) @property def human_seconds(self): @@ -562,11 +562,11 @@ def is_due(self, last_run_at): """ rem_delta = self.remaining_estimate(last_run_at) - rem = timedelta_seconds(rem_delta) + rem = max(rem_delta.total_seconds(), 0) due = rem == 0 if due: rem_delta = self.remaining_estimate(self.now()) - rem = timedelta_seconds(rem_delta) + rem = max(rem_delta.total_seconds(), 0) return schedstate(due, rem) def __eq__(self, other): diff --git a/celery/tests/backends/test_redis.py b/celery/tests/backends/test_redis.py index 0ecc5258b..ad8b50fc7 100644 --- a/celery/tests/backends/test_redis.py +++ b/celery/tests/backends/test_redis.py @@ -10,7 +10,6 @@ from celery import uuid from celery.datastructures import AttributeDict from celery.exceptions import ImproperlyConfigured -from celery.utils.timeutils import timedelta_seconds from celery.tests.case import ( AppCase, Mock, MockCallbacks, SkipTest, depends_on_current_app, patch, @@ -202,8 +201,10 @@ def test_default_is_old_join(self): def test_expires_is_None(self): b = self.Backend(expires=None, app=self.app, new_join=True) - self.assertEqual(b.expires, timedelta_seconds( - self.app.conf.CELERY_TASK_RESULT_EXPIRES)) + self.assertEqual( + b.expires, + self.app.conf.CELERY_TASK_RESULT_EXPIRES.total_seconds(), + ) def test_expires_is_timedelta(self): b = self.Backend( diff --git a/celery/tests/compat_modules/test_compat.py b/celery/tests/compat_modules/test_compat.py index 58f0cea0c..aa7be5dd4 100644 --- a/celery/tests/compat_modules/test_compat.py +++ b/celery/tests/compat_modules/test_compat.py @@ -10,7 +10,6 @@ periodic_task, PeriodicTask ) -from celery.utils.timeutils import timedelta_seconds from celery.tests.case import AppCase, depends_on_current_app @@ -51,8 +50,9 @@ def test_is_due(self): self.now() - p.run_every.run_every, ) self.assertTrue(due) - self.assertEqual(remaining, - timedelta_seconds(p.run_every.run_every)) + self.assertEqual( + remaining, p.run_every.run_every.total_seconds(), + ) def test_schedule_repr(self): p = self.my_periodic diff --git a/celery/tests/utils/test_timeutils.py b/celery/tests/utils/test_timeutils.py index 2258d064d..f72794017 100644 --- a/celery/tests/utils/test_timeutils.py +++ b/celery/tests/utils/test_timeutils.py @@ -10,7 +10,6 @@ humanize_seconds, maybe_iso8601, maybe_timedelta, - timedelta_seconds, timezone, rate, remaining, @@ -83,19 +82,6 @@ def test_delta_resolution(self): for delta, shoulda in deltamap: self.assertEqual(D(dt, delta), shoulda) - def test_timedelta_seconds(self): - deltamap = ((timedelta(seconds=1), 1), - (timedelta(seconds=27), 27), - (timedelta(minutes=3), 3 * 60), - (timedelta(hours=4), 4 * 60 * 60), - (timedelta(days=3), 3 * 86400)) - for delta, seconds in deltamap: - self.assertEqual(timedelta_seconds(delta), seconds) - - def test_timedelta_seconds_returns_0_on_negative_time(self): - delta = timedelta(days=-2) - self.assertEqual(timedelta_seconds(delta), 0) - def test_humanize_seconds(self): t = ((4 * 60 * 60 * 24, '4.00 days'), (1 * 60 * 60 * 24, '1.00 day'), diff --git a/celery/utils/timeutils.py b/celery/utils/timeutils.py index bd0b066e6..453be35de 100644 --- a/celery/utils/timeutils.py +++ b/celery/utils/timeutils.py @@ -16,7 +16,6 @@ from datetime import date, datetime, timedelta, tzinfo from kombu.utils import cached_property, reprcall -from kombu.utils.compat import timedelta_seconds from pytz import timezone as _timezone, AmbiguousTimeError @@ -26,7 +25,7 @@ from .iso8601 import parse_iso8601 from .text import pluralize -__all__ = ['LocalTimezone', 'timezone', 'maybe_timedelta', 'timedelta_seconds', +__all__ = ['LocalTimezone', 'timezone', 'maybe_timedelta', 'delta_resolution', 'remaining', 'rate', 'weekday', 'humanize_seconds', 'maybe_iso8601', 'is_naive', 'make_aware', 'localize', 'to_utc', 'maybe_make_aware', 'ffwd', 'utcoffset', @@ -149,7 +148,7 @@ def delta_resolution(dt, delta): which will just return the original datetime. """ - delta = timedelta_seconds(delta) + delta = max(delta.total_seconds(), 0) resolutions = ((3, lambda x: x / 86400), (4, lambda x: x / 3600), diff --git a/docs/internals/app-overview.rst b/docs/internals/app-overview.rst index 602f33d29..0213ac91a 100644 --- a/docs/internals/app-overview.rst +++ b/docs/internals/app-overview.rst @@ -98,29 +98,11 @@ Deprecations Inferior to the ping remote control command. Will be removed in Celery 2.3. -Removed deprecations -==================== - -* `celery.utils.timedelta_seconds` - Use: :func:`celery.utils.timeutils.timedelta_seconds` - -* `celery.utils.defaultdict` - Use: :func:`celery.utils.compat.defaultdict` - -* `celery.utils.all` - Use: :func:`celery.utils.compat.all` - -* `celery.task.apply_async` - Use app.send_task - -* `celery.task.tasks` - Use :data:`celery.registry.tasks` - Aliases (Pending deprecation) ============================= * celery.task.base - * .Task -> {app.create_task_cls} + * .Task -> {app.Task / :class:`celery.app.task.Task`} * celery.task.sets * .TaskSet -> {app.TaskSet} diff --git a/extra/release/verify_config_reference.py b/extra/release/verify_config_reference.py index c36af7a20..8f4038cc4 100644 --- a/extra/release/verify_config_reference.py +++ b/extra/release/verify_config_reference.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import print_function, unicode_literals from fileinput import input as _input from sys import exit, stderr @@ -27,8 +27,9 @@ def is_ignored(setting, option): def find_undocumented_settings(directive='.. setting:: '): settings = dict(flatten(NAMESPACES)) all = set(settings) + inp = (l.decode('utf-8') for l in _input()) documented = set( - line.strip()[len(directive):].strip() for line in _input() + line.strip()[len(directive):].strip() for line in inp if line.strip().startswith(directive) ) return [setting for setting in all ^ documented diff --git a/funtests/benchmarks/bench_worker.py b/funtests/benchmarks/bench_worker.py index 8de8a3af6..8663e8d7a 100644 --- a/funtests/benchmarks/bench_worker.py +++ b/funtests/benchmarks/bench_worker.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import print_function, unicode_literals import os import sys diff --git a/funtests/stress/stress/__main__.py b/funtests/stress/stress/__main__.py index 6b6f6ceda..f83c8c192 100644 --- a/funtests/stress/stress/__main__.py +++ b/funtests/stress/stress/__main__.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals from celery.bin.base import Command, Option diff --git a/funtests/stress/stress/fbi.py b/funtests/stress/stress/fbi.py index 9b06af654..f0b40fdcf 100644 --- a/funtests/stress/stress/fbi.py +++ b/funtests/stress/stress/fbi.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import socket import sys diff --git a/funtests/suite/test_leak.py b/funtests/suite/test_leak.py index bd23c45c1..b19c23f41 100644 --- a/funtests/suite/test_leak.py +++ b/funtests/suite/test_leak.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import print_function, unicode_literals import gc import os @@ -15,7 +15,7 @@ import suite # noqa -GET_RSIZE = '/bin/ps -p {pid} -o rss=' +GET_RSIZE = b'/bin/ps -p {pid} -o rss=' class Sizes(list): diff --git a/pavement.py b/pavement.py index 7b077cc27..dd7916acd 100644 --- a/pavement.py +++ b/pavement.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import print_function, unicode_literals import sys import traceback From 04ddf8ea88a39448185effbf01ba186f88d27dc2 Mon Sep 17 00:00:00 2001 From: Roger Hu Date: Thu, 8 May 2014 01:33:47 +0000 Subject: [PATCH 0145/1103] Be more selective about how file descriptors get removed from Kombu's hub. Given that file descriptor changes appear to get triggered on the Kombu side, these changes may not make a material impact. However, to make things more consistent with the changes introduced in https://github.com/celery/kombu/pull/353, the changes have been updated here. This change would also help allow refactoring for remove_reader()/remove_writer() to be smarter about how file descriptors get managed in the future (i.e. using a counter instead of removes() to avoid possible race conditions with file descriptors being reused) --- celery/concurrency/asynpool.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index c2dbb0241..5471359c2 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -241,21 +241,21 @@ def _make_process_result(self, hub): fileno_to_outq = self.fileno_to_outq on_state_change = self.on_state_change add_reader = hub.add_reader - hub_remove = hub.remove + remove_reader = hub.remove_reader recv_message = self._recv_message def on_result_readable(fileno): try: fileno_to_outq[fileno] except KeyError: # process gone - return hub_remove(fileno) + return remove_reader(fileno) it = recv_message(add_reader, fileno, on_state_change) try: next(it) except StopIteration: pass except (IOError, OSError, EOFError): - hub_remove(fileno) + remove_reader(fileno) else: add_reader(fileno, it) return on_result_readable @@ -477,7 +477,7 @@ def on_job_ready(self, job, i, obj, inqW_fd): def _create_process_handlers(self, hub, READ=READ, ERR=ERR): """For async pool this will create the handlers called when a process is up/down and etc.""" - add_reader, hub_remove = hub.add_reader, hub.remove + add_reader, remove_reader, remove_writer = hub.add_reader, hub.remove_reader, hub.remove_writer cache = self._cache all_inqueues = self._all_inqueues fileno_to_inq = self._fileno_to_inq @@ -528,7 +528,7 @@ def on_process_up(proc): self.on_process_up = on_process_up - def _remove_from_index(obj, proc, index, callback=None): + def _remove_from_index(obj, proc, index, remove_func, callback=None): # this remove the file descriptors for a process from # the indices. we have to make sure we don't overwrite # another processes fds, as the fds may be reused. @@ -544,7 +544,7 @@ def _remove_from_index(obj, proc, index, callback=None): except KeyError: pass else: - hub_remove(fd) + remove_func(fd) if callback is not None: callback(fd) return fd @@ -554,23 +554,24 @@ def on_process_down(proc): if proc.dead: return process_flush_queues(proc) - _remove_from_index(proc.outq._reader, proc, fileno_to_outq) + _remove_from_index(proc.outq._reader, proc, fileno_to_outq, remove_func=remove_reader) if proc.synq: - _remove_from_index(proc.synq._writer, proc, fileno_to_synq) + _remove_from_index(proc.synq._writer, proc, fileno_to_synq, remove_func=remove_writer) inq = _remove_from_index(proc.inq._writer, proc, fileno_to_inq, + remove_func=remove_writer, callback=all_inqueues.discard) if inq: busy_workers.discard(inq) - hub_remove(proc.sentinel) + remove_reader(proc.sentinel) waiting_to_start.discard(proc) self._active_writes.discard(proc.inqW_fd) - hub_remove(proc.inqW_fd) - hub_remove(proc.outqR_fd) + remove_writer(proc.inqW_fd) + remove_reader(proc.outqR_fd) if proc.synqR_fd: - hub_remove(proc.synqR_fd) + remove_reader(proc.synqR_fd) if proc.synqW_fd: self._active_writes.discard(proc.synqW_fd) - hub_remove(proc.synqW_fd) + remove_reader(proc.synqW_fd) self.on_process_down = on_process_down def _create_write_handlers(self, hub, From 318f039f935453f415a6a99f4e5b401af2286aa5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 May 2014 16:01:23 +0100 Subject: [PATCH 0146/1103] Fixes func -> fun (Issue #2032) --- celery/concurrency/asynpool.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 5471359c2..78e8a2136 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -528,7 +528,7 @@ def on_process_up(proc): self.on_process_up = on_process_up - def _remove_from_index(obj, proc, index, remove_func, callback=None): + def _remove_from_index(obj, proc, index, remove_fun, callback=None): # this remove the file descriptors for a process from # the indices. we have to make sure we don't overwrite # another processes fds, as the fds may be reused. @@ -544,7 +544,7 @@ def _remove_from_index(obj, proc, index, remove_func, callback=None): except KeyError: pass else: - remove_func(fd) + remove_fun(fd) if callback is not None: callback(fd) return fd @@ -554,11 +554,11 @@ def on_process_down(proc): if proc.dead: return process_flush_queues(proc) - _remove_from_index(proc.outq._reader, proc, fileno_to_outq, remove_func=remove_reader) + _remove_from_index(proc.outq._reader, proc, fileno_to_outq, remove_fun=remove_reader) if proc.synq: - _remove_from_index(proc.synq._writer, proc, fileno_to_synq, remove_func=remove_writer) + _remove_from_index(proc.synq._writer, proc, fileno_to_synq, remove_fun=remove_writer) inq = _remove_from_index(proc.inq._writer, proc, fileno_to_inq, - remove_func=remove_writer, + remove_fun=remove_writer, callback=all_inqueues.discard) if inq: busy_workers.discard(inq) From fe22f278fb130ee4d97792c27361f4a6a548f98e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 May 2014 17:01:15 +0100 Subject: [PATCH 0147/1103] Fixes unicode problem with beat after adding unicode_literals --- celery/apps/beat.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/celery/apps/beat.py b/celery/apps/beat.py index 97fa4829a..0d053de91 100644 --- a/celery/apps/beat.py +++ b/celery/apps/beat.py @@ -98,10 +98,10 @@ def start_scheduler(self): scheduler_cls=self.scheduler_cls, schedule_filename=self.schedule) - print(str(c.blue('__ ', c.magenta('-'), - c.blue(' ... __ '), c.magenta('-'), - c.blue(' _\n'), - c.reset(self.startup_info(beat))))) + print(unicode(c.blue('__ ', c.magenta('-'), + c.blue(' ... __ '), c.magenta('-'), + c.blue(' _\n'), + c.reset(self.startup_info(beat))))) self.setup_logging() if self.socket_timeout: logger.debug('Setting default socket timeout to %r', From 8187301dfdc1a0bafc29f0818198c722a0dea94d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 May 2014 17:03:57 +0100 Subject: [PATCH 0148/1103] Fixes flakes for Issue #2032 --- celery/concurrency/asynpool.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 78e8a2136..70b89cef9 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -477,7 +477,9 @@ def on_job_ready(self, job, i, obj, inqW_fd): def _create_process_handlers(self, hub, READ=READ, ERR=ERR): """For async pool this will create the handlers called when a process is up/down and etc.""" - add_reader, remove_reader, remove_writer = hub.add_reader, hub.remove_reader, hub.remove_writer + add_reader, remove_reader, remove_writer = ( + hub.add_reader, hub.remove_reader, hub.remove_writer, + ) cache = self._cache all_inqueues = self._all_inqueues fileno_to_inq = self._fileno_to_inq @@ -554,12 +556,17 @@ def on_process_down(proc): if proc.dead: return process_flush_queues(proc) - _remove_from_index(proc.outq._reader, proc, fileno_to_outq, remove_fun=remove_reader) + _remove_from_index( + proc.outq._reader, proc, fileno_to_outq, remove_reader, + ) if proc.synq: - _remove_from_index(proc.synq._writer, proc, fileno_to_synq, remove_fun=remove_writer) - inq = _remove_from_index(proc.inq._writer, proc, fileno_to_inq, - remove_fun=remove_writer, - callback=all_inqueues.discard) + _remove_from_index( + proc.synq._writer, proc, fileno_to_synq, remove_writer, + ) + inq = _remove_from_index( + proc.inq._writer, proc, fileno_to_inq, remove_writer, + callback=all_inqueues.discard, + ) if inq: busy_workers.discard(inq) remove_reader(proc.sentinel) From dc28e8a54d0c654e473f949be650ac53433ed9c5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 May 2014 17:05:50 +0100 Subject: [PATCH 0149/1103] Worker now preserves exit code. Closes #2024 --- celery/apps/worker.py | 14 ++++++----- celery/bin/worker.py | 6 +++-- celery/tests/bin/test_worker.py | 40 +++++++++++++++++------------- celery/tests/case.py | 9 +++++++ celery/tests/worker/test_loops.py | 11 ++++---- celery/tests/worker/test_state.py | 33 ++++++++++++++++++++++-- celery/tests/worker/test_worker.py | 5 ++-- celery/worker/__init__.py | 19 +++++++++----- celery/worker/loops.py | 12 ++++++--- celery/worker/state.py | 12 ++++----- 10 files changed, 111 insertions(+), 50 deletions(-) diff --git a/celery/apps/worker.py b/celery/apps/worker.py index d19071108..521ef5f95 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -30,7 +30,7 @@ ) from celery.five import string, string_t from celery.loaders.app import AppLoader -from celery.platforms import check_privileges +from celery.platforms import EX_FAILURE, EX_OK, check_privileges from celery.utils import cry, isatty from celery.utils.imports import qualname from celery.utils.log import get_logger, in_sighandler, set_in_sighandler @@ -277,7 +277,7 @@ def set_process_status(self, info): def _shutdown_handler(worker, sig='TERM', how='Warm', - exc=WorkerShutdown, callback=None): + exc=WorkerShutdown, callback=None, exitcode=EX_OK): def _handle_request(*args): with in_sighandler(): @@ -288,9 +288,9 @@ def _handle_request(*args): safe_say('worker: {0} shutdown (MainProcess)'.format(how)) if active_thread_count() > 1: setattr(state, {'Warm': 'should_stop', - 'Cold': 'should_terminate'}[how], True) + 'Cold': 'should_terminate'}[how], exitcode) else: - raise exc() + raise exc(exitcode) _handle_request.__name__ = str('worker_{0}'.format(how)) platforms.signals[sig] = _handle_request install_worker_term_handler = partial( @@ -299,6 +299,7 @@ def _handle_request(*args): if not is_jython: # pragma: no cover install_worker_term_hard_handler = partial( _shutdown_handler, sig='SIGQUIT', how='Cold', exc=WorkerTerminate, + exitcode=EX_FAILURE, ) else: # pragma: no cover install_worker_term_handler = \ @@ -310,7 +311,8 @@ def on_SIGINT(worker): install_worker_term_hard_handler(worker, sig='SIGINT') if not is_jython: # pragma: no cover install_worker_int_handler = partial( - _shutdown_handler, sig='SIGINT', callback=on_SIGINT + _shutdown_handler, sig='SIGINT', callback=on_SIGINT, + exitcode=EX_FAILURE, ) else: # pragma: no cover install_worker_int_handler = lambda *a, **kw: None @@ -332,7 +334,7 @@ def restart_worker_sig_handler(*args): import atexit atexit.register(_reload_current_worker) from celery.worker import state - state.should_stop = True + state.should_stop = EX_OK platforms.signals[sig] = restart_worker_sig_handler diff --git a/celery/bin/worker.py b/celery/bin/worker.py index d5592f85f..05b249d69 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -205,12 +205,14 @@ def run(self, hostname=None, pool_cls=None, app=None, uid=None, gid=None, loglevel, '|'.join( l for l in LOG_LEVELS if isinstance(l, string_t)))) - return self.app.Worker( + worker = self.app.Worker( hostname=hostname, pool_cls=pool_cls, loglevel=loglevel, logfile=logfile, # node format handled by celery.app.log.setup pidfile=self.node_format(pidfile, hostname), state_db=self.node_format(state_db, hostname), **kwargs - ).start() + ) + worker.start() + return worker.exitcode def with_pool_option(self, argv): # this command support custom pools diff --git a/celery/tests/bin/test_worker.py b/celery/tests/bin/test_worker.py index e4ebf7157..864271172 100644 --- a/celery/tests/bin/test_worker.py +++ b/celery/tests/bin/test_worker.py @@ -17,6 +17,7 @@ from celery.exceptions import ( ImproperlyConfigured, WorkerShutdown, WorkerTerminate, ) +from celery.platforms import EX_FAILURE, EX_OK from celery.utils.log import ensure_process_aware_logger from celery.worker import state @@ -490,8 +491,8 @@ def test_worker_int_handler(self): worker = self._Worker() handlers = self.psig(cd.install_worker_int_handler, worker) next_handlers = {} - state.should_stop = False - state.should_terminate = False + state.should_stop = None + state.should_terminate = None class Signals(platforms.Signals): @@ -504,15 +505,17 @@ def __setitem__(self, sig, handler): try: handlers['SIGINT']('SIGINT', object()) self.assertTrue(state.should_stop) + self.assertEqual(state.should_stop, EX_FAILURE) finally: platforms.signals = p - state.should_stop = False + state.should_stop = None try: next_handlers['SIGINT']('SIGINT', object()) self.assertTrue(state.should_terminate) + self.assertEqual(state.should_terminate, EX_FAILURE) finally: - state.should_terminate = False + state.should_terminate = None with patch('celery.apps.worker.active_thread_count') as c: c.return_value = 1 @@ -543,7 +546,7 @@ def test_worker_int_handler_only_stop_MainProcess(self): self.assertTrue(state.should_stop) finally: process.name = name - state.should_stop = False + state.should_stop = None with patch('celery.apps.worker.active_thread_count') as c: c.return_value = 1 @@ -554,7 +557,7 @@ def test_worker_int_handler_only_stop_MainProcess(self): handlers['SIGINT']('SIGINT', object()) finally: process.name = name - state.should_stop = False + state.should_stop = None @disable_stdouts def test_install_HUP_not_supported_handler(self): @@ -580,14 +583,17 @@ def test_worker_term_hard_handler_only_stop_MainProcess(self): handlers['SIGQUIT']('SIGQUIT', object()) self.assertTrue(state.should_terminate) finally: - state.should_terminate = False + state.should_terminate = None with patch('celery.apps.worker.active_thread_count') as c: c.return_value = 1 worker = self._Worker() handlers = self.psig( cd.install_worker_term_hard_handler, worker) - with self.assertRaises(WorkerTerminate): - handlers['SIGQUIT']('SIGQUIT', object()) + try: + with self.assertRaises(WorkerTerminate): + handlers['SIGQUIT']('SIGQUIT', object()) + finally: + state.should_terminate = None finally: process.name = name @@ -599,9 +605,9 @@ def test_worker_term_handler_when_threads(self): handlers = self.psig(cd.install_worker_term_handler, worker) try: handlers['SIGTERM']('SIGTERM', object()) - self.assertTrue(state.should_stop) + self.assertEqual(state.should_stop, EX_OK) finally: - state.should_stop = False + state.should_stop = None @disable_stdouts def test_worker_term_handler_when_single_thread(self): @@ -613,7 +619,7 @@ def test_worker_term_handler_when_single_thread(self): with self.assertRaises(WorkerShutdown): handlers['SIGTERM']('SIGTERM', object()) finally: - state.should_stop = False + state.should_stop = None @patch('sys.__stderr__') @skip_if_pypy @@ -637,7 +643,7 @@ def test_worker_term_handler_only_stop_MainProcess(self): worker = self._Worker() handlers = self.psig(cd.install_worker_term_handler, worker) handlers['SIGTERM']('SIGTERM', object()) - self.assertTrue(state.should_stop) + self.assertEqual(state.should_stop, EX_OK) with patch('celery.apps.worker.active_thread_count') as c: c.return_value = 1 worker = self._Worker() @@ -646,7 +652,7 @@ def test_worker_term_handler_only_stop_MainProcess(self): handlers['SIGTERM']('SIGTERM', object()) finally: process.name = name - state.should_stop = False + state.should_stop = None @disable_stdouts @patch('celery.platforms.close_open_fds') @@ -665,14 +671,14 @@ def _execv(*args): worker = self._Worker() handlers = self.psig(cd.install_worker_restart_handler, worker) handlers['SIGHUP']('SIGHUP', object()) - self.assertTrue(state.should_stop) + self.assertEqual(state.should_stop, EX_OK) self.assertTrue(register.called) callback = register.call_args[0][0] callback() self.assertTrue(argv) finally: os.execv = execv - state.should_stop = False + state.should_stop = None @disable_stdouts def test_worker_term_hard_handler_when_threaded(self): @@ -684,7 +690,7 @@ def test_worker_term_hard_handler_when_threaded(self): handlers['SIGQUIT']('SIGQUIT', object()) self.assertTrue(state.should_terminate) finally: - state.should_terminate = False + state.should_terminate = None @disable_stdouts def test_worker_term_hard_handler_when_single_threaded(self): diff --git a/celery/tests/case.py b/celery/tests/case.py index 77a2dbc5d..551d0dfbb 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -464,6 +464,15 @@ def _teardown_app(self): self._threads_at_setup, list(threading.enumerate()), ) + # Make sure no test left the shutdown flags enabled. + from celery.worker import state as worker_state + # check for EX_OK + self.assertIsNot(worker_state.should_stop, False) + self.assertIsNot(worker_state.should_terminate, False) + # check for other true values + self.assertFalse(worker_state.should_stop) + self.assertFalse(worker_state.should_terminate) + def _get_test_name(self): return '.'.join([self.__class__.__name__, self._testMethodName]) diff --git a/celery/tests/worker/test_loops.py b/celery/tests/worker/test_loops.py index fd6c8046f..4030782f4 100644 --- a/celery/tests/worker/test_loops.py +++ b/celery/tests/worker/test_loops.py @@ -7,6 +7,7 @@ from celery.bootsteps import CLOSE, RUN from celery.exceptions import InvalidTaskError, WorkerShutdown, WorkerTerminate from celery.five import Empty +from celery.platforms import EX_FAILURE from celery.worker import state from celery.worker.consumer import Consumer from celery.worker.loops import asynloop, synloop @@ -179,27 +180,27 @@ def test_should_terminate(self): with self.assertRaises(WorkerTerminate): asynloop(*x.args) finally: - state.should_terminate = False + state.should_terminate = None def test_should_terminate_hub_close_raises(self): x = X(self.app) # XXX why aren't the errors propagated?!? - state.should_terminate = True + state.should_terminate = EX_FAILURE x.hub.close.side_effect = MemoryError() try: with self.assertRaises(WorkerTerminate): asynloop(*x.args) finally: - state.should_terminate = False + state.should_terminate = None def test_should_stop(self): x = X(self.app) - state.should_stop = True + state.should_stop = 303 try: with self.assertRaises(WorkerShutdown): asynloop(*x.args) finally: - state.should_stop = False + state.should_stop = None def test_updates_qos(self): x = X(self.app) diff --git a/celery/tests/worker/test_state.py b/celery/tests/worker/test_state.py index ede9a00a1..707fb1fe8 100644 --- a/celery/tests/worker/test_state.py +++ b/celery/tests/worker/test_state.py @@ -48,13 +48,42 @@ class MyPersistent(state.Persistent): class test_maybe_shutdown(AppCase): def teardown(self): - state.should_stop = False - state.should_terminate = False + state.should_stop = None + state.should_terminate = None def test_should_stop(self): state.should_stop = True with self.assertRaises(WorkerShutdown): state.maybe_shutdown() + state.should_stop = 0 + with self.assertRaises(WorkerShutdown): + state.maybe_shutdown() + state.should_stop = False + try: + state.maybe_shutdown() + except SystemExit: + raise RuntimeError('should not have exited') + state.should_stop = None + try: + state.maybe_shutdown() + except SystemExit: + raise RuntimeError('should not have exited') + + state.should_stop = 0 + try: + state.maybe_shutdown() + except SystemExit as exc: + self.assertEqual(exc.code, 0) + else: + raise RuntimeError('should have exited') + + state.should_stop = 303 + try: + state.maybe_shutdown() + except SystemExit as exc: + self.assertEqual(exc.code, 303) + else: + raise RuntimeError('should have exited') def test_should_terminate(self): state.should_terminate = True diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index f3a3e1494..5ac5f6a9a 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -20,6 +20,7 @@ WorkerShutdown, WorkerTerminate, TaskRevokedError, InvalidTaskError, ) from celery.five import Empty, range, Queue as FastQueue +from celery.platforms import EX_FAILURE from celery.utils import uuid from celery.worker import components from celery.worker import consumer @@ -864,7 +865,7 @@ def test_shutdown_no_blueprint(self): self.worker.blueprint = None self.worker._shutdown() - @patch('celery.platforms.create_pidlock') + @patch('celery.worker.create_pidlock') def test_use_pidfile(self, create_pidlock): create_pidlock.return_value = Mock() worker = self.create_worker(pidfile='pidfilelockfilepid') @@ -1112,7 +1113,7 @@ def test_step_raises(self): step.start.side_effect = TypeError() worker.stop = Mock() worker.start() - worker.stop.assert_called_with() + worker.stop.assert_called_with(exitcode=EX_FAILURE) def test_state(self): self.assertTrue(self.worker.state) diff --git a/celery/worker/__init__.py b/celery/worker/__init__.py index 29a095939..74c981d82 100644 --- a/celery/worker/__init__.py +++ b/celery/worker/__init__.py @@ -26,12 +26,12 @@ from celery import bootsteps from celery.bootsteps import RUN, TERMINATE from celery import concurrency as _concurrency -from celery import platforms from celery import signals from celery.exceptions import ( ImproperlyConfigured, WorkerTerminate, TaskRevokedError, ) from celery.five import string_t, values +from celery.platforms import EX_FAILURE, create_pidlock from celery.utils import default_nodename, worker_direct from celery.utils.imports import reload_from_cwd from celery.utils.log import mlevel, worker_logger as logger @@ -73,6 +73,9 @@ class WorkController(object): pool = None semaphore = None + #: contains the exit code if a :exc:`SystemExit` event is handled. + exitcode = None + class Blueprint(bootsteps.Blueprint): """Worker bootstep blueprint.""" name = 'Worker' @@ -150,7 +153,7 @@ def on_after_init(self, **kwargs): def on_start(self): if self.pidfile: - self.pidlock = platforms.create_pidlock(self.pidfile) + self.pidlock = create_pidlock(self.pidfile) def on_consumer_ready(self, consumer): pass @@ -207,9 +210,11 @@ def start(self): self.terminate() except Exception as exc: logger.error('Unrecoverable error: %r', exc, exc_info=True) - self.stop() - except (KeyboardInterrupt, SystemExit): - self.stop() + self.stop(exitcode=EX_FAILURE) + except SystemExit as exc: + self.stop(exitcode=exc.code) + except KeyboardInterrupt: + self.stop(exitcode=EX_FAILURE) def register_with_event_loop(self, hub): self.blueprint.send_all( @@ -243,8 +248,10 @@ def should_use_eventloop(self): return (detect_environment() == 'default' and self._conninfo.is_evented and not self.app.IS_WINDOWS) - def stop(self, in_sighandler=False): + def stop(self, in_sighandler=False, exitcode=None): """Graceful shutdown of the worker server.""" + if exitcode is not None: + self.exitcode = exitcode if self.blueprint.state == RUN: self.signal_consumer_close() if not in_sighandler or self.pool.signal_safe: diff --git a/celery/worker/loops.py b/celery/worker/loops.py index 406633e00..5faa99e24 100644 --- a/celery/worker/loops.py +++ b/celery/worker/loops.py @@ -57,10 +57,14 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos, try: while blueprint.state == RUN and obj.connection: # shutdown if signal handlers told us to. - if state.should_stop: - raise WorkerShutdown() - elif state.should_terminate: - raise WorkerTerminate() + should_stop, should_terminate = ( + state.should_stop, state.should_terminate, + ) + # False == EX_OK, so must use is not False + if should_stop is not None and should_stop is not False: + raise WorkerShutdown(should_stop) + elif should_terminate is not None and should_stop is not False: + raise WorkerTerminate(should_terminate) # We only update QoS when there is no more messages to read. # This groups together qos calls, and makes sure that remote diff --git a/celery/worker/state.py b/celery/worker/state.py index 3e1ab95c5..2e2773e56 100644 --- a/celery/worker/state.py +++ b/celery/worker/state.py @@ -60,15 +60,15 @@ #: Update global state when a task has been reserved. task_reserved = reserved_requests.add -should_stop = False -should_terminate = False +should_stop = None +should_terminate = None def maybe_shutdown(): - if should_stop: - raise WorkerShutdown() - elif should_terminate: - raise WorkerTerminate() + if should_stop is not None and should_stop is not False: + raise WorkerShutdown(should_stop) + elif should_terminate is not None and should_terminate is not False: + raise WorkerTerminate(should_terminate) def task_accepted(request, _all_total_count=all_total_count): From e0389ec83d081ac882e7c943bdd10bdc7f7c379a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 May 2014 17:32:19 +0100 Subject: [PATCH 0150/1103] Fixes Signature.link* to work when link option is not a list. Closes #2019 --- celery/canvas.py | 4 +++- celery/tests/tasks/test_canvas.py | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index f7ba6d1cd..4d8e713af 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -12,7 +12,7 @@ """ from __future__ import absolute_import -from collections import deque +from collections import MutableSequence, deque from copy import deepcopy from functools import partial as _partial, reduce from operator import itemgetter @@ -244,6 +244,8 @@ def apply_async(self, args=(), kwargs={}, **options): def append_to_list_option(self, key, value): items = self.options.setdefault(key, []) + if not isinstance(items, MutableSequence): + items = self.options[key] = [items] if value not in items: items.append(value) return value diff --git a/celery/tests/tasks/test_canvas.py b/celery/tests/tasks/test_canvas.py index 4c071a8a1..25080252b 100644 --- a/celery/tests/tasks/test_canvas.py +++ b/celery/tests/tasks/test_canvas.py @@ -59,6 +59,14 @@ def test_getitem_property(self): self.assertEqual(SIG.options, {'task_id': 'TASK_ID'}) self.assertEqual(SIG.subtask_type, '') + def test_link_on_scalar(self): + x = Signature('TASK', link=Signature('B')) + self.assertTrue(x.options['link']) + x.link(Signature('C')) + self.assertIsInstance(x.options['link'], list) + self.assertIn(Signature('B'), x.options['link']) + self.assertIn(Signature('C'), x.options['link']) + def test_replace(self): x = Signature('TASK', ('A'), {}) self.assertTupleEqual(x.replace(args=('B', )).args, ('B', )) From c68e2b9441f435981ae77c2b6ff12cf1a8281f3a Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Tue, 29 Apr 2014 17:01:13 +0700 Subject: [PATCH 0151/1103] Added myself to contributors Conflicts: CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 3484a6712..ac27a0c74 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -160,4 +160,5 @@ Martin Davidsson, 2014/02/08 Chris Clark, 2014/02/20 Matthew Duggan, 2014/04/10 Brian Bouterse, 2014/04/10 +Dmitry Malinovsky, 2014/04/28 Luke Pomfrey, 2014/05/06 From 233aae2f8e6c107b75f4f62191e0b2ef18b6e680 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 May 2014 17:57:04 +0100 Subject: [PATCH 0152/1103] Programs: amqp command no longer messes up argv --- celery/bin/amqp.py | 16 +++++++++------- celery/tests/bin/test_amqp.py | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/celery/bin/amqp.py b/celery/bin/amqp.py index 4dab1527a..638b5ed7a 100644 --- a/celery/bin/amqp.py +++ b/celery/bin/amqp.py @@ -246,32 +246,34 @@ def completenames(self, text, *ignored): return [cmd for cmd in names if cmd.partition('.')[2].startswith(text)] - def dispatch(self, cmd, argline): + def dispatch(self, cmd, arglist): """Dispatch and execute the command. Lookup order is: :attr:`builtins` -> :attr:`amqp`. """ - arglist = shlex.split(safe_str(argline)) + if isinstance(arglist, string_t): + arglist = shlex.split(safe_str(arglist)) if cmd in self.builtins: return getattr(self, self.builtins[cmd])(*arglist) fun, args, formatter = self.get_amqp_api_command(cmd, arglist) return formatter(fun(*args)) - def parseline(self, line): + def parseline(self, parts): """Parse input line. :returns: tuple of three items: `(command_name, arglist, original_line)` """ - parts = line.split() if parts: - return parts[0], ' '.join(parts[1:]), line - return '', '', line + return parts[0], parts[1:], ' '.join(parts) + return '', '', '' def onecmd(self, line): """Parse line and execute command.""" + if isinstance(line, string_t): + line = shlex.split(safe_str(line)) cmd, arg, line = self.parseline(line) if not line: return self.emptyline() @@ -326,7 +328,7 @@ def connect(self, conn=None): def run(self): shell = self.Shell(connect=self.connect, out=self.out) if self.args: - return shell.onecmd(' '.join(self.args)) + return shell.onecmd(self.args) try: return shell.cmdloop() except KeyboardInterrupt: diff --git a/celery/tests/bin/test_amqp.py b/celery/tests/bin/test_amqp.py index 8840a9f10..20ab44168 100644 --- a/celery/tests/bin/test_amqp.py +++ b/celery/tests/bin/test_amqp.py @@ -124,7 +124,7 @@ def test_note(self): self.assertNotIn('FOO', self.fh.getvalue()) def test_run(self): - a = self.create_adm('queue.declare foo') + a = self.create_adm('queue.declare', 'foo') a.run() self.assertIn('ok', self.fh.getvalue()) From 41c7307da401af8ff921e9526e724350bdf4ad5e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 May 2014 19:01:54 +0100 Subject: [PATCH 0153/1103] Sanitized result backend URI in banner, reports and inspect conf. Depends on celery/kombu@f4ef17236e0085b0d948162cfbaa6d42935e2dca + celery/kombu@66419eb780c8392286212c7a73c525277b10c970 Closes #2004 --- celery/app/utils.py | 7 ++++++- celery/apps/worker.py | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/celery/app/utils.py b/celery/app/utils.py index d017de2a3..f8bd9837f 100644 --- a/celery/app/utils.py +++ b/celery/app/utils.py @@ -15,6 +15,8 @@ from collections import Mapping from types import ModuleType +from kombu.utils.url import maybe_sanitize_url + from celery.datastructures import ConfigurationView from celery.five import items, string_t, values from celery.platforms import pyimplementation @@ -177,9 +179,12 @@ def maybe_censor(key, value, mask='*' * 8): if isinstance(key, string_t): if HIDDEN_SETTINGS.search(key): return mask - if 'BROKER_URL' in key.upper(): + elif 'BROKER_URL' in key.upper(): from kombu import Connection return Connection(value).as_uri(mask=mask) + elif key.upper() in ('CELERY_RESULT_BACKEND', 'CELERY_BACKEND'): + return maybe_sanitize_url(value, mask=mask) + return value return {k: maybe_censor(k, v) for k, v in items(conf)} diff --git a/celery/apps/worker.py b/celery/apps/worker.py index 521ef5f95..ae0c8d509 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -22,6 +22,7 @@ from billiard import current_process from kombu.utils.encoding import safe_str +from kombu.utils.url import maybe_sanitize_url from celery import VERSION_BANNER, platforms, signals from celery.app import trace @@ -227,7 +228,9 @@ def startup_info(self): hostname=safe_str(self.hostname), version=VERSION_BANNER, conninfo=self.app.connection().as_uri(), - results=self.app.conf.CELERY_RESULT_BACKEND or 'disabled', + results=maybe_sanitize_url( + self.app.conf.CELERY_RESULT_BACKEND or 'disabled', + ), concurrency=concurrency, platform=safe_str(_platform.platform()), events=events, From b6e0aff80bdc7b80f62214aebd4f688b6bb9398b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 May 2014 19:14:09 +0100 Subject: [PATCH 0154/1103] CI must now depend on dev branches --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index bde53e19a..304980a15 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ basepython = python3.4 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt -r{toxinidir}/requirements/test-ci.txt + -r{toxinidir}/requirements/dev.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] @@ -23,6 +24,7 @@ basepython = python3.3 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt -r{toxinidir}/requirements/test-ci.txt + -r{toxinidir}/requirements/dev.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] @@ -32,6 +34,7 @@ basepython = python2.7 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/test-ci.txt + -r{toxinidir}/requirements/dev.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] @@ -41,6 +44,7 @@ basepython = pypy deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/test-ci.txt + -r{toxinidir}/requirements/dev.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] From e20ba0df2c5fb78dcb53369121c33d1360203c96 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 May 2014 19:20:18 +0100 Subject: [PATCH 0155/1103] tox no like no zipballs :cow: --- tox.ini | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 304980a15..a15e36635 100644 --- a/tox.ini +++ b/tox.ini @@ -14,9 +14,9 @@ basepython = python3.4 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt -r{toxinidir}/requirements/test-ci.txt - -r{toxinidir}/requirements/dev.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} + pip install -U -r{toxinidir}/requirements/dev.txt nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:3.3] @@ -24,9 +24,9 @@ basepython = python3.3 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt -r{toxinidir}/requirements/test-ci.txt - -r{toxinidir}/requirements/dev.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} + pip install -U -r{toxinidir}/requirements/dev.txt nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:2.7] @@ -34,9 +34,9 @@ basepython = python2.7 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/test-ci.txt - -r{toxinidir}/requirements/dev.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} + pip install -U -r{toxinidir}/requirements/dev.txt nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:pypy] @@ -47,4 +47,5 @@ deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/dev.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} + pip install -U -r{toxinidir}/requirements/dev.txt nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] From f92131ac2a4eb9c6e5b9278fcd8fe5db3d66cb51 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 May 2014 20:23:15 +0100 Subject: [PATCH 0156/1103] Tests passing on Python3 --- celery/apps/beat.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/celery/apps/beat.py b/celery/apps/beat.py index 0d053de91..356a3753d 100644 --- a/celery/apps/beat.py +++ b/celery/apps/beat.py @@ -17,6 +17,7 @@ import sys from celery import VERSION_BANNER, platforms, beat +from celery.five import text_t from celery.utils.imports import qualname from celery.utils.log import LOG_LEVELS, get_logger from celery.utils.timeutils import humanize_seconds @@ -98,10 +99,12 @@ def start_scheduler(self): scheduler_cls=self.scheduler_cls, schedule_filename=self.schedule) - print(unicode(c.blue('__ ', c.magenta('-'), - c.blue(' ... __ '), c.magenta('-'), - c.blue(' _\n'), - c.reset(self.startup_info(beat))))) + print(text_t( + c.blue('__ ', c.magenta('-'), + c.blue(' ... __ '), c.magenta('-'), + c.blue(' _\n'), + c.reset(self.startup_info(beat))), + )) self.setup_logging() if self.socket_timeout: logger.debug('Setting default socket timeout to %r', From 66672c0f17003712ab58705d0b213f47b784b41b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 May 2014 22:12:36 +0100 Subject: [PATCH 0157/1103] Use kombu.utils.json instead of anyjson :sad: --- celery/apps/beat.py | 2 +- celery/bin/celery.py | 7 ++++--- celery/events/__init__.py | 15 --------------- celery/five.py | 8 -------- celery/loaders/base.py | 4 ++-- celery/task/http.py | 5 +++-- celery/tests/bin/test_celery.py | 3 ++- celery/tests/compat_modules/test_http.py | 2 +- celery/tests/compat_modules/test_sets.py | 5 +++-- celery/worker/consumer.py | 2 +- celery/worker/strategy.py | 2 +- docs/faq.rst | 15 +-------------- docs/userguide/remote-tasks.rst | 4 ++-- examples/httpexample/views.py | 2 +- funtests/benchmarks/bench_worker.py | 7 ------- 15 files changed, 22 insertions(+), 61 deletions(-) diff --git a/celery/apps/beat.py b/celery/apps/beat.py index 356a3753d..3daecd11f 100644 --- a/celery/apps/beat.py +++ b/celery/apps/beat.py @@ -99,7 +99,7 @@ def start_scheduler(self): scheduler_cls=self.scheduler_cls, schedule_filename=self.schedule) - print(text_t( + print(text_t( # noqa (pyflakes chokes on print) c.blue('__ ', c.magenta('-'), c.blue(' ... __ '), c.magenta('-'), c.blue(' _\n'), diff --git a/celery/bin/celery.py b/celery/bin/celery.py index 3d0cf5d8f..57c243040 100644 --- a/celery/bin/celery.py +++ b/celery/bin/celery.py @@ -8,7 +8,6 @@ """ from __future__ import absolute_import, unicode_literals -import anyjson import numbers import os import sys @@ -16,6 +15,8 @@ from functools import partial from importlib import import_module +from kombu.utils import json + from celery.five import string_t, values from celery.platforms import EX_OK, EX_FAILURE, EX_UNAVAILABLE, EX_USAGE from celery.utils import term @@ -162,12 +163,12 @@ def run(self, name, *_, **kw): # Positional args. args = kw.get('args') or () if isinstance(args, string_t): - args = anyjson.loads(args) + args = json.loads(args) # Keyword args. kwargs = kw.get('kwargs') or {} if isinstance(kwargs, string_t): - kwargs = anyjson.loads(kwargs) + kwargs = json.loads(kwargs) # Expires can be int/float. expires = kw.get('expires') or None diff --git a/celery/events/__init__.py b/celery/events/__init__.py index a4142f76a..93747cf56 100644 --- a/celery/events/__init__.py +++ b/celery/events/__init__.py @@ -13,7 +13,6 @@ import os import time import threading -import warnings from collections import deque from contextlib import contextmanager @@ -36,14 +35,6 @@ _TZGETTER = itemgetter('utcoffset', 'timestamp') -W_YAJL = """ -anyjson is currently using the yajl library. -This json implementation is broken, it severely truncates floats -so timestamps will not work. - -Please uninstall yajl or force anyjson to use a different library. -""" - CLIENT_CLOCK_SKEW = -1 @@ -151,12 +142,6 @@ def __init__(self, connection=None, hostname=None, enabled=True, self.enable() self.headers = {'hostname': self.hostname} self.pid = os.getpid() - self.warn_if_yajl() - - def warn_if_yajl(self): - import anyjson - if anyjson.implementation.name == 'yajl': - warnings.warn(UserWarning(W_YAJL)) def __enter__(self): return self diff --git a/celery/five.py b/celery/five.py index 5a272c9d6..4f0f3a4be 100644 --- a/celery/five.py +++ b/celery/five.py @@ -27,14 +27,6 @@ def Counter(): # noqa return defaultdict(int) -try: - buffer_t = buffer -except NameError: # pragma: no cover - # Py3 does not have buffer, but we only need isinstance. - - class buffer_t(object): # noqa - pass - ############## py3k ######################################################### import sys PY3 = sys.version_info[0] == 3 diff --git a/celery/loaders/base.py b/celery/loaders/base.py index d73547aad..533530b1d 100644 --- a/celery/loaders/base.py +++ b/celery/loaders/base.py @@ -8,7 +8,6 @@ """ from __future__ import absolute_import -import anyjson import imp as _imp import importlib import os @@ -17,6 +16,7 @@ from datetime import datetime +from kombu.utils import json from kombu.utils import cached_property from kombu.utils.encoding import safe_str @@ -178,7 +178,7 @@ def find_module(self, module): def cmdline_config_parser( self, args, namespace='celery', re_type=re.compile(r'\((\w+)\)'), - extra_types={'json': anyjson.loads}, + extra_types={'json': json.loads}, override_types={'tuple': 'json', 'list': 'json', 'dict': 'json'}): diff --git a/celery/task/http.py b/celery/task/http.py index 62b89b896..d3739b8ef 100644 --- a/celery/task/http.py +++ b/celery/task/http.py @@ -8,7 +8,6 @@ """ from __future__ import absolute_import -import anyjson import sys try: @@ -17,6 +16,8 @@ from urllib import urlencode # noqa from urlparse import urlparse, parse_qsl # noqa +from kombu.utils import json + from celery import shared_task, __version__ as celery_version from celery.five import items, reraise from celery.utils.log import get_task_logger @@ -62,7 +63,7 @@ class UnknownStatusError(InvalidResponseError): """The remote server gave an unknown status.""" -def extract_response(raw_response, loads=anyjson.loads): +def extract_response(raw_response, loads=json.loads): """Extract the response text from a raw JSON response.""" if not raw_response: raise InvalidResponseError('Empty response') diff --git a/celery/tests/bin/test_celery.py b/celery/tests/bin/test_celery.py index fbfdb62f6..573810eec 100644 --- a/celery/tests/bin/test_celery.py +++ b/celery/tests/bin/test_celery.py @@ -2,9 +2,10 @@ import sys -from anyjson import dumps from datetime import datetime +from kombu.utils.json import dumps + from celery import __main__ from celery.platforms import EX_FAILURE, EX_USAGE, EX_OK from celery.bin.base import Error diff --git a/celery/tests/compat_modules/test_http.py b/celery/tests/compat_modules/test_http.py index 08505f87e..c3a23b613 100644 --- a/celery/tests/compat_modules/test_http.py +++ b/celery/tests/compat_modules/test_http.py @@ -8,8 +8,8 @@ except ImportError: # py3k from urllib.request import addinfourl # noqa -from anyjson import dumps from kombu.utils.encoding import from_utf8 +from kombu.utils.json import dumps from celery.five import WhateverIO, items from celery.task import http diff --git a/celery/tests/compat_modules/test_sets.py b/celery/tests/compat_modules/test_sets.py index c1d2c16fa..dc38d19a5 100644 --- a/celery/tests/compat_modules/test_sets.py +++ b/celery/tests/compat_modules/test_sets.py @@ -1,8 +1,9 @@ from __future__ import absolute_import -import anyjson import warnings +from kombu.utils import json + from celery import uuid from celery.result import TaskSetResult from celery.task import Task @@ -136,7 +137,7 @@ def test_is_JSON_serializable(self): ) s.args = list(s.args) # tuples are not preserved # but this doesn't matter. - self.assertEqual(s, self.subtask(anyjson.loads(anyjson.dumps(s)))) + self.assertEqual(s, self.subtask(json.loads(json.dumps(s)))) def test_repr(self): s = self.MockTask.subtask((2, ), {'cache': True}) diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 71cf7c635..84bc8d9cb 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -26,6 +26,7 @@ from billiard.exceptions import RestartFreqExceeded from kombu.async.semaphore import DummyLock from kombu.common import QoS, ignore_errors +from kombu.five import buffer_t, items, values from kombu.syn import _detect_environment from kombu.utils.compat import get_errno from kombu.utils.encoding import safe_repr, bytes_t @@ -35,7 +36,6 @@ from celery.app.trace import build_tracer from celery.canvas import signature from celery.exceptions import InvalidTaskError -from celery.five import buffer_t, items, values from celery.utils.functional import noop from celery.utils.log import get_logger from celery.utils.text import truncate diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index a4ed1cac2..801e58c3f 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -11,9 +11,9 @@ import logging from kombu.async.timer import to_timestamp +from kombu.five import buffer_t from celery.exceptions import InvalidTaskError -from celery.five import buffer_t from celery.utils.log import get_logger from celery.utils.timeutils import timezone diff --git a/docs/faq.rst b/docs/faq.rst index d1cfc0ddf..875798b6c 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -129,22 +129,9 @@ broker this is a natural dependency. .. _`amqp`: http://pypi.python.org/pypi/amqp -- `anyjson`_ - -anyjson is an utility library to select the best possible -JSON implementation. - -.. _`anyjson`: http://pypi.python.org/pypi/anyjson - - .. note:: - For compatibility reasons additional packages may be installed - if you are running on older Python versions, - for example Python 2.6 depends on the ``importlib``, - and ``ordereddict`` libraries. - - Also, to handle the dependencies for popular configuration + To handle the dependencies for popular configuration choices Celery defines a number of "bundle" packages, see :ref:`bundles`. diff --git a/docs/userguide/remote-tasks.rst b/docs/userguide/remote-tasks.rst index e5f4aa8c7..f9cfa76fb 100644 --- a/docs/userguide/remote-tasks.rst +++ b/docs/userguide/remote-tasks.rst @@ -59,7 +59,7 @@ With this information you could define a simple task in Django: .. code-block:: python from django.http import HttpResponse - from anyjson import serialize + from json import dumps def multiply(request): @@ -67,7 +67,7 @@ With this information you could define a simple task in Django: y = int(request.GET['y']) result = x * y response = {'status': 'success', 'retval': result} - return HttpResponse(serialize(response), mimetype='application/json') + return HttpResponse(dumps(response), mimetype='application/json') .. _webhook-rails-example: diff --git a/examples/httpexample/views.py b/examples/httpexample/views.py index 5069255e6..e1f4bf0f5 100644 --- a/examples/httpexample/views.py +++ b/examples/httpexample/views.py @@ -1,6 +1,6 @@ from django.http import HttpResponse -from anyjson import dumps +from json import dumps def multiply(request): diff --git a/funtests/benchmarks/bench_worker.py b/funtests/benchmarks/bench_worker.py index 8663e8d7a..87f3615f0 100644 --- a/funtests/benchmarks/bench_worker.py +++ b/funtests/benchmarks/bench_worker.py @@ -9,13 +9,6 @@ USE_FAST_LOCALS='yes', ) -import anyjson -JSONIMP = os.environ.get('JSONIMP') -if JSONIMP: - anyjson.force_implementation(JSONIMP) - -print('anyjson implementation: {0!r}'.format(anyjson.implementation.name)) - from celery import Celery, group from celery.five import range From 65a5d2866eb734c563105fd699d98c61f5a8bbc7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 May 2014 22:13:52 +0100 Subject: [PATCH 0158/1103] Adds Signature.__json__ --- celery/canvas.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index 4d8e713af..e664d5394 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -286,7 +286,10 @@ def __invert__(self): def __reduce__(self): # for serialization, the task type is lazily loaded, # and not stored in the dict itself. - return subtask, (dict(self), ) + return signature, (dict(self), ) + + def __json__(self): + return dict(self) def reprcall(self, *args, **kwargs): args, kwargs, _ = self._merge(args, kwargs, {}) From bd8a989a46971ae94446e173e677f971320edbe8 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 May 2014 22:51:42 +0100 Subject: [PATCH 0159/1103] celery.five now extends amqp.five --- celery/five.py | 172 ++----------------------------------------------- 1 file changed, 7 insertions(+), 165 deletions(-) diff --git a/celery/five.py b/celery/five.py index 4f0f3a4be..94a4ab8ca 100644 --- a/celery/five.py +++ b/celery/five.py @@ -10,159 +10,15 @@ """ from __future__ import absolute_import -__all__ = ['Counter', 'reload', 'UserList', 'UserDict', 'Queue', 'Empty', - 'zip_longest', 'map', 'string', 'string_t', - 'long_t', 'text_t', 'range', 'int_types', 'items', 'keys', 'values', - 'nextfun', 'reraise', 'WhateverIO', 'with_metaclass', - 'THREAD_TIMEOUT_MAX', 'format_d', 'class_property', 'reclassmethod', - 'create_module', 'recreate_module', 'monotonic'] +__all__ = [ + 'class_property', 'reclassmethod', 'create_module', 'recreate_module', +] -import io - -try: - from collections import Counter -except ImportError: # pragma: no cover - from collections import defaultdict - - def Counter(): # noqa - return defaultdict(int) - -############## py3k ######################################################### -import sys -PY3 = sys.version_info[0] == 3 - -try: - reload = reload # noqa -except NameError: # pragma: no cover - from imp import reload # noqa - -try: - from UserList import UserList # noqa -except ImportError: # pragma: no cover - from collections import UserList # noqa - -try: - from UserDict import UserDict # noqa -except ImportError: # pragma: no cover - from collections import UserDict # noqa - - -from kombu.five import monotonic - -if PY3: # pragma: no cover - import builtins - - from queue import Queue, Empty - from itertools import zip_longest - - map = map - string = str - string_t = str - long_t = int - text_t = str - range = range - int_types = (int, ) - _byte_t = bytes - - open_fqdn = 'builtins.open' - - def items(d): - return d.items() - - def keys(d): - return d.keys() - - def values(d): - return d.values() - - def nextfun(it): - return it.__next__ - - exec_ = getattr(builtins, 'exec') - - def reraise(tp, value, tb=None): - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value - -else: - import __builtin__ as builtins # noqa - from Queue import Queue, Empty # noqa - from itertools import imap as map, izip_longest as zip_longest # noqa - string = unicode # noqa - string_t = basestring # noqa - text_t = unicode # noqa - long_t = long # noqa - range = xrange # noqa - int_types = (int, long) # noqa - _byte_t = (str, bytes) # noqa - - open_fqdn = '__builtin__.open' - - def items(d): # noqa - return d.iteritems() - - def keys(d): # noqa - return d.iterkeys() - - def values(d): # noqa - return d.itervalues() - - def nextfun(it): # noqa - return it.next - - def exec_(code, globs=None, locs=None): # pragma: no cover - """Execute code in a namespace.""" - if globs is None: - frame = sys._getframe(1) - globs = frame.f_globals - if locs is None: - locs = frame.f_locals - del frame - elif locs is None: - locs = globs - exec("""exec code in globs, locs""") - - exec_("""def reraise(tp, value, tb=None): raise tp, value, tb""") - - -def with_metaclass(Type, skip_attrs={'__dict__', '__weakref__'}): - """Class decorator to set metaclass. - - Works with both Python 2 and Python 3 and it does not add - an extra class in the lookup order like ``six.with_metaclass`` does - (that is -- it copies the original class instead of using inheritance). - - """ - - def _clone_with_metaclass(Class): - attrs = {key: value for key, value in items(vars(Class)) - if key not in skip_attrs} - return Type(Class.__name__, Class.__bases__, attrs) - - return _clone_with_metaclass - - -############## threading.TIMEOUT_MAX ####################################### -try: - from threading import TIMEOUT_MAX as THREAD_TIMEOUT_MAX -except ImportError: - THREAD_TIMEOUT_MAX = 1e10 # noqa - -############## format(int, ',d') ########################## - -if sys.version_info >= (2, 7): # pragma: no cover - def format_d(i): - return format(i, ',d') -else: # pragma: no cover - def format_d(i): # noqa - s = '%d' % i - groups = [] - while s and s[-1].isdigit(): - groups.append(s[-3:]) - s = s[:-3] - return s + ','.join(reversed(groups)) +# extends amqp.five +from amqp.five import * # noqa +from amqp.five import __all__ as _all_five +__all__ += _all_five ############## Module Generation ########################## @@ -209,7 +65,6 @@ def _compat_periodic_task_decorator(*args, **kwargs): from celery.task import periodic_task return periodic_task(*args, **kwargs) - COMPAT_MODULES = { 'celery': { 'execute': { @@ -368,16 +223,3 @@ def get_origins(defs): for module, attrs in items(defs): origins.update({attr: module for attr in attrs}) return origins - - -_SIO_write = io.StringIO.write -_SIO_init = io.StringIO.__init__ - - -class WhateverIO(io.StringIO): - - def __init__(self, v=None, *a, **kw): - _SIO_init(self, v.decode() if isinstance(v, _byte_t) else v, *a, **kw) - - def write(self, data): - _SIO_write(self, data.decode() if isinstance(data, _byte_t) else data) From f90fce49964db2f6b93111ced6fff8296e454490 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 21 May 2014 13:03:15 +0100 Subject: [PATCH 0160/1103] kombu.utils.get_errno is now removed as it is not needed on Py2.7 --- celery/concurrency/asynpool.py | 33 +++++++++++++++------------------ celery/platforms.py | 3 +-- celery/worker/consumer.py | 3 +-- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 70b89cef9..5dd6ac815 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -41,7 +41,6 @@ from kombu.async import READ, WRITE, ERR from kombu.serialization import pickle as _pickle from kombu.utils import fxrange -from kombu.utils.compat import get_errno from kombu.utils.eventio import SELECT_BAD_FD from celery.five import Counter, items, values from celery.utils.log import get_logger @@ -139,14 +138,14 @@ def _select(readers=None, writers=None, err=None, timeout=0): r = list(set(r) | set(e)) return r, w, 0 except (select.error, socket.error) as exc: - if get_errno(exc) == errno.EINTR: + if exc.errno == errno.EINTR: return [], [], 1 - elif get_errno(exc) in SELECT_BAD_FD: + elif exc.errno in SELECT_BAD_FD: for fd in readers | writers | err: try: select.select([fd], [], [], 0) except (select.error, socket.error) as exc: - if get_errno(exc) not in SELECT_BAD_FD: + if exc.errno not in SELECT_BAD_FD: raise readers.discard(fd) writers.discard(fd) @@ -196,7 +195,7 @@ def _recv_message(self, add_reader, fd, callback, fd, bufv[Hr:] if readcanbuf else bufv, 4 - Hr, ) except OSError as exc: - if get_errno(exc) not in UNAVAIL: + if exc.errno not in UNAVAIL: raise yield else: @@ -218,7 +217,7 @@ def _recv_message(self, add_reader, fd, callback, fd, bufv[Br:] if readcanbuf else bufv, body_size - Br, ) except OSError as exc: - if get_errno(exc) not in UNAVAIL: + if exc.errno not in UNAVAIL: raise yield else: @@ -722,7 +721,7 @@ def schedule_writes(ready_fds, shuffle=random.shuffle): except StopIteration: pass except OSError as exc: - if get_errno(exc) != errno.EBADF: + if exc.errno != errno.EBADF: raise else: add_writer(ready_fd, cor) @@ -765,7 +764,7 @@ def _write_job(proc, fd, job): try: Hw += send(header, Hw) except Exception as exc: - if get_errno(exc) not in UNAVAIL: + if getattr(exc, 'errno', None) not in UNAVAIL: raise # suspend until more data errors += 1 @@ -781,7 +780,7 @@ def _write_job(proc, fd, job): try: Bw += send(body, Bw) except Exception as exc: - if get_errno(exc) not in UNAVAIL: + if getattr(exc, 'errno', None) not in UNAVAIL: raise # suspend until more data errors += 1 @@ -830,7 +829,7 @@ def _write_ack(fd, ack, callback=None): try: Hw += send(header, Hw) except Exception as exc: - if get_errno(exc) not in UNAVAIL: + if getattr(exc, 'errno', None) not in UNAVAIL: raise yield @@ -839,7 +838,7 @@ def _write_ack(fd, ack, callback=None): try: Bw += send(body, Bw) except Exception as exc: - if get_errno(exc) not in UNAVAIL: + if getattr(exc, 'errno', None) not in UNAVAIL: raise # suspend until more data yield @@ -1041,7 +1040,7 @@ def _stop_task_handler(task_handler): try: proc.inq.put(None) except OSError as exc: - if get_errno(exc) != errno.EBADF: + if exc.errno != errno.EBADF: raise def create_result_handler(self): @@ -1092,14 +1091,12 @@ def process_flush_queues(self, proc): try: task = resq.recv() except (OSError, IOError, EOFError) as exc: - if get_errno(exc) == errno.EINTR: + _errno = getattr(exc, 'errno', None) + if _errno == errno.EINTR: continue - elif get_errno(exc) == errno.EAGAIN: + elif _errno == errno.EAGAIN: break - else: - debug('got %r while flushing process %r', - exc, proc, exc_info=1) - if get_errno(exc) not in UNAVAIL: + elif _errno not in UNAVAIL: debug('got %r while flushing process %r', exc, proc, exc_info=1) break diff --git a/celery/platforms.py b/celery/platforms.py index 651b8f5fa..0ddc3d6ae 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -24,7 +24,6 @@ from billiard import current_process # fileno used to be in this module from kombu.utils import maybe_fileno -from kombu.utils.compat import get_errno from kombu.utils.encoding import safe_str from contextlib import contextmanager @@ -531,7 +530,7 @@ def maybe_drop_privileges(uid=None, gid=None): try: setuid(0) except OSError as exc: - if get_errno(exc) != errno.EPERM: + if exc.errno != errno.EPERM: raise pass # Good: cannot restore privileges. else: diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 84bc8d9cb..6a3a56379 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -28,7 +28,6 @@ from kombu.common import QoS, ignore_errors from kombu.five import buffer_t, items, values from kombu.syn import _detect_environment -from kombu.utils.compat import get_errno from kombu.utils.encoding import safe_repr, bytes_t from kombu.utils.limits import TokenBucket @@ -271,7 +270,7 @@ def start(self): try: blueprint.start(self) except self.connection_errors as exc: - if isinstance(exc, OSError) and get_errno(exc) == errno.EMFILE: + if isinstance(exc, OSError) and exc.errno == errno.EMFILE: raise # Too many open files maybe_shutdown() try: From 4b40a7266b6e7ecd2cdc2b3230d189c331bb5d2f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 21 May 2014 14:10:29 +0100 Subject: [PATCH 0161/1103] Fixes typo Publisher -> Producer --- celery/tests/app/test_amqp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/tests/app/test_amqp.py b/celery/tests/app/test_amqp.py index 9ef9f572e..e4e8873a2 100644 --- a/celery/tests/app/test_amqp.py +++ b/celery/tests/app/test_amqp.py @@ -22,7 +22,7 @@ def test_accept_content(self): ) -class test_PublisherPool(AppCase): +class test_ProducerPool(AppCase): def test_setup_nolimit(self): self.app.conf.BROKER_POOL_LIMIT = None From 91af06d50bbfb23ccd74ca24f2402a7b70e2847e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 21 May 2014 14:11:43 +0100 Subject: [PATCH 0162/1103] Beat: Scheduler.Publisher renamed to .Producer, and .publisher to .producer --- celery/beat.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/celery/beat.py b/celery/beat.py index 64ef932e7..5623dbf5f 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -174,13 +174,13 @@ class Scheduler(object): logger = logger # compat def __init__(self, app, schedule=None, max_interval=None, - Publisher=None, lazy=False, sync_every_tasks=None, **kwargs): + Producer=None, lazy=False, sync_every_tasks=None, **kwargs): self.app = app self.data = maybe_evaluate({} if schedule is None else schedule) self.max_interval = (max_interval or app.conf.CELERYBEAT_MAX_LOOP_INTERVAL or self.max_interval) - self.Publisher = Publisher or app.amqp.TaskProducer + self.Producer = Producer or app.amqp.TaskProducer self._heap = None self.sync_every_tasks = ( app.conf.CELERYBEAT_SYNC_EVERY if sync_every_tasks is None @@ -199,11 +199,11 @@ def install_default_entries(self, data): 'options': {'expires': 12 * 3600}} self.update_from_dict(entries) - def maybe_due(self, entry, publisher=None): + def maybe_due(self, entry, producer=None): is_due, next_time_to_run = entry.is_due() if is_due: - self.apply_entry(entry, producer=publisher, advance=True) + self.apply_entry(entry, producer=producer, advance=True) return next_time_to_run def apply_entry(self, entry, producer=None): @@ -234,7 +234,7 @@ def tick(self, event_t=event_t, min=min, verify = heappop(H) if verify is event: next_entry = self.reserve(entry) - self.apply_entry(entry, producer=self.publisher) + self.apply_entry(entry, producer=self.producer) heappush(H, event_t(next_time_to_run, event[1], next_entry)) return 0 else: @@ -355,8 +355,8 @@ def connection(self): return self.app.connection() @cached_property - def publisher(self): - return self.Publisher(self._ensure_connected()) + def producer(self): + return self.Producer(self._ensure_connected()) @property def info(self): From 494f1d6e70dd708ab30c307f111b810bcbab886f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 21 May 2014 14:25:02 +0100 Subject: [PATCH 0163/1103] Programs: multi must passhthrough %i and %I logfile formats --- celery/bin/multi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/bin/multi.py b/celery/bin/multi.py index 7bb90cf92..911366e53 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -508,7 +508,7 @@ def multi_args(p, cmd='celery worker', append='', prefix='', suffix=''): expand = partial( node_format, nodename=nodename, N=shortname, d=hostname, - h=nodename, + h=nodename, i='%i', I='%I', ) argv = ([expand(cmd)] + [format_opt(opt, expand(value)) From e99172d82ff65b69a1d48d3e27090c6bd7e1d4c7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 21 May 2014 14:31:18 +0100 Subject: [PATCH 0164/1103] Default log format is now includes %I to avoid race conditions (one logfile per child process) --- celery/bin/multi.py | 6 +++--- docs/tutorials/daemonizing.rst | 19 ++++++++++++------- extra/centos/celeryd | 4 ++-- extra/centos/celeryd.sysconfig | 6 ++++-- extra/generic-init.d/celeryd | 2 +- extra/systemd/celery.conf | 2 +- 6 files changed, 23 insertions(+), 16 deletions(-) diff --git a/celery/bin/multi.py b/celery/bin/multi.py index 911366e53..e61b19e7c 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -16,13 +16,13 @@ # this. The abbreviation %n will be expanded to the current # node name. $ celery multi start Leslie -E --pidfile=/var/run/celery/%n.pid - --logfile=/var/log/celery/%n.log + --logfile=/var/log/celery/%n%I.log # You need to add the same arguments when you restart, # as these are not persisted anywhere. $ celery multi restart Leslie -E --pidfile=/var/run/celery/%n.pid - --logfile=/var/run/celery/%n.log + --logfile=/var/run/celery/%n%I.log # To stop the node, you need to specify the same pidfile. $ celery multi stop Leslie --pidfile=/var/run/celery/%n.pid @@ -252,7 +252,7 @@ def start(self, argv, cmd): def with_detacher_default_options(self, p): _setdefaultopt(p.options, ['--pidfile', '-p'], '%n.pid') - _setdefaultopt(p.options, ['--logfile', '-f'], '%n.log') + _setdefaultopt(p.options, ['--logfile', '-f'], '%n%I.log') p.options.setdefault( '--cmd', '-m {0}'.format(celery_exe('worker', '--detach')), diff --git a/docs/tutorials/daemonizing.rst b/docs/tutorials/daemonizing.rst index 6ba461ee0..311ceae85 100644 --- a/docs/tutorials/daemonizing.rst +++ b/docs/tutorials/daemonizing.rst @@ -56,7 +56,7 @@ must also export them (e.g. ``export DISPLAY=":0"``) $ celery multi start worker1 \ --pidfile="$HOME/run/celery/%n.pid" \ - --logfile="$HOME/log/celery/%n.log" + --logfile="$HOME/log/celery/%n%I.log" $ celery multi restart worker1 --pidfile="$HOME/run/celery/%n.pid" @@ -97,7 +97,7 @@ This is an example configuration for a Python project. CELERYD_OPTS="--time-limit=300 --concurrency=8" # %n will be replaced with the first part of the nodename. - CELERYD_LOG_FILE="/var/log/celery/%n.log" + CELERYD_LOG_FILE="/var/log/celery/%n%I.log" CELERYD_PID_FILE="/var/run/celery/%n.pid" # Workers should run as an unprivileged user. @@ -156,7 +156,9 @@ Available options Full path to the PID file. Default is /var/run/celery/%n.pid * CELERYD_LOG_FILE - Full path to the worker log file. Default is /var/log/celery/%n.log + Full path to the worker log file. Default is /var/log/celery/%n%I.log + **Note**: Using `%I` is important when using the prefork pool as having + multiple processes share the same log file will lead to race conditions. * CELERYD_LOG_LEVEL Worker log level. Default is INFO. @@ -311,8 +313,10 @@ This is an example configuration for a Python project: # Extra command-line arguments to the worker CELERYD_OPTS="--time-limit=300 --concurrency=8" - # %n will be replaced with the first part of the nodename. - CELERYD_LOG_FILE="/var/log/celery/%n.log" + # - %n will be replaced with the first part of the nodename. + # - %I will be replaced with the current child process index + # and is important when using the prefork pool to avoid race conditions. + CELERYD_LOG_FILE="/var/log/celery/%n%I.log" CELERYD_PID_FILE="/var/run/celery/%n.pid" .. _generic-systemd-celeryd-django-example: @@ -339,8 +343,9 @@ This is an example configuration for those using `django-celery`: # Extra command-line arguments to the worker CELERYD_OPTS="--time-limit=300 --concurrency=8" - # %n will be replaced with the first part of the nodename. - CELERYD_LOG_FILE="/var/log/celery/%n.log" + # - %n will be replaced with the first part of the nodename. + # - %I will be replaced with the current child process index + CELERYD_LOG_FILE="/var/log/celery/%n%I.log" CELERYD_PID_FILE="/var/run/celery/%n.pid" To add an environment variable such as DJANGO_SETTINGS_MODULE use the diff --git a/extra/centos/celeryd b/extra/centos/celeryd index c5e3b555c..1292cc84c 100644 --- a/extra/centos/celeryd +++ b/extra/centos/celeryd @@ -29,7 +29,7 @@ # Setting `prog` here allows you to symlink this init script, making it easy # to run multiple processes on the system. -# If we're invoked via SysV-style runlevel scripts we need to follow the +# If we're invoked via SysV-style runlevel scripts we need to follow the # link from rcX.d before working out the script name. if [[ `dirname $0` == /etc/rc*.d ]]; then target="$(readlink $0)" @@ -48,7 +48,7 @@ source /etc/init.d/functions SLEEP_SECONDS=5 DEFAULT_PID_FILE="/var/run/celery/$prog-%n.pid" -DEFAULT_LOG_FILE="/var/log/celery/$prog-%n.log" +DEFAULT_LOG_FILE="/var/log/celery/$prog-%n%I.log" DEFAULT_LOG_LEVEL="INFO" DEFAULT_NODES="celery" DEFAULT_CELERYD="-m celery.bin.celeryd_detach" diff --git a/extra/centos/celeryd.sysconfig b/extra/centos/celeryd.sysconfig index c6f2d54c6..c243b8b57 100644 --- a/extra/centos/celeryd.sysconfig +++ b/extra/centos/celeryd.sysconfig @@ -16,8 +16,10 @@ # Create log/pid dirs, if they don't already exist #CELERY_CREATE_DIRS=1 -# %n will be replaced with the nodename -#CELERYD_LOG_FILE="/path/to/my_application/log/%n.log" +# - %n will be replaced with the first part of the nodename. +# - %I will be replaced with the current child process index +# and is important when using the prefork pool to avoid race conditions. +#CELERYD_LOG_FILE="/path/to/my_application/log/%n%I.log" #CELERYD_PID_FILE="/var/run/celery/%n.pid" # Workers run as an unprivileged user diff --git a/extra/generic-init.d/celeryd b/extra/generic-init.d/celeryd index 0fe704fdf..df918bca2 100755 --- a/extra/generic-init.d/celeryd +++ b/extra/generic-init.d/celeryd @@ -48,7 +48,7 @@ SCRIPT_NAME="$(basename "$SCRIPT_FILE")" DEFAULT_USER="celery" DEFAULT_PID_FILE="/var/run/celery/%n.pid" -DEFAULT_LOG_FILE="/var/log/celery/%n.log" +DEFAULT_LOG_FILE="/var/log/celery/%n%I.log" DEFAULT_LOG_LEVEL="INFO" DEFAULT_NODES="celery" DEFAULT_CELERYD="-m celery worker --detach" diff --git a/extra/systemd/celery.conf b/extra/systemd/celery.conf index 08b90cf28..6662d43d5 100644 --- a/extra/systemd/celery.conf +++ b/extra/systemd/celery.conf @@ -6,7 +6,7 @@ CELERYD_NODES="worker" CELERYD_OPTS="" CELERY_BIN="/usr/bin/python2 -m celery" CELERYD_PID_FILE="/var/run/celery/%n.pid" -CELERYD_LOG_FILE="/var/log/celery/%n.log" +CELERYD_LOG_FILE="/var/log/celery/%n%I.log" CELERYD_LOG_LEVEL="INFO" d /run/celery 0755 user users - From a2deeb3ecd68ea5e9337c83762382508610902d1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 21 May 2014 15:15:00 +0100 Subject: [PATCH 0165/1103] flakes --- celery/beat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/beat.py b/celery/beat.py index 5623dbf5f..3579ecf4f 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -180,7 +180,7 @@ def __init__(self, app, schedule=None, max_interval=None, self.max_interval = (max_interval or app.conf.CELERYBEAT_MAX_LOOP_INTERVAL or self.max_interval) - self.Producer = Producer or app.amqp.TaskProducer + self.Producer = Producer or app.amqp.Producer self._heap = None self.sync_every_tasks = ( app.conf.CELERYBEAT_SYNC_EVERY if sync_every_tasks is None From 2a6065514000be6af9c0501eaf4def93ad343e2e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 21 May 2014 15:17:01 +0100 Subject: [PATCH 0166/1103] [Taskv2] callbacks, errbacks, chord and chain moved to message body to avoid header limitations. --- celery/app/amqp.py | 13 ++++--- celery/app/trace.py | 43 ++++++++++++---------- celery/apps/worker.py | 4 +-- celery/concurrency/prefork.py | 2 +- celery/worker/request.py | 4 +-- docs/internals/protov2.rst | 68 +++++++++++++++-------------------- 6 files changed, 66 insertions(+), 68 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 4a4fbc15c..7fc59c43f 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -300,11 +300,7 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, 'id': task_id, 'eta': eta, 'expires': expires, - 'callbacks': callbacks, - 'errbacks': errbacks, - 'chain': None, # TODO 'group': group_id, - 'chord': chord, 'retries': retries, 'timelimit': [time_limit, soft_time_limit], 'root_id': root_id, @@ -314,7 +310,14 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, 'correlation_id': task_id, 'reply_to': reply_to or '', }, - body=(args, kwargs), + body=( + args, kwargs, { + 'callbacks': callbacks, + 'errbacks': errbacks, + 'chain': None, # TODO + 'chord': chord, + }, + ), sent_event={ 'uuid': task_id, 'root': root_id, diff --git a/celery/app/trace.py b/celery/app/trace.py index d06ed9f48..c26961cde 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -466,36 +466,40 @@ def _trace_task_ret(name, uuid, request, body, content_type, content_encoding, loads=loads_message, app=None, **extra_request): app = app or current_app._get_current_object() - accept = prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT) - args, kwargs = loads(body, content_type, content_encoding, accept=accept) - request.update(args=args, kwargs=kwargs, **extra_request) + embed = None + if content_type: + accept = prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT) + args, kwargs, embed = loads( + body, content_type, content_encoding, accept=accept, + ) + else: + args, kwargs = body + hostname = socket.gethostname() + request.update({ + 'args': args, 'kwargs': kwargs, + 'hostname': hostname, 'is_eager': False, + }, **embed or {}) R, I, T, Rstr = trace_task(app.tasks[name], uuid, args, kwargs, request, app=app) return (1, R, T) if I else (0, Rstr, T) trace_task_ret = _trace_task_ret -def _fast_trace_task_v1(task, uuid, args, kwargs, request={}, _loc=_localized): - # setup_worker_optimizations will point trace_task_ret to here, - # so this is the function used in the worker. - tasks, _ = _loc - R, I, T, Rstr = tasks[task].__trace__(uuid, args, kwargs, request)[0] - # exception instance if error, else result text - return (1, R, T) if I else (0, Rstr, T) - - def _fast_trace_task(task, uuid, request, body, content_type, content_encoding, loads=loads_message, _loc=_localized, hostname=None, **_): - tasks, accept = _loc + embed = None + tasks, accept, hostname = _loc if content_type: - args, kwargs = loads(body, content_type, content_encoding, - accept=accept) + args, kwargs, embed = loads( + body, content_type, content_encoding, accept=accept, + ) else: args, kwargs = body request.update({ - 'args': args, 'kwargs': kwargs, 'hostname': hostname, - }) + 'args': args, 'kwargs': kwargs, + 'hostname': hostname, 'is_eager': False, + }, **embed or {}) R, I, T, Rstr = tasks[task].__trace__( uuid, args, kwargs, request, ) @@ -515,9 +519,11 @@ def report_internal_error(task, exc): del(_tb) -def setup_worker_optimizations(app): +def setup_worker_optimizations(app, hostname=None): global trace_task_ret + hostname = hostname or socket.gethostname() + # make sure custom Task.__call__ methods that calls super # will not mess up the request/task stack. _install_stack_protection() @@ -538,6 +544,7 @@ def setup_worker_optimizations(app): _localized[:] = [ app._tasks, prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT), + hostname, ] trace_task_ret = _fast_trace_task diff --git a/celery/apps/worker.py b/celery/apps/worker.py index ae0c8d509..d3c220853 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -112,7 +112,7 @@ def safe_say(msg): class Worker(WorkController): def on_before_init(self, **kwargs): - trace.setup_worker_optimizations(self.app) + trace.setup_worker_optimizations(self.app, self.hostname) # this signal can be used to set up configuration for # workers by name. @@ -144,7 +144,7 @@ def on_init_blueprint(self): self._custom_logging = self.setup_logging() # apply task execution optimizations # -- This will finalize the app! - trace.setup_worker_optimizations(self.app) + trace.setup_worker_optimizations(self.app, self.hostname) def on_start(self): if not self._custom_logging and self.redirect_stdouts: diff --git a/celery/concurrency/prefork.py b/celery/concurrency/prefork.py index b579d0e10..ef4de4bed 100644 --- a/celery/concurrency/prefork.py +++ b/celery/concurrency/prefork.py @@ -68,7 +68,7 @@ def process_initializer(app, hostname): hostname=hostname) if os.environ.get('FORKED_BY_MULTIPROCESSING'): # pool did execv after fork - trace.setup_worker_optimizations(app) + trace.setup_worker_optimizations(app, hostname) else: app.set_current() set_default_app(app) diff --git a/celery/worker/request.py b/celery/worker/request.py index 41b1d765e..7193a9013 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -179,7 +179,7 @@ def execute_using_pool(self, pool, **kwargs): result = pool.apply_async( trace_task_ret, args=(self.name, task_id, self.request_dict, self.body, - self.content_type, self.content_encoding, self.hostname), + self.content_type, self.content_encoding), accept_callback=self.on_accepted, timeout_callback=self.on_timeout, callback=self.on_success, @@ -444,7 +444,6 @@ def create_request_cls(base, task, pool, hostname, eventer, default_soft_time_limit = task.soft_time_limit apply_async = pool.apply_async acks_late = task.acks_late - std_kwargs = {'hostname': hostname, 'is_eager': False} events = eventer and eventer.enabled class Request(base): @@ -461,7 +460,6 @@ def execute_using_pool(self, pool, **kwargs): trace, args=(self.name, task_id, self.request_dict, self.body, self.content_type, self.content_encoding), - kwargs=std_kwargs, accept_callback=self.on_accepted, timeout_callback=self.on_timeout, callback=self.on_success, diff --git a/docs/internals/protov2.rst b/docs/internals/protov2.rst index 01b2e1325..626c333fb 100644 --- a/docs/internals/protov2.rst +++ b/docs/internals/protov2.rst @@ -21,7 +21,7 @@ Notes - Body is only for language specific data. - - Python stores args/kwargs in body. + - Python stores args/kwargs and embedded signatures in body. - If a message uses raw encoding then the raw data will be passed as a single argument to the function. @@ -43,7 +43,7 @@ Notes when sending the next message:: execute_task(message) - chain = message.headers['chain'] + chain = embed['chain'] if chain: sig = maybe_signature(chain.pop()) sig.apply_async(chain=chain) @@ -74,16 +74,6 @@ Notes return fun(*args, **kwargs) - -Undecided ---------- - -- May consider moving callbacks/errbacks/chain into body. - - Will huge lists in headers cause overhead? - The downside of keeping them in the body is that intermediates - won't be able to introspect these values. - Definition ========== @@ -93,35 +83,40 @@ Definition # 'class' header existing means protocol is v2 properties = { - 'correlation_id': (uuid)task_id, - 'content_type': (string)mime, - 'content_encoding': (string)encoding, + 'correlation_id': uuid task_id, + 'content_type': string mimetype, + 'content_encoding': string encoding, # optional - 'reply_to': (string)queue_or_url, + 'reply_to': string queue_or_url, } headers = { - 'lang': (string)'py' - 'task': (string)task, - 'id': (uuid)task_id, - 'root_id': (uuid)root_id, - 'parent_id': (uuid)parent_id, + 'lang': string 'py' + 'task': string task, + 'id': uuid task_id, + 'root_id': uuid root_id, + 'parent_id': uuid parent_id, + 'group': uuid group_id, # optional - 'meth': (string)unused, - 'shadow': (string)replace_name, - 'eta': (iso8601)eta, - 'expires'; (iso8601)expires, - 'callbacks': (list)Signature, - 'errbacks': (list)Signature, - 'chain': (list)Signature, # non-recursive, reversed list of signatures - 'group': (uuid)group_id, - 'chord': (uuid)chord_id, - 'retries': (int)retries, - 'timelimit': (tuple)(soft, hard), + 'meth': string method_name, + 'shadow': string alias_name, + 'eta': iso8601 eta, + 'expires'; iso8601 expires, + 'retries': int retries, + 'timelimit': (soft, hard), } - body = (args, kwargs) + body = ( + object[] args, + Mapping kwargs, + Mapping embed { + 'callbacks': Signature[] callbacks, + 'errbacks': Signature[] errbacks, + 'chain': Signature[] chain, + 'chord': Signature chord_callback, + } + ) Example ======= @@ -132,15 +127,10 @@ Example task_id = uuid() basic_publish( - message=json.dumps([[2, 2], {}]), + message=json.dumps(([2, 2], {}, None), application_headers={ 'lang': 'py', 'task': 'proj.tasks.add', - 'chain': [ - # reversed chain list - {'task': 'proj.tasks.add', 'args': (8, )}, - {'task': 'proj.tasks.add', 'args': (4, )}, - ] } properties={ 'correlation_id': task_id, From 24a4a3dc796fc13bd48a55c70229fdbb69290463 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 21 May 2014 15:25:34 +0100 Subject: [PATCH 0167/1103] Task protocol v2 document moved into single protocol document --- docs/internals/index.rst | 1 - docs/internals/protocol.rst | 189 ++++++++++++++++++++++++++++++++---- docs/internals/protov2.rst | 140 -------------------------- 3 files changed, 169 insertions(+), 161 deletions(-) delete mode 100644 docs/internals/protov2.rst diff --git a/docs/internals/index.rst b/docs/internals/index.rst index d10ed013e..4521a22fa 100644 --- a/docs/internals/index.rst +++ b/docs/internals/index.rst @@ -14,6 +14,5 @@ deprecation worker protocol - protov2 app-overview reference/index diff --git a/docs/internals/protocol.rst b/docs/internals/protocol.rst index 6b7360b31..9a12ba2e1 100644 --- a/docs/internals/protocol.rst +++ b/docs/internals/protocol.rst @@ -1,16 +1,174 @@ +.. _message-protocol: + +=================== + Message Protocol +=================== + +.. contents:: + :local: + +.. _message-protocol-task: .. _internals-task-message-protocol: +Task messages +============= + +.. _message-protocol-task-v2: + +Version 2 +--------- + +Definition +~~~~~~~~~~ + +.. code-block:: python + + # protocol v2 implies UTC=True + # 'class' header existing means protocol is v2 + + properties = { + 'correlation_id': uuid task_id, + 'content_type': string mimetype, + 'content_encoding': string encoding, + + # optional + 'reply_to': string queue_or_url, + } + headers = { + 'lang': string 'py' + 'task': string task, + 'id': uuid task_id, + 'root_id': uuid root_id, + 'parent_id': uuid parent_id, + 'group': uuid group_id, + + # optional + 'meth': string method_name, + 'shadow': string alias_name, + 'eta': iso8601 eta, + 'expires'; iso8601 expires, + 'retries': int retries, + 'timelimit': (soft, hard), + } + + body = ( + object[] args, + Mapping kwargs, + Mapping embed { + 'callbacks': Signature[] callbacks, + 'errbacks': Signature[] errbacks, + 'chain': Signature[] chain, + 'chord': Signature chord_callback, + } + ) + +Example +~~~~~~~ + +This example sends a task message using version 2 of the protocol: + +.. code-block:: python + + # chain: add(add(add(2, 2), 4), 8) == 2 + 2 + 4 + 8 + + task_id = uuid() + basic_publish( + message=json.dumps(([2, 2], {}, None), + application_headers={ + 'lang': 'py', + 'task': 'proj.tasks.add', + } + properties={ + 'correlation_id': task_id, + 'content_type': 'application/json', + 'content_encoding': 'utf-8', + } + ) + +Changes from version 1 +~~~~~~~~~~~~~~~~~~~~~~ + +- Protocol version detected by the presence of a ``task`` message header. + +- Support for multiple languages via the ``lang`` header. + + Worker may redirect the message to a worker that supports + the language. + +- Metadata moved to headers. + + This means that workers/intermediates can inspect the message + and make decisions based on the headers without decoding + the payload (which may be language specific, e.g. serialized by the + Python specific pickle serializer). + +- Body is only for language specific data. + + - Python stores args/kwargs and embedded signatures in body. + + - If a message uses raw encoding then the raw data + will be passed as a single argument to the function. + + - Java/C, etc. can use a thrift/protobuf document as the body + +- Dispatches to actor based on ``task``, ``meth`` headers + + ``meth`` is unused by python, but may be used in the future + to specify class+method pairs. + +- Chain gains a dedicated field. + + Reducing the chain into a recursive ``callbacks`` argument + causes problems when the recursion limit is exceeded. + + This is fixed in the new message protocol by specifying + a list of signatures, each task will then pop a task off the list + when sending the next message:: + + execute_task(message) + chain = embed['chain'] + if chain: + sig = maybe_signature(chain.pop()) + sig.apply_async(chain=chain) + +- ``correlation_id`` replaces ``task_id`` field. + +- ``root_id`` and ``parent_id`` fields helps keep track of workflows. + +- ``shadow`` lets you specify a different name for logs, monitors + can be used for e.g. meta tasks that calls any function:: + + from celery.utils.imports import qualname + + class PickleTask(Task): + abstract = True + + def unpack_args(self, fun, args=()): + return fun, args + + def apply_async(self, args, kwargs, **options): + fun, real_args = self.unpack_args(*args) + return super(PickleTask, self).apply_async( + (fun, real_args, kwargs), shadow=qualname(fun), **options + ) + + @app.task(base=PickleTask) + def call(fun, args, kwargs): + return fun(*args, **kwargs) + + +.. _message-protocol-task-v1: .. _task-message-protocol-v1: -======================= - Task Messages -======================= +Version 1 +========= -.. contents:: - :local: +In version 1 of the protocol all fields are stored in the message body, +which means workers and intermediate consumers must deserialize the payload +to read the fields. -Message format -============== +Message Body +~~~~~~~~~~~~ * task :`string`: @@ -56,15 +214,6 @@ Message format will be expired when the message is received and the expiration date has been exceeded. -Extensions -========== - -Extensions are additional keys in the message body that the worker may or -may not support. If the worker finds an extension key it doesn't support -it should optimally reject the message so another worker gets a chance -to process it. - - * taskset :`string`: @@ -116,9 +265,9 @@ to process it. Example message -=============== +~~~~~~~~~~~~~~~ -This is an example invocation of the `celery.task.PingTask` task in JSON +This is an example invocation of a `celery.task.ping` task in JSON format: .. code-block:: javascript @@ -130,8 +279,8 @@ format: "retries": 0, "eta": "2009-11-17T12:30:56.527191"} -Serialization -============= +Task Serialization +------------------ Several types of serialization formats are supported using the `content_type` message header. diff --git a/docs/internals/protov2.rst b/docs/internals/protov2.rst deleted file mode 100644 index 626c333fb..000000000 --- a/docs/internals/protov2.rst +++ /dev/null @@ -1,140 +0,0 @@ -.. _protov2draft: - -======================================== - Task Message Protocol v2 (Draft Spec.) -======================================== - -Notes -===== - -- Support for multiple languages via the ``lang`` header. - - Worker may redirect the message to a worker that supports - the language. - -- Metadata moved to headers. - - This means that workers/intermediates can inspect the message - and make decisions based on the headers without decoding - the payload (which may be language specific, e.g. serialized by the - Python specific pickle serializer). - -- Body is only for language specific data. - - - Python stores args/kwargs and embedded signatures in body. - - - If a message uses raw encoding then the raw data - will be passed as a single argument to the function. - - - Java/C, etc. can use a thrift/protobuf document as the body - -- Dispatches to actor based on ``task``, ``meth`` headers - - ``meth`` is unused by python, but may be used in the future - to specify class+method pairs. - -- Chain gains a dedicated field. - - Reducing the chain into a recursive ``callbacks`` argument - causes problems when the recursion limit is exceeded. - - This is fixed in the new message protocol by specifying - a list of signatures, each task will then pop a task off the list - when sending the next message:: - - execute_task(message) - chain = embed['chain'] - if chain: - sig = maybe_signature(chain.pop()) - sig.apply_async(chain=chain) - -- ``correlation_id`` replaces ``task_id`` field. - -- ``root_id`` and ``parent_id`` fields helps keep track of workflows. - -- ``shadow`` lets you specify a different name for logs, monitors - can be used for e.g. meta tasks that calls any function:: - - from celery.utils.imports import qualname - - class PickleTask(Task): - abstract = True - - def unpack_args(self, fun, args=()): - return fun, args - - def apply_async(self, args, kwargs, **options): - fun, real_args = self.unpack_args(*args) - return super(PickleTask, self).apply_async( - (fun, real_args, kwargs), shadow=qualname(fun), **options - ) - - @app.task(base=PickleTask) - def call(fun, args, kwargs): - return fun(*args, **kwargs) - - -Definition -========== - -.. code-block:: python - - # protocol v2 implies UTC=True - # 'class' header existing means protocol is v2 - - properties = { - 'correlation_id': uuid task_id, - 'content_type': string mimetype, - 'content_encoding': string encoding, - - # optional - 'reply_to': string queue_or_url, - } - headers = { - 'lang': string 'py' - 'task': string task, - 'id': uuid task_id, - 'root_id': uuid root_id, - 'parent_id': uuid parent_id, - 'group': uuid group_id, - - # optional - 'meth': string method_name, - 'shadow': string alias_name, - 'eta': iso8601 eta, - 'expires'; iso8601 expires, - 'retries': int retries, - 'timelimit': (soft, hard), - } - - body = ( - object[] args, - Mapping kwargs, - Mapping embed { - 'callbacks': Signature[] callbacks, - 'errbacks': Signature[] errbacks, - 'chain': Signature[] chain, - 'chord': Signature chord_callback, - } - ) - -Example -======= - -.. code-block:: python - - # chain: add(add(add(2, 2), 4), 8) == 2 + 2 + 4 + 8 - - task_id = uuid() - basic_publish( - message=json.dumps(([2, 2], {}, None), - application_headers={ - 'lang': 'py', - 'task': 'proj.tasks.add', - } - properties={ - 'correlation_id': task_id, - 'content_type': 'application/json', - 'content_encoding': 'utf-8', - } - ) From b3cd3fcec4e6be0baa68b7ec2fc62c53e7d0404d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 21 May 2014 16:01:29 +0100 Subject: [PATCH 0168/1103] Tests passing for beat --- celery/beat.py | 30 +++++++++++++++--------------- celery/tests/app/test_beat.py | 12 +++--------- 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/celery/beat.py b/celery/beat.py index 3579ecf4f..d316ac251 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -199,13 +199,6 @@ def install_default_entries(self, data): 'options': {'expires': 12 * 3600}} self.update_from_dict(entries) - def maybe_due(self, entry, producer=None): - is_due, next_time_to_run = entry.is_due() - - if is_due: - self.apply_entry(entry, producer=producer, advance=True) - return next_time_to_run - def apply_entry(self, entry, producer=None): info('Scheduler: Sending due task %s (%s)', entry.name, entry.task) try: @@ -216,20 +209,26 @@ def apply_entry(self, entry, producer=None): else: debug('%s sent. id->%s', entry.task, result.id) + def is_due(self, entry): + return entry.is_due() + def tick(self, event_t=event_t, min=min, - heappop=heapq.heappop, heappush=heapq.heappush): + heappop=heapq.heappop, heappush=heapq.heappush, + heapify=heapq.heapify): """Run a tick, that is one iteration of the scheduler. Executes all due tasks. """ + max_interval = self.max_interval H = self._heap if H is None: - H = self._heap = [event_t(e.is_due()[1], 5, e) + H = self._heap = [event_t(e.is_due()[1] or 0, 5, e) for e in values(self.schedule)] + heapify(H) event = H[0] entry = event[2] - is_due, next_time_to_run = entry.is_due() + is_due, next_time_to_run = self.is_due(entry) if is_due: verify = heappop(H) if verify is event: @@ -239,8 +238,8 @@ def tick(self, event_t=event_t, min=min, return 0 else: heappush(H, verify) - return min(verify[0], self.max_interval) - return min(next_time_to_run, self.max_interval) + return min(verify[0], max_interval) + return min(next_time_to_run or max_interval, max_interval) def should_sync(self): return ( @@ -477,9 +476,10 @@ def start(self, embedded_process=False): try: while not self._is_shutdown.is_set(): interval = self.scheduler.tick() - debug('beat: Waking up %s.', - humanize_seconds(interval, prefix='in ')) - time.sleep(interval) + if interval: + debug('beat: Waking up %s.', + humanize_seconds(interval, prefix='in ')) + time.sleep(interval) except (KeyboardInterrupt, SystemExit): self._is_shutdown.set() finally: diff --git a/celery/tests/app/test_beat.py b/celery/tests/app/test_beat.py index 62310805a..04a610df0 100644 --- a/celery/tests/app/test_beat.py +++ b/celery/tests/app/test_beat.py @@ -123,7 +123,7 @@ def send_task(self, *args, **kwargs): class mSchedulerRuntimeError(mScheduler): - def maybe_due(self, *args, **kwargs): + def is_due(self, *args, **kwargs): raise RuntimeError('dict modified while itervalues') @@ -273,22 +273,16 @@ def test_due_tick(self): schedule=always_due, args=(1, 2), kwargs={'foo': 'bar'}) - self.assertEqual(scheduler.tick(), 1) + self.assertEqual(scheduler.tick(), 0) @patch('celery.beat.error') def test_due_tick_SchedulingError(self, error): scheduler = mSchedulerSchedulingError(app=self.app) scheduler.add(name='test_due_tick_SchedulingError', schedule=always_due) - self.assertEqual(scheduler.tick(), 1) + self.assertEqual(scheduler.tick(), 0) self.assertTrue(error.called) - def test_due_tick_RuntimeError(self): - scheduler = mSchedulerRuntimeError(app=self.app) - scheduler.add(name='test_due_tick_RuntimeError', - schedule=always_due) - self.assertEqual(scheduler.tick(), scheduler.max_interval) - def test_pending_tick(self): scheduler = mScheduler(app=self.app) scheduler.add(name='test_pending_tick', From b21189f0f6516f6b162a5a1e8cd8ff9ca16e5554 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 May 2014 13:14:05 +0100 Subject: [PATCH 0169/1103] Enables universal wheel --- setup.cfg | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.cfg b/setup.cfg index 2a032e4d1..68aa7cdd1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,3 +13,9 @@ upload-dir = docs/.build/html requires = pytz >= 2011b billiard >= 3.3.0.17 kombu >= 3.0.15 + +[wheel] +universal = 1 + +[egg_info] +tag_date = true From bc31507d26267c9bbf93f4eb6470a0bc3f141ba8 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 May 2014 13:21:39 +0100 Subject: [PATCH 0170/1103] setup.py cleanup --- setup.py | 46 +++++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/setup.py b/setup.py index 2767346f0..6a8a370c2 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -try: - from setuptools import setup, find_packages - from setuptools.command.test import test - is_setuptools = True -except ImportError: - raise - from ez_setup import use_setuptools - use_setuptools() - from setuptools import setup, find_packages # noqa - from setuptools.command.test import test # noqa - is_setuptools = False +from setuptools import setup, find_packages import os import sys @@ -22,6 +12,8 @@ if sys.version_info < (2, 7): raise Exception('Celery 3.2 requires Python 2.7 or higher.') +# -*- Upgrading from older versions -*- + downgrade_packages = [ 'celery.app.task', ] @@ -53,6 +45,9 @@ finally: sys.path[:] = orig_path +PY3 = sys.version_info[0] == 3 +JYTHON = sys.platform.startswith('java') +PYPY = hasattr(sys, 'pypy_version_info') NAME = 'celery' entrypoints = {} @@ -75,16 +70,9 @@ Programming Language :: Python :: Implementation :: PyPy Programming Language :: Python :: Implementation :: Jython Operating System :: OS Independent - Operating System :: POSIX - Operating System :: Microsoft :: Windows - Operating System :: MacOS :: MacOS X """ classifiers = [s.strip() for s in classes.split('\n') if s] -PY3 = sys.version_info[0] == 3 -JYTHON = sys.platform.startswith('java') -PYPY = hasattr(sys, 'pypy_version_info') - # -*- Distribution Meta -*- import re @@ -123,8 +111,6 @@ def add_doc(m): # -*- Installation Requires -*- -py_version = sys.version_info - def strip_comments(l): return l.split('#', 1)[0].strip() @@ -165,15 +151,16 @@ def reqs(*f): 'celeryd-multi = celery.__main__:_compat_multi', ]) -if is_setuptools: - extras = lambda *p: reqs('extras', *p) - # Celery specific - specific_list = ['auth', 'cassandra', 'memcache', 'couchbase', 'threads', - 'eventlet', 'gevent', 'msgpack', 'yaml', 'redis', - 'mongodb', 'sqs', 'couchdb', 'beanstalk', 'zookeeper', - 'zeromq', 'sqlalchemy', 'librabbitmq', 'pyro', 'slmq'] - extras_require = dict((x, extras(x + '.txt')) for x in specific_list) - extra['extras_require'] = extras_require +# -*- Extras -*- + +extras = lambda *p: reqs('extras', *p) +# Celery specific +specific_list = ['auth', 'cassandra', 'memcache', 'couchbase', 'threads', + 'eventlet', 'gevent', 'msgpack', 'yaml', 'redis', + 'mongodb', 'sqs', 'couchdb', 'beanstalk', 'zookeeper', + 'zeromq', 'sqlalchemy', 'librabbitmq', 'pyro', 'slmq'] +extras_require = dict((x, extras(x + '.txt')) for x in specific_list) +extra['extras_require'] = extras_require # -*- %%% -*- @@ -187,6 +174,7 @@ def reqs(*f): platforms=['any'], license='BSD', packages=find_packages(exclude=['ez_setup', 'tests', 'tests.*']), + include_package_data=False, zip_safe=False, install_requires=install_requires, tests_require=tests_require, From 2a5a1409558950f1358b052a6f5b4e01884d8d67 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 May 2014 13:49:23 +0100 Subject: [PATCH 0171/1103] Tox: Adds docs target --- tox.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tox.ini b/tox.ini index a15e36635..9cfbd75ce 100644 --- a/tox.ini +++ b/tox.ini @@ -49,3 +49,9 @@ setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -U -r{toxinidir}/requirements/dev.txt nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + +[testenv:docs] +deps = -r{toxinidir}/requirements/docs.txt +commands = + pip install -U -r{toxinidir}/requirements/dev.txt + sphinx-build -W -b linkcheck -d {envtmpdir}/doctrees docs docs/_build/linkcheck From 506a0817ac553349715c371ffc826dd6aadbecbd Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 May 2014 13:49:31 +0100 Subject: [PATCH 0172/1103] Fixes doc problems --- Changelog | 2 +- docs/.templates/page.html | 2 +- docs/getting-started/brokers/redis.rst | 91 ++++++++++++++------------ docs/internals/protocol.rst | 4 +- docs/reference/celery.rst | 6 +- docs/userguide/signals.rst | 10 +-- 6 files changed, 60 insertions(+), 55 deletions(-) diff --git a/Changelog b/Changelog index 7c7e35165..c523701e4 100644 --- a/Changelog +++ b/Changelog @@ -109,7 +109,7 @@ new in Celery 3.1. Celery), so if you do enable it then make sure you do so on all nodes. - See :ref:`redis-caveats-fanout-patterns`. + See :ref:`redis-caveat-fanout-patterns`. This will be the default in Celery 3.2. diff --git a/docs/.templates/page.html b/docs/.templates/page.html index e4d1c2132..7562de304 100644 --- a/docs/.templates/page.html +++ b/docs/.templates/page.html @@ -7,7 +7,7 @@ This document is for Celery's development version, which can be significantly different from previous releases. Get old docs here: - 3.0. + 3.1.

{% else %}

diff --git a/docs/getting-started/brokers/redis.rst b/docs/getting-started/brokers/redis.rst index 6a1d6e31f..543f4ee90 100644 --- a/docs/getting-started/brokers/redis.rst +++ b/docs/getting-started/brokers/redis.rst @@ -69,72 +69,77 @@ Caveats .. _redis-caveat-fanout-prefix: -- Broadcast messages will be seen by all virtual hosts by default. +Fanout prefix +------------- - You have to set a transport option to prefix the messages so that - they will only be received by the active virtual host:: +Broadcast messages will be seen by all virtual hosts by default. - BROKER_TRANSPORT_OPTIONS = {'fanout_prefix': True} +You have to set a transport option to prefix the messages so that +they will only be received by the active virtual host:: - Note that you will not be able to communicate with workers running older - versions or workers that does not have this setting enabled. + BROKER_TRANSPORT_OPTIONS = {'fanout_prefix': True} - This setting will be the default in the future, so better to migrate - sooner rather than later. +Note that you will not be able to communicate with workers running older +versions or workers that does not have this setting enabled. + +This setting will be the default in the future, so better to migrate +sooner rather than later. .. _redis-caveat-fanout-patterns: -- Workers will receive all task related events by default. +Fanout patterns +--------------- - To avoid this you must set the ``fanout_patterns`` fanout option so that - the workers may only subscribe to worker related events:: +Workers will receive all task related events by default. - BROKER_TRANSPORT_OPTIONS = {'fanout_patterns': True} +To avoid this you must set the ``fanout_patterns`` fanout option so that +the workers may only subscribe to worker related events:: - Note that this change is backward incompatible so all workers in the - cluster must have this option enabled, or else they will not be able to - communicate. + BROKER_TRANSPORT_OPTIONS = {'fanout_patterns': True} - This option will be enabled by default in the future. +Note that this change is backward incompatible so all workers in the +cluster must have this option enabled, or else they will not be able to +communicate. -- If a task is not acknowledged within the :ref:`redis-visibility_timeout` - the task will be redelivered to another worker and executed. +This option will be enabled by default in the future. - This causes problems with ETA/countdown/retry tasks where the - time to execute exceeds the visibility timeout; in fact if that - happens it will be executed again, and again in a loop. +Visibility timeout +------------------ - So you have to increase the visibility timeout to match - the time of the longest ETA you are planning to use. +If a task is not acknowledged within the :ref:`redis-visibility_timeout` +the task will be redelivered to another worker and executed. - Note that Celery will redeliver messages at worker shutdown, - so having a long visibility timeout will only delay the redelivery - of 'lost' tasks in the event of a power failure or forcefully terminated - workers. +This causes problems with ETA/countdown/retry tasks where the +time to execute exceeds the visibility timeout; in fact if that +happens it will be executed again, and again in a loop. - Periodic tasks will not be affected by the visibility timeout, - as this is a concept separate from ETA/countdown. +So you have to increase the visibility timeout to match +the time of the longest ETA you are planning to use. - You can increase this timeout by configuring a transport option - with the same name:: +Note that Celery will redeliver messages at worker shutdown, +so having a long visibility timeout will only delay the redelivery +of 'lost' tasks in the event of a power failure or forcefully terminated +workers. - BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 43200} +Periodic tasks will not be affected by the visibility timeout, +as this is a concept separate from ETA/countdown. - The value must be an int describing the number of seconds. +You can increase this timeout by configuring a transport option +with the same name:: + BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 43200} -- Monitoring events (as used by flower and other tools) are global - and is not affected by the virtual host setting. +The value must be an int describing the number of seconds. - This is caused by a limitation in Redis. The Redis PUB/SUB channels - are global and not affected by the database number. +Key eviction +------------ -- Redis may evict keys from the database in some situations +Redis may evict keys from the database in some situations - If you experience an error like:: +If you experience an error like:: - InconsistencyError, Probably the key ('_kombu.binding.celery') has been - removed from the Redis database. + InconsistencyError, Probably the key ('_kombu.binding.celery') has been + removed from the Redis database. - you may want to configure the redis-server to not evict keys by setting - the ``timeout`` parameter to 0. +you may want to configure the redis-server to not evict keys by setting +the ``timeout`` parameter to 0 in the redis configuration file. diff --git a/docs/internals/protocol.rst b/docs/internals/protocol.rst index 9a12ba2e1..c51aa396f 100644 --- a/docs/internals/protocol.rst +++ b/docs/internals/protocol.rst @@ -161,13 +161,13 @@ Changes from version 1 .. _task-message-protocol-v1: Version 1 -========= +--------- In version 1 of the protocol all fields are stored in the message body, which means workers and intermediate consumers must deserialize the payload to read the fields. -Message Body +Message body ~~~~~~~~~~~~ * task diff --git a/docs/reference/celery.rst b/docs/reference/celery.rst index 0363c446b..75541f059 100644 --- a/docs/reference/celery.rst +++ b/docs/reference/celery.rst @@ -382,15 +382,15 @@ and creating Celery applications. Finalizes the app by loading built-in tasks, and evaluating pending task decorators - .. signal:: on_configure + .. data:: on_configure Signal sent when app is loading configuration. - .. signal:: on_after_configure + .. data:: on_after_configure Signal sent after app has prepared the configuration. - .. signal:: on_after_finalize + .. data:: on_after_finalize Signal sent after app has been finalized. diff --git a/docs/userguide/signals.rst b/docs/userguide/signals.rst index 00dab2dd9..fd6dae378 100644 --- a/docs/userguide/signals.rst +++ b/docs/userguide/signals.rst @@ -88,7 +88,7 @@ Provides arguements: Task message body. This is a mapping containing the task message fields - (see :ref:`task-message-protocol-v1`). + (see :ref:`message-protocol-task-v1`). * exchange @@ -131,14 +131,14 @@ Provides arguments: * headers - The task message headers, see :ref:`task-message-protocol-v2` - and :ref:`task-message-protocol-v1`. + The task message headers, see :ref:`message-protocol-task-v2` + and :ref:`message-protocol-task-v1`. for a reference of possible fields that can be defined. * body - The task message body, see :ref:`task-message-protocol-v2` - and :ref:`task-message-protocol-v1`. + The task message body, see :ref:`message-protocol-task-v2` + and :ref:`message-protocol-task-v1`. for a reference of possible fields that can be defined. * exchange From 100168871a3bc4607bc07b11d7de6d7c72e8cc4b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 May 2014 14:44:09 +0100 Subject: [PATCH 0173/1103] Fixes docs and test issues --- README.rst | 4 ++-- celery/app/trace.py | 4 +++- celery/beat.py | 2 +- celery/tests/case.py | 8 +++++--- celery/tests/worker/test_loops.py | 2 +- celery/tests/worker/test_request.py | 1 + celery/tests/worker/test_worker.py | 2 +- celery/worker/request.py | 4 ++-- docs/configuration.rst | 8 ++++++++ docs/images/worker_graph_full.png | Bin 101018 -> 107927 bytes 10 files changed, 24 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 8e349b866..392965271 100644 --- a/README.rst +++ b/README.rst @@ -234,9 +234,9 @@ by using brackets. Multiple bundles can be specified by separating them by commas. :: - $ pip install celery[librabbitmq] + $ pip install "celery[librabbitmq]" - $ pip install celery[librabbitmq,redis,auth,msgpack] + $ pip install "celery[librabbitmq,redis,auth,msgpack]" The following bundles are available: diff --git a/celery/app/trace.py b/celery/app/trace.py index c26961cde..3e04628a2 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -491,9 +491,11 @@ def _fast_trace_task(task, uuid, request, body, content_type, embed = None tasks, accept, hostname = _loc if content_type: - args, kwargs, embed = loads( + X = loads( body, content_type, content_encoding, accept=accept, ) + print(X) + args, kwargs, embed = X else: args, kwargs = body request.update({ diff --git a/celery/beat.py b/celery/beat.py index d316ac251..372441221 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -478,7 +478,7 @@ def start(self, embedded_process=False): interval = self.scheduler.tick() if interval: debug('beat: Waking up %s.', - humanize_seconds(interval, prefix='in ')) + humanize_seconds(interval, prefix='in ')) time.sleep(interval) except (KeyboardInterrupt, SystemExit): self._is_shutdown.set() diff --git a/celery/tests/case.py b/celery/tests/case.py index 551d0dfbb..a05c8c5a1 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -869,7 +869,8 @@ def restore_logging(): root.handlers[:] = handlers -def TaskMessage(name, id=None, args=(), kwargs={}, **options): +def TaskMessage(name, id=None, args=(), kwargs={}, callbacks=None, + errbacks=None, chain=None, **options): from celery import uuid from kombu.serialization import dumps id = id or uuid() @@ -878,9 +879,10 @@ def TaskMessage(name, id=None, args=(), kwargs={}, **options): 'id': id, 'task': name, } + embed = {'callbacks': callbacks, 'errbacks': errbacks, 'chain': chain} message.headers.update(options) message.content_type, message.content_encoding, message.body = dumps( - (args, kwargs), serializer='json', + (args, kwargs, embed), serializer='json', ) - message.payload = (args, kwargs) + message.payload = (args, kwargs, embed) return message diff --git a/celery/tests/worker/test_loops.py b/celery/tests/worker/test_loops.py index 4030782f4..4473eb47e 100644 --- a/celery/tests/worker/test_loops.py +++ b/celery/tests/worker/test_loops.py @@ -158,7 +158,7 @@ def test_on_task_message_missing_name(self): x, on_task, msg, strategy = self.task_context(self.add.s(2, 2)) msg.headers.pop('task') on_task(msg) - x.on_unknown_message.assert_called_with(((2, 2), {}), msg) + x.on_unknown_message.assert_called_with(msg.payload, msg) def test_on_task_not_registered(self): x, on_task, msg, strategy = self.task_context(self.add.s(2, 2)) diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index 2700d26ef..7a202fa72 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -619,6 +619,7 @@ def test_fast_trace_task(self): self.assertIs(trace.trace_task_ret, trace._fast_trace_task) tid = uuid() message = TaskMessage(self.mytask.name, tid, args=[4]) + assert len(message.payload) == 3 try: self.mytask.__trace__ = build_tracer( self.mytask.name, self.mytask, self.app.loader, 'test', diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index 5ac5f6a9a..ebf4425c6 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -912,7 +912,7 @@ def Loader(*args, **kwargs): os.environ['FORKED_BY_MULTIPROCESSING'] = "1" try: process_initializer(app, 'luke.worker.com') - S.assert_called_with(app) + S.assert_called_with(app, 'luke.worker.com') finally: os.environ.pop('FORKED_BY_MULTIPROCESSING', None) diff --git a/celery/worker/request.py b/celery/worker/request.py index 7193a9013..c03b42d54 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -207,10 +207,10 @@ def execute(self, loglevel=None, logfile=None): self.acknowledge() request = self.request_dict - args, kwargs = self.message.payload + args, kwargs, embed = self.message.payload request.update({'loglevel': loglevel, 'logfile': logfile, 'hostname': self.hostname, 'is_eager': False, - 'args': args, 'kwargs': kwargs}) + 'args': args, 'kwargs': kwargs}, **embed or {}) retval = trace_task(self.task, self.id, args, kwargs, request, hostname=self.hostname, loader=self.app.loader, app=self.app)[0] diff --git a/docs/configuration.rst b/docs/configuration.rst index 864b255dd..2aa37164e 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1110,6 +1110,14 @@ compression schemes registered in the Kombu compression registry. The default is to send uncompressed messages. +.. setting:: CELERY_TASK_PROTOCOL + +CELERY_TASK_PROTOCOL +~~~~~~~~~~~~~~~~~~~~ + +Default task message protocol version. +Supports protocols: 1 and 2 (default is 1 for backwards compatibility). + .. setting:: CELERY_TASK_RESULT_EXPIRES CELERY_TASK_RESULT_EXPIRES diff --git a/docs/images/worker_graph_full.png b/docs/images/worker_graph_full.png index 867bcfb7429cfb6bd1611dacbda6a124d78829a9..38cb75c902b5f9076ba54adc12d5ff387e1bd66c 100644 GIT binary patch literal 107927 zcmY(rWmH^E6D{ERa~Ku zGT}Up)SX^q+)q1P%o`V0JQ_PzbQl@taM>lGVBUR-7m!#-A%O;c<^T(^lAz+>>dsi& zUf!%kG{mcQ+^#IXzEs_qPqVVP+wOFvIy`Aw?sPn!wB34$r(OQ#?JG4baS7E(fx^%S z$@n5LK~r>tQ+x&CIq;1#cx2u_{NG=Q5o$ogMBxADOJOv9*cpZ4z5wI@yBrF`&KL|l z%uIOA!~id|Eml@o_{gLR+2CY)1>V)eZGzI-hLn=hBOUKw2(V)#@Yr%T#giBWcub)1bwkIUT~S96A|9O ze?M`_=uHe!XP3(EO3BF7QR#$n=?5Q02)Lcg+B-QEQW7qcR8&{AFU%fXSf%)~f#=}V z@T-=}VK7`z1AeJ!&q{>tUuwEJt^I?RU;YXjmAzI`R+K{V{LUg89FpdY*8CkL)4dKH z^ze)b`dto%AMNbL-qt=XW_G+_maeXa^QhMs=r*#k#YtK6d(H;WeIBh=*Q({w7|P#( zIGw>7e$&zPw@3Qm1N^l8M)q4}4M#fIATf@)BezzWZXS?ufUpY0CDT9PX@5lrf3z({ zn+2L;?F?AwbQc2!qYxmV=flr?+xtkWKOC`vJ~6cy|ELdgmd&Z)$aZhz-mi~2{>-la zp}*#^Z)fleG%`FOQfQTviF5e&=M|DTx17Xy=AF0%7PwrF6-r-Is}Fuf4_amAG{tbI zT>(PEv2MOWZ@SkfgjUlZ-R!GO?FNH(8Mnt;E@!DpSd}&Oc_}k?VE~;jQH(85ixvj4 zn~jzQHQ0GS9AiAaDT6})-P=;%KH1c6R}gC>^3@JE95pG0AIP86-<^lZ`yW^g1NY;n zf{iCjOu~;cw^pl6513shZ&s5c3TU$4?uQ|TUAlm+9ue>DhRwyzO<_vl)*|H{7GMJj z=v1^r`F<;esi)I@L0#SyLBHWrp)i=jpk86WAYib62Vo$z-`@yYm>=f^UeU>Q@}Z#+ z8GsDozs{&B>o}3~K8>-(xe@D+aC-lGvuLJdDB7AG^vxR?0aOmJ8ycN<&j9qvz}0f@ zuVF30(2Bs;q*w0)Jp{8Mo zdx8-bi@rnxT!R7~_tU4GjU`J=3<9ym({Jpx(7;K6TWSaZw?xW0Cy$oSX)8|7!tt)= z>wLi^NZ$HM?7Ab~0+B+y;MByTfefZdlC?!4F^K}K9(KPNlGofU5(N~|2#>F~ zMFr;LvyZm7O@i;5rL+IxNEm*y4tp0paW7Jly2nu5~@q7y|LXa?75+Z(|5Tot{6HpO4?%rwfM*ATo{*$;i8w2|_vkjnmD2$p= zU?H}_X2emZfQLBc^L^nmCN8=Z+W3zfKAkKgj1*SCjr94glLI;>6YF#*y(Q>3k?2zZ zBKa#4`j7jQUoJ^rvhH*2yLX}JO3VZ{FFXbm`V4*t;pi6D7N~uQ&hatL$t}qa9{*i% z{>{?x@$~dh!8bmh;o;#l#>FXVnME=4%-fa;Th{f2;+AgZ@Og?Pu^4fC1HdVI7@p)dlg-3>EEBX#dE24e3{XgPQBO5NQg_H>bcB5^>x9()#>RJ!4?iT$ z8(gRVQ1>4n9SLC-!YXI&V#suV0$J0`s_3YR+@37Rd>LQg`8H(zpO=&YUZRE-xfB+M zu(gFXo=dh%{e?B5-P0496N^&GsxkpZ?+01R6P@DAwY1~iujY~lrsUrI@6-CKb9Scls!ohyV3mwzrHF2^9J zre?*5_Pz%9;SX`+>rKY^vv&`0Lx(r*FFp{;lNh)VPNbx9O2T_kON+-Ct%@Eftktdp$E=KVYx#hhfpI0xh8F}vK_f9=1qb(5S z_)MBMwlkK+zHj?-jRW~&8)oHly@YUac9m!NxRQv1ESV=``!})v>V@`Ix>415qm!RP z<35K+n2UR{w53)zvO~O+{o02JQS@Q@^||HBGOO8<)hWW8?_~{{ivn|@+UJ&^n%*+* zsI0Uz2}p_6s5_iK^B;Q*!W0e|C-A=944X!eCwE>8uV;RFijDQ!f3KRH{Qi+3!q8!d zG1p|h+(W<)9AW!%Hj7CH(QCt8w}9E6o}G;#bYGKe%OI71OR$E3vCZHf{?4EB?Ot0W z=6&6O6C!aazAJD37-NTX8X;yn!=YCb%7w{`W$;jOyLsfK!Q^I@eO>Si8P5*0>)PbJQ z>8^iBOnZA>L#X%NXjcLM%f{o?vtAcz z)tZ{Gx9^#t$vg;LY_a}D+TI*5>5f`!X>$jBeY zg?;X?LNCG{=Ga11uR`ZqLn;;;GS|dCZua=@315X<%Ql`qX|*{O^Sl$ip6k2~WPTb% z+s2~Ikyw5Gom^U)n^dLCLA^-R@8WsDD7LcM7tfl$?V%xJN()>sy#jbeF%1>uOGjWVigv)!X3ne2zB*}?wVZ2dw)dby1yGWnO| zw{zLmYP4MwwtAmK8c1VoY*DJUMRGx4q?Ow+SxhW63l!xI&NqCf53=%OJ=-`Rqo=f^ zf21JbcazjY;q3^fcSyQ5d8AD|wvGfdz~-;Y<1UwAYGd`mDk(lzpFUp)njkw(PmQn8 z#5}^ii=czE+5>1~wh6PP9tghk(^FVmQ-Ug|`@L*<(V%E^a&qCRzH66#5!Lg`{`D)>lWw)LzAq!M z-6?j!zu=I60DA0rb7Eb!_+0~Zzn2&2_1o6;Z$U!Xv`X^x5yTwjrL}Tex(04b&j&<< z)7BcP{><8{x-t@tth%TJu;qJu#N>st@-E3s{vj8Lnho{=OLF{VOYnJ{@g-}2R38qS zUxm%(6n}(WTu-oSoAnE46L3c3D7@rQwYH&XYxHMK?a2lX5JY?-#pt~&vw)xR&yf{z zZ1+UYo*x8TE6xci>W;sg$9Z^gA3Vl3+~xwi2u7;F3*$Yi@DB$icD9u#bl$bdN8Dei z|53@f;{fD8L*C6-GqXMFQvb!z#hIw6^?_oYizy}}^JsH3X5fYU#4AvV1t;9-6=`7h zpI)oj=ZFTmyJQGo@=WMgwc*)ilKmK!L-bSZ&|^;xoOKv3t(>dpWxbO;z2~!2@eggS zEoe)yYhu15AO%F6WvR>{v7S7CO{?!^5}Mb9Q7+KFLeJA7Y=aPGGJ>@8?*oxdQp(0w z9xJYLoS%I@e0U$~sOf{PrXn9Fs81afD6S|Y6h0txMo1FTYu4g=z%(5y&>TdXIb-H$ zUwsEh+iy_H=e_22fn(l4;h?RsfVWRclS@K%*7Mk`?0BAjhDWHK_z zpR)NyB-@Uq)~i#KJ9J(}wR`=Ut6s|vr}q*?WsQ-6K3mO@OsZTKI)i)u_3uTL%EI`o z={mCoI;TR6=T)?ffzvG|5z%CyR$LU+bJydqTA-oEMTGp!JtU)zaCB*A{-|6r4dxQ*XwPZ-ULjY|1jQ25lhqJfKR7o|TQ)k9O?6b2Nd zv9%u+bkRI-*7Q7OBW>rLHsTn)1G4V_&hEkaTxzDGzsFYDiV^**PF2uG`0|vZ-AfZy z@m}Gqn{dAqbr06~mPPN8a^L>T4hW{xFJ71CNmsopMLq)C2-of%j}ZW&j#2?2eAC{_ z(~Jgcbc(wAvk2Fh^dsZ!-+O#cn`*|F&Fn06?G=V!J>Qx?85!5Ytjj?~e9YU%y5B+Z zCco(BBMV325kGCetKj*BPAIAO^5~QACap$0y7g}>(rrQ&$_CN0>Afit?y~;N1S>Pj z!DZLOm**SGS1S3QAcMZ(HhA;ak;K>>jCZgCG~XZ7d%sXvr(eN{`KpCxJIKmlBb)C z*Td!2#ikx}I>|7#TZv?l&L57KR%y3aZ2 z^RD;jt3GjB>G*rA-nYS5$CagYSXiB&V3tQWCK~#mf|Yr8zh}E{sDi*K+CBXaYq$T^ z0@~;)ceC?J`Rg7m04LCX|5wn$#wK}jGrFi9w0jo`-dv>}$ej`sDNlYFT@hI=Uef{ z2Yutm`ttGOJq5|w0ChHz$Ym_N`*M;GtjoAU4Xe>2bO3Ah!m$ zP39pP`MP^oSE&CLV?b$ZiAZi^hzr6>Y`l7b6BntPFuF)5(TO*p=z}o^eMH0cUA0_s z3Xh1&v398#-(y&!e-n~`*V~@|ARY~9epM%g?k7RFgR}e+?`-DHa6Ppt^l?E0w+A8t zS9irOG7`-uDUxRb!2RqbfK^;ub&I&G5lY*d;Dlj-| zW&eZ3eM{M;zC0>%{_+@5zCsKTcMfW|vaAHw3f1}ZR$c_9FbltH0zai>3CN04H-2Ru z+l)E-tVeXZkAJE8R_*ylzp;y(ee1ch@@w6>EGh~b6i>SK1AU3*r1J6+|G2pPZf3B{ zsOSOdR}~fnnu+)KT$%z#y7pFw-hsFd&XB@~%vb%CG1F6|0Kz6Z+;4XjEJHfBIx9_n zKTAzUAIDayXEDLL1W*~!6gj4V1`gbuT&)s-!uhlSNB6dU{%`+Cb`yyD% zW+yfz1fTb5O{W>bj6p_+s}!OK0P{7nYz@SpDHkb!l_Ce+71q%HU4j%Q<<;nFgcj6`~m*-Zap5k~w4 zd~6x9woNCOMliM7EXL{FH>Ce_pa(LboO_uG6#t6gmo($-exAsSiB2e(8}~_&7lD-T zH`oRPwTexY)NHtaF@f)TklU7MSN%U<;C?FvGwBL=L5&_XKk2>D%!Io8&6-Ne1 z56){sP`?8g(w72Njv}k5s>x)x+tP}P2g}Y!%LD4@21m*OMoLzlMG(``UI?+bbSNpR zOtUy>rRLMT>u|rHJCAg;kkP*ZyL)()aPWs7kB`4h`HL0n1SMVmkD>%~E%-h7kU=jToH$ zA^e|q(cZNCTeZIaOY?1$qTHxcX*(A;r?R~S4LMx#q=V0r@bYmLugCyOQ7fSv)PXk=F;|7 zuC|HAQ!kmu>nO+MFE-cC9W*nWiekcI3#}&F_;);Q|3AUp4Sp{HNC$=? zZ9L@LxE2Fl)oe*qWw5`OkeSYeSU8dlYGY%@))a%@XnsO*wSxU0o{Oss<_AZ&Cm8`{ z5bv^%3XN1B!?!FEL1-Jty&Z4Fn-94yahFtlOp22p4!i$pUHh#;ptePD)(Jn-s^yfL z#)VJKMI78zWSbAaTH9H9jVychp#4Zq$CczS^ot-CT|N~dMNZ$$Ztj$oV<7va?NH9% z!Kv_1)w@<*uf%990t%lPX$W2%xeFTvPGr(!H|f7^p3&_Na1RJ3PS0G8ZUmqhK(l4l z3d-vF4Gn)Iz6l8G_Fxpv%&y9J-CYk)%`eOEYf1+mcGrkH3kmbRMTyS}S}Ef6$gEe_ z)qGO9Bz}e^4L1#ucS#;?j|ysAo>B;s`1AX?lPRnJ+0=V6KxzT>DRIHt6Dpsv)VJ*W z)OOlSU@`abzHVdcdU-Hr@+S+|hLvue>~k7(^Ys9pswl5csS3fIFw9<&Ln=aW6cLL# zWK}ngEl5shHx?a9#?}POyaimqVx@qm=T~J!_v_4S2`1ArN<>%+DWLakd|cz4;VO9V z6B=C7j>oBItKcR}zU?_xma&n&gO2*{aDX#w>*P?V(3E?mLu>km(EBgur&B@JmgTQ= z{ExC(u|_Tj=0pWI3oq3Dco}8EZ29t`8iL_N7kLLqHXrxWpK--+tt)2_ zXBmn-nrxZ&oin`%CK_{cNV0i!I>hMeDhAR5WAK5@k;C#9>#PJLc`_cNxms1+bl;iw zf&Yy6?)52aeQSGY+cWXXdYYY{mHhHj3&Ca!$$K`8jO;cs3`sl|IajnevdST^9j2!L z`pd+yC+eJ&%7!lQafi2Lh~sA0hw6zF#!U3-fcmjb%PaLl_42 zasDf%=Sk%3oG=VkTM=V#I(eqp3+28wSE2T1=A< zRSy>mR1+^i-1`f#RemCL6JbjU8HxwUWH! z#*gS;&tKHPxgA@z=cYNS>T1l$ma#GCH~Yj3BA}h+zJoT7EUI|$082c2OSm`9MQQ)w z5vhkdgxrF90~vb;G2c=BRG_#xbSQ1BmNp^Qz;3gbI;_}%Y0k$dR-HJ{Bb!=W6~zv!`=UL7Zfv5o*)g9e`?#>HT{%SGhd7EEI1P+Dn zHHGeHN6zRBI#T6r{u&OeHC;DIS<+(SsZ9H)3$&iuUOQiopN~YcG-NCsGE00I>LV~; zEFj9)6?e(T&mI7yHjX zbK?!-B0{I4$y_=HwMliz7jcTBx;aiKjr z4n!b5sIA*o6!cQ5Pz9N&fu|6%3ngr7M2_?}+JpDxR`rZ5TP)jZVy6pn$M_j8fbKti zUHDrV-9DAgg>}%kYkwybElILiDtZ##K=3zP3pMis5jNk#-Y-R1)%K=vI5P5dc+m2h z$9p18Wj8Jw?ksj7bIrWv>1z%A3pqQS0j#)X^)iMb#_CaX7bS?^RZ5FW17wgmw)I8} z@f;yhEn}|RHQItu*xcDRML>fJ8#Bx-(4_88)zy$Tl&1`6#6N1G){EUT2iW zIY{8M2C8j1)#H5}lBreoSKmJ8PK60O>z}s23;Ng~P0?o|JCpCBFGW zEa_0jm4cZbz5iHj{Lu;LZA~79*``BC2o%uIJW$e=kDN)YjF!x`%=nR6)ksN!ZmSI&waPlhbiT?_(Or${QIJ^w$?H!UrH)c@LQ3M!#VxQtSsM(pj6I4C2-e zM3<)AOvM@Od~sZjq~;s(Pfz=hcRX$tXm#a-#~0m{Y?1R_lJCGu+~!i1TO++>pp*8i47#n!%2HKkz$ zB#hR%rU`#Yg@5-B&IoE?DD{VWBRY`qG69~jVw#3?m851KVmVY$_s)RiURJ!YOW-I+;h^o7L+@z14V zYBWe=wfsQA`9y5g$HZQU1JU)IQ6wh)%mU#*h2oL|@dk(x?`DEuZ!bw{2R-I(C2`1% zC={0>#aMTf{|f-qZ$`wWwCNO@k$QH2h~vgq>y?HJp~F(Y(=nO-me3+rxIqKr$eF4$ z6NWm3N`}6}Aw%TpWmihaHy@*2fh>dC;NR$F5R%FSMevLVq+SQJL*%tk~N+6An-2{KU zif->ZlU3!uJv0&y!iTOiOraUjb1%HeQ>m$~*RKM51Px4S0eE9z?JGU~$WFAZ?Qw2J zmGpwd^hPf9y!6UHZ|%Bt48B=Z8z%S7!Ob_=?GBP75bhA3WM8kp5zF~m8v8dXD}J%u zGJ#EksAqUXx>MC^l+?`9d#V>f6!9p^k8mfT;1$S)QZr(Ix9o^t$?P3|6eAC&v-XEC zL*H5OH=HRZoH`Yp87c-h&3JOOKk^LU%nWiVXv%%*5D+p#3bEl`@;rfN?m1w$WBPB|Ul| zrvKJ5pym6X^3#^_2T_}0pvP$hP%PYdSlrxdBjm)RHRW|Ix0{RaP`X3-INsTyAo~3$JN<;0?ssTx2)Az` z=+y}GMnDZzkZkxR6Y44Ebg)&=WJVws2lpoQ+vNWev-De~L(LiRE#y56(drWAGhLfI z_qH=59kG(%h6=lyrl^&RA+o}0s{g^(AnEhlw~k3;zTEMpsk$Ko!4H) z=?{NT)n%smuBH2E=v)4209;Jpm*D~w2LN-!QA7nBZdm?ASz@{6!;6IDKVv>usO9la z%vI`7CrNzCGv9n>fb74| zuf3$faY#~)Dazmaz{N@1Lnqlw^TCp?%#DD7gUXpU8pD((g`qU;<)Q5P9(43EV;6uZXe__=9X#&3IAiTZxh6;W~=$cI%vsJ)|+rBRCP|SBo z>s&DGqUU>)B`6vE6MZX`MlUK8x&HWXZgu}J0?&xJ#C)lrOP;ZpRSiR2Uoo@Z`aCRe zeI7B92V1C!RhwE^myU7}Q&@|u0nnPhzLoNc_pdy?Qx~*Y)sP5{P z)0$+95bI9hD?He(ddL!0tpwVgY&{QcYUbQp-O{Z6`nJ!6Kb#>#I1Q#Hh~U&gJUl>- z%BMpk9%SYQWmRSMqPHaiczXX0$)bKPgIzfMwRU87J}uK-i%ek!hyCO%ogS`Q zdm>nyMk&RpSKcaf+_7UUR)fV&wuHm!a+>9UvQ9$4S;4uSgr*kNKLDJ&4M-gWj;%QMKm-+>pM*4BU$= z;Kw;{a}*>7zS&rL?>A45nvGlWL{9T~j#>+%AfGZUe8(66p&B3ygy?q?c4D`n>SyYAh}z|UoQEIFPPjwq6I3|=K(4GgMFvItS~r*2z~`1Q zVjd;soTRlfjgNKvHk3#BHX|3Jnqu(220IgT@GQ=RVuD|*D5KP-JlnR)q234d%#8UOi=&XDflyh z<>Zo)lSnjlVlX}F=&QI{P7^uN<9r-C6m;Y4gZsF^_7>F`O$;5Pc;f`5qcQn)jHo@@ zao&opRN?ECm>Lq|F~;6CCk!W~G1e6gROQ#yLfHddk>9#A@YU$c3?CK0Ap$75_^I_9 zS4vlM>tkC#+lC7OSvbHz)NuMaA{Fds5od{^rvCZ3rkl6?`k+6EfA|-Bh!fW`9+PC4Bol;t+==Q1o}20Kxr9ep(TM8 z0KW*JZLtZ=b?GzDdpymiiJ8&}M}MhF11c1>mPXEr*$+eT=)2sln5>u*+p6;3U3^M@ zjY}Y;ARaD(pJ<^K=Rq>mTIM)VbeXSFJrss-DeI`%?ZuRp(5+#m&ogvQvrV4z=D z%Y73EGDGbJNe#OTyJSS3?h+hC9>k317*4-64{zVwYurDfUsHP0LF`^D(;EYVqg%l6 zod{o@$M2m!uv-@t`h)rghmX<$i)Q<#2eu$!=s&K?)zgxAV1GZTb_R1r}hF~aa= z{aroB+WH2Gk8Q5PdQYgR2k0)dgY4MGVy`5D0d^o^v=?yXXCOPwh*{E3G9v_)ih<*- zF3AG%CZ2UF0OZid9yv%O>`=wg9rc}`YVCu*kU%jz5C zD}NiXJQ$EHnx9W@3sI0YYqP@>P^A}DpgFHm(9IpOLA@6b zY?6#fy+F+!C5W1(9jg59p@Y(g^k?$Y{T5ZeiML4IO&fc&>qSLdpt<|k69Y5Ds8 zM_AHUe)F(RmkEv;(gLWT3ICEljFSw#Y+B`>3bYp8XI>BmFhC4U{n48iEDct*P)rh7 z%dGUGAS&wzdBE~j50aEBH`E*(6pZ^jUTdmFN#pTC2#)+AVH0&)0~;yEfjor+&fVX6 zL36T##5+J%aV)#z&?)cy-jRNp3Ao%?22}C*#HTNV^c9<;A%$3$0F@L-Uv=)!$Wbty zM%1F90w0d&leSte0EGyqTUzS^>IuZF_e#0ykH;YITm=vyff#=RL{rmUO;ghDrSG84 zJI(umdEhdKd%YcK<~&i>x)m>o*{r*jRLGe7QO~n z4Wpu&TNIPIsYam*-CHw(y85hxn}+#AjtnIYUu}sr6t&Y^I&pIbDs*777L9SDZJ#%W zn#SJg#wm>>zcjt?-EZmbMkN*eNg?d;1}&ZM8FR)4wS3S{)-%uMuV1peBezhj11n&lh^=dzIX`>T$DA&DusD0)C+@FNEu@wW4Af+ziN* zrqTW%kqNE?J&WA28QB>RvI>!)cXsy=Bt!MZ$I`H;Ph!82@LGK^R@X}u{V>&Zao`qp z#mrXl+0p4o!al+}AxuzG<+{5j0U;a)Q#O>g{-`P2CRXTz*cGk65UK=I<^>`W{XuQD z{KD_f6qb0_YC*YUr+t@5eIm?If!5F1lkhnyFnLenrUOjJQin|!n1Znq+{M9bm$`|s z*qg(cohbv)8R{;=QS0_Z*r)>LHczruHkAcrd27%#r~lnr2B1dD+HI%=YS?dK)-9)L zFVWlfBq=Pnl&AJNf<(DSfy(AxmQ7Zf zhW1RMjv~S+4IUDn)5!ogUBARZx|3kj9Dk;5aB22#rB*S;E`n3Y+n z;)AGLr2CTDq7bFwXB$sU@v2_9k^b^Y2SqNqz< z1$eVH+1BTJoqhq z8>dCTEcfgOEh_vCdXh%&Z|{sQ`FAdYVXB)BMnSok^eMi-fPn3cO$6;}p2fgP!h&2O zqajR9V$O0CY#Eyg92lV0-uO&hT-=-*2os{DzMf{#pb9@sT@eFlTwy{0=6d129r?qD z_>d4d@=Dn>3s+CikD$!aOcnRssjan-8v1?J5Jxgxy=j}YMHx35p(kA-zfv^6#B;we z$2aRG$(N)w|{e3cZW#cjRpoh%Kmb#4jJ;m$+ZIRSW{U)!AYaoxC$pZ#J zZxm_veanU3njtB%QASbs2d###R-3k&+Y-&?=}0LO;DIlQm3UlHT_jzz@xCF4b}h!N zO6CR`a}y}%2v~{-liKH|+{IUCGI%_{-3&~nl`;>}B770Uj7Q1q*8I3S>>1y-k_4a< z(jMv1v%re@)Y7zwOTntO{WhHu#ZQJIJd<#N=zUN-9WJ_FFZdI%i5-dWRVYuF1p;#j zF|o3=p9d)Z@E1-}{#xT_Yxh;O&bS5!bF$@%e;BHh3@P3u%|<{&o#L&bj^SY`A2IS3 z3W<&xS^sfRWw5k`qM0%dB_41yzj+K_(oj<3<&9h=`mn2*qUbE=xWPQtN(RbfI6Q3pnb-v9C&?xNs#KR`Sx%a9RKQ@?UOiFkDT_jkG?_L#8lu? zhn;YU<|j9+caAngU=w0u7=Zfj&Ld4Ok0?J2sLWdbThEQ0y$l)qT1xjr=y|?Y<%3v@-benKOzQoNb!xi$l(XVSXvDQEb=|G zy#9B)I?)zToEucn$I{&W4AU|2%2FUgiZB$mMq@Z0)h)o)F{7=ys7nh5acDf%m5ZL} zN1YaK^BmQlZl9MI^la1&pe;_%n#U+_aejdoelzjlZJ=wWSu`Zn`3C@>1t^E_lVv!f z2Gm3(>#vkGpnu4HP!V5@g{@a$ z*Gh3AQok1?0B{B)GQLkWtj7x`#bc{SZXz-Q*>8_-WH`$g_5-O52^B#~A2QDJ^r2{|1JeSd?p?SvzPIN1LuOvCEw>>hek-CAg9O+*<@vG}%68VJF z<^T}5{~MlX`UKQWw)C3fw%o_Vi^5WV{>0V#s@NEDBcWy{0R1X}531=Fnz{)H`qx?= zLNslWfufd+encRoxM2%hc3r!d&|MbT2u@DS2LQjr?SVs-t4g#mV9K>KXS|EF5OsA^ zb#ph?Vu2n0y9`DAArnps>4{Q%CAjS+Z z9sX8Q1(j7%G*xN}s&ZIP&CbdSR;Yy)lDR;rO^>#0VrOhq5#uG`g|xCcHR?F2isrj!>-}{He_OX?%W&|_4y-;?RsWT1VsXs&xc+qf_|NO>7`s4KR@D|Tjop#i z(eT=Jqde0@w{ZQhZD3k>F{WqRg;Ckknzr(_*Im`8F#35NcN%AUa`KeF(xR(rucE#* zVdbfxz0f}N@iFjD!B(8{+Ua)idhxC>s^K6^wF!42#e*%0Z32JQJ;2$^76)V@2VScZH_!HSqid%)M2AZ~>DbF{q5hlG z=V(GtH4MY`L>aIY0BbX46ERh#sbPHV2;RBeW%*2pS#h?x;u%k&%FM zeuiA1dossqmvhHv4m}XVTT0?%rJP1_*hM&gQ$#lt5z6Y9+olrdy=a!i|C$+D*rapznTF$u1E~eNcDs)_A5zi5Y-~=g5F%>*7mRFhC0X1vHJP z9mBXixCi=|*`}L6yxuV7HasMCEs!S8oMSj$I_77t&Qm8Y?Sz|@uF+UaF+wI1Z>trY zs?L60IVbO`@pZ;O{d!KMP}=9cexhD@x-uzi*OHuntS`;|>F2-UZ+4AQPHm%@an;6q zIe}L;gx%R}y?iyp-y!-E_cAe5qTzS?u%i?BsQ0`^!`Z?y-ehZ&aLZ);o-E+YS;U_@ zvDT*-<8VE6d55NvR7VQJPQnSVwS_E86>SCSIK1ZOy7ljN4?&YE&MlGpgUHe!@*A$r zzxwn)eevpndl(ruEn(|YvuTE-1W!)DB@cz*6_Pyk3d_I5c^;S@5uXa!O6exb&=Tqh zW_kbeF2X%sU;R`>j>3=|NRBWOw7GX0GjSo$n0y%ePf*z{;ZuLcsDs&wjzi)J>+}B3 zhL~5LSt#~*K;$FP+`>RnCe>%=mT{~j-mqqY;zjE4J1?csv2>$-fDJ>mN0;P=i9QPk zg9!{BqNML&Q9apr;m`musK%m^&;IHjjrzzRaX-~)-p^e~Z)3(0k}SU~O2enCy<^wP zftZkcUTI(hu_Sd1S0{jtiWx58i}(OP7p4cyuJPpMRMe z>*>(z*e}PcQw~20j`!_C`T4{1{n|xP+l;>?xBs-QKD=LLYu42f?tbkB&qAeIElE)hv-n`?%A&3swJRc#shvbo&tH+=-qm)%w{Z_MG!S zZ)fj4pLWUXxo=BPU!oD#s5ll$&`8&;^`72@nP%sxzuu^pF0Ea= zy4`x50{&W2#G0)`M&FkdY2cYH<{pznC1Iwtmtycs9WVUY^@lxnS=1dI=UqgAy)~ix z4oRuw1}-z6=iMoh6p<-adChNKx6^P(-n%7-Y_hu=AvP~#jOT+Dz41PFMk&~)!qy-} zd2A`7#kSWMoB4OI0gE8WTx+@$W)?P|z2{TJ(2RGfcfOWeEB zpNM=iwN7;kVb$=TKx z{^WHTd;{0QcinkUM37mmALW&K?Hi(BOo;!_e*q*EXav+Wnp^Li;Z~ssRGK<$Jss{A zPBa{AFBTZ9wqOrod9tjND!3LYaMJly-CczP?;*jVCsU%IAO#IzhUtA`tjFkP$N4^p zE7KCAyV#A6e2zePXc1q}AM#?o-6ebHa_7>^DgsZJYey}y3gH8`=YT9$>a9v;y<|-g zK=ngHBBW1sv+ll)Ey@^rd0ekbkk8@uy_STLvh_}I_+*}UQ(&IDeMEYx!v>E<&ZJzfCy?YL!n=?o1IYX+N- z)8zm^k3b~0=%C7s@fD{Fqsu-eQWEM<${cUlK%4jn(oV2W;MK`e@ZvSU)-~C}Wdd&L>VBKFk#{O6m^AnqSW{Tg~F73KCP|0G#xK4 z)swbFPxP$nY_y{k&!u;4G5lehLmK_r%9p$qiD#U%QaLF(K3mRBMcuFnC@Xi|Cf3{Y zX_pPj9z6>Fbz5`ihHubS8&p5Z*FdmX;U034rkzffs4jA)rL7+pICmfo7qiOPqxnezYR>aD|~`o1?%7>4c|5RmQ?QNW;)?jCxikuGVNAwM*TG$^15 zl0!EPrL@v5-Q5lM@cljadG2%npEG;yv)5kvzU#~pCp?EWOHPLsWabLi-4)b6gvs`Ykmzg<~`=a+<3@>{4yiM)z)402ey@WhqXyBw3o{dctY^mJ{q zO+~%+_IpVW)`MQX9||x2W8nyv?0KS@xO7m}g|lrZl}JwssOP0Xwu-e?7$tqwGcD&Q zCbwcB95(w>AiHyN52cl=<5Y}24dT6KhYPMoTcq@wNKoX#9{2SY^zkQZ#?YtRjSDi> zp8Jz8XNHewxbay@l9VQPV4x)TiX1-tU54#vXj~;VG88Tvk!E0eh0w+nH5{c=!t5Qc2B_bs#WF?D$ z%bSMF3EAyE8vAVd4=*{pd!R64vTa&^@<}5+Ns=sX2*_;T(pAge1-Z{f6mzyeJxxKwLKP-_$L!| z8!ge7_4WCQg(X~f{EYj}{qL2r`G2bMw-+oduf<9E;^3Jp4?IIacCkL8(r|~rR>NHo zB^=~frQ==DxS{jA1=MV5syX-Ke}RA68~ji`i?k08zOP!3hm&=Z^kp7lVDFOab%B(Juyfmm{wSe`2|bmX2`DB&%Bx}ji1r73t+vj@*l*` z&FY6Hwbr{~tv?Sp)8LKZT_r@=g679TAp?_sh!e)Q)L$~6YeP>@i_N|UoCf^x;|uY6 zxBZPc=$0wl|9&;CTBYX%7gNO7=`@XBqXz#zQNvin2@=eCZn7{di|@rpH##vUqD&8! z)+iymyCz}M-}S)by&f|s_z^b_*(0e4dLTYXEu-VOKLGFDZT(App94o5Ni6kuU=zDo zb!Yi~#Oz}w7_ffvHJ3Q>6usqU-fQjkY1lHx+lK|UbOSVOeX$B9@&9s41l(%5gJVct zxZr{@pY0Zjcnr}92&-V+>Y%n*R z$#=ci%vSI;fJ!27GOk5M(Vp+3Dg2B!AO8NS(0C9W?I(9U8~)H3$n{9Wi` z!%uMxT(Ap%$ zrdYss+3?*L9tl2-l6{p0@oQvJ`4Z9T%o>%Q&nkBEP2VXx)dMT1jX?T9x2EB_oq91KX# zyDo3I{`KdYTf2JMw*r%XBbNmg)DJMsb#N+6Dv;FT*QOn<#4^U^?8z<#Jf-SS3qB&ZY=ryfXe!8!Re>g66Dx8TDs`KWxu?zgrD!7ls zH<|GOw!OQDOs|nEH}*DGk#q!U5R~MimgzY^{AKLxn^8(g!HwcgO|2QnOAA`?(Zt6U zzXMRb89~p@073!kJD8vVwH&>@I2hnY>u7S!eih+AlD@UbsP$74#`nMS&|8p#{sup{ zm4S@3zqF$@qWg9(Yu-{r!^wo`M%U1yUAG>4!a?fP9DtrD?}NELdQAk-BRF83n2-kg zC1H5RDy?>1pTw%KsnqH{(4Vx%)cX_NWl@C>61$tP94i)LC*LM#=S6nrCzram_3kH< zrf~K5Gv`EiQ-c%!k;=tJQ*OjOhtac<|9y=v!-cd_R@aoV5pz_TmNqRjeIx%?w0Ws; zjYcLW85=ln1vGSH@4~a{kp(amc7t{8@Imp!^&%Pf*m=yF*0q=J<3O}Us-~54*ud5f zlEiyiMUo8^t_iC!`B=rIzAm9tyRK`VV<*yc7g7CoqLAYI*1LLl_8e`HNu3*?Un8M@ z+o*5a<;@DS*2nSLzr+yt0fIlDQeZ~$GWP$@$0S;b0RYi79#Ea`bFX-UF2jke*%R`} zz2)VLF=ck!?w%${VG|)DxATIPzL$V1ZypJ$z^p+g0Vb8CBgzIcB$8oD(AtkIa2^?? zI!?h@<*&SA?|9kg%4-)=cJ38XEpGZMs3V+BQ$(2O8g!Vc!6b$9`GN}%#o`mrI~aga zBM|vWFF-B;Lcfib46PHIkIE6x8 zg^esZ_LzEDj<{>%GGLj>k~5S2LMy}cn!!^IJ>n0je{9QJ9zUg6b$3Ai2V6)|Km z^sG=KRat1khe)T%BL0&`gSf{Pf&<-2QHMa<^ruu`O) zsXTp?IBpIUhXSBY^0q1@7GR5jD1_l`^@SCv44g~TD%J$b21&hj^PMYRspB;wWyw@L zhq|6E@5a?vY?uWaZZ1VjRKL4>!^|<3f7Vz0J#=3J>X|L1n+bR&o_t7H4wVFSH8Wh@ z%9O8(baSuj^}ILIY3sc^XTds+(fb07?u|4Q1=!;!O4`TM^<%!3D^CG+?GX_Nm_yS< zo)56K)Xs;Z=Il8ZpuHBy)tx1qGtzMHh?%)C$#f6jOS%vHAC_#2V&CXdn^X@ zpdq}}#M=5`j{+3rN3vfqd)#xpT!2HN7wXVLV*M|uni3Ml`&bV9LGB{d^7EYUNER;W2M{VK z*2&B62ur=O!A2;&Xc)foDB{fHRKv(5lTzL>s179Kb@bJm8RVxy88z7}*a2m5{|;DE z;fpE+CIqkKvL;1uiuwWS9$uG0Jk#VP z_3?HaDo0n>m-+D_X~O;%HPT4T+p!Nb#kiCP1BsFgjEx3~>ZL-Z>&DjOwfw8h2O^T! zap7vws*^FwaA&l$i!<$R-MjU>GgkEqX^b(B`_XUSjnSkTP~GCvg5<31?B3bIYjNcM zGC(|tkP!MfmPAi6RgVH|s$G<*48D~3kVv1J^7|z&`tHEe5=JZXFw&E9-~Caz0ToG+ zHm)_ke4Pm$=||8cSZw+m*cXd*yO8DLYXjb{7yT90JV#65eTzh|m7n>f*{1ImjBrv( z?|XS3<$`hkPL1!ny2OvGt?11%Aus!|_j-33RpJ`3+;=ldu&{CEWHB2K8zo%Tq#(0( zSw4!dZhPVzwabp>7un>AY&=EnVMKlZJDwNhf5V7*wNP1<6O*e6I#I{bUwp=}oDv}^ zOgtOVWs!(ytYQxZywP8N|5srkb2@;A{wbtmP^#SGrA>fgqtPQjmY_+rXIX1TM|7LB zZR{;NfPDHm-GSFX-f-G#C@S7U@;3~G-!_?>EATuv4buRyp?IvR3|i3N8c>CSJknXIl5pMsapqIME4zO+BBme>89eMI)FE6?*CSxo9@HQ znHg6UJ854wG?!-d*qegA!2GCj*=$OVV=@L9=gcI?)a>#UAx^b*;e1 z=e;!(GZu5$_x)a=0`B{bk6r7&3suD$bld5b*_Vgc$V{ff_KsYd;TNb{fhr z2~;WbIG=R|ht$DXc||j^GY%^R8S;H#2T2rQKWb~r@Vg1yRcaL^oiK)y{<)csfj%7A zB^d=I2``4INMZo3qrBUqVNBqs#*TalQz^lO;-6K+6EaWSgX&4g>LxYay++ekZsi`p zf}K@34a2j34aa%+`qA>XJRo6Bn7h3@PJ>w$Gu70Ge~tC!z7!Q$gP+-Voc>K-1ttI4DrGG?pBM0@Pj`08icHLoQOU zQmVw7TvwvZjSB}bCH%VRkkk=qQ89~e$&wpjyYCx zQDLBe|@BGjnfQxbd=| zNm#&#k4a)xU{lx=Q{ssl6lWx;e19V3x7D@vd;o<81~dK3LFP(6;eU9EgAT^dt27J@ zD3g*K=(GeFW-uYlT1D`yv6@LOIp~iwYnz<6?UQx}b4o=HKNkxeF-kK@Bk=HdQSge8 z`tO)}9O%Y<&+uW}aCJYloT=kg~W-6`P9t9M-Nuwc6(~}q-7-3cz+E$T<^h9^B ztiC-c`ta{x5+<2odQ3g&E#G<;($_e9>o*t^vh^1k_$Vy_IWh`lGzOKN!dmLU{1LD# z$Wmz;IiSt<0!Kh;s6V6}{vO{@IZ+z#J^M5uL zC+OysbBZuTDmQc<=d@n*MI4tbCvu)yS=GmVE5@+_ogC=}%HT#)4qB~l?1!6$d|0L^eCJ0`iM!GxQ9^RdH_0I4x^1M73|BT^aer*bQf2P;4@ zX5RI|7y$B7ERz?_vK_k|C9qkYm7tjE(NmyZ97yVTwBQDf^k@djn7WUm=AO8=KTyn* zu-G1Pl0AlR`~N*fiqw;pC>lOX`fm2Kc@qxy?)yL{$;RIUZ+B}qH)^e_lc@tPE6vtC zF(gPGu+LVmPpk%RM_&Q9dO06oHTN*g>OCkUUqzHn>-pHkYWQq zvu{tr(|663E)lNDhm7sG=3b7+aT4-{a3X#AtaDtVsa$b6d1j|4bLp19Vllz7NlMII zEpq?QuFR19RY8QEjsF=nlN+{w;qiM%NQl*hy-Hlu|1&UyfsPpM50nyghzO(Rqv91L z$$`R6G3q~eCqkz7*-e^m7}>~Qr3hEhA}XEat|J}5-U=UfX@>gVf3G-!ro#=v)!H!G z1WBOGVYspy=tK%7Z8L0rW76@n=2GVU6ZCG-TU(?G7)iZ7e;q}Mdx-bFyP1Sfz(r;#a0lf`dfP_*ANsGAlXpsL#^7kF$DK!lv*eLWymO3ot{jW&I z#F|Lj!o2(#08&qDJtwMhN8$l)7e6P0=FQG) zY6XK=A$RZW7PS&>=Kq_1M-ZM7QLGKh;6!@v)d;E*8^LM}&H$Te2w;N&x+=#d4S*5_+NEHCGvhaF8K}zwg}t8-Ee`p22V0VZJAkdSw8yf{80M>0$?$lKMs#cRwyjk} zNgmnCGn~jgZ}d;GehofG?)*fav;*vyOJLPQUuwkCeg`4|0BOHm0)D9pBjs}dD@P9N zQ|mSwo6K!Omze&^30|8Uy50^A)fwXutYY9l=zoqEQc~MO`-5h3gW_hWdzGk$-}H)( zqKM@A#Gmhr6b8xoPJWM98OQb;il$3{a5Fk~ukEZi6Q>nH#WVCI;EiY?52;H_GByMu zCrA1OmA?YQT6vA-jirRCcW@uc`2kItH4Ws1xYwYXabdx)@(A2{O$U1G=B$FC- zY8wze89Up1^@qCqzCjDx#A#D7m{wF@S*}QvLye9nG0WEUlaUT|19-Vu_%@z{w#_#c zVfx*t%eK6Z!e$OEgOO><0HeGAnQtaxsIvnONbUp?c8q5wF!1{o9-tg@J@4Se4l0s# zy3vx1&|_$fJen+F(L1~8)^_m~!Y}dzU~4JK0L?-@vm9{LUJ%2XVgnwgp#mA80-n(S1+NXI$s zeOJ@fiVz>N%_}$4PLl6*0<(Wx4*w{(SX-Mr(q;H=m5+NE2~CA36;?C|+p#n>`ThlJ zf|i5@_vgTz61$79TXL>(S-iZjqA0O{yAU}-Pi0|U*b|IW#-7Z@iiHS(S1=w%qZ_f;@Z-tWJABxryf6$YhNvJyP`a76OL16!$T|hE}xPkB|t^oNaQWkRL24f z>R#cAtz03zR@8~oq_pw7&!?S1p8W*N#>Th6Bo`H6&HvogT9+QSqR{4WViqx?PYrsE z?7MO_>Pe`X$kD~qkS=~W7vG{%+)M)0d<1@wqV{RIi(PevM!j8mZWwclK(iC0>^~X3 z5#6<5Etyd}r!LX-^qj_0rK;-Wekz|9v!Cw{8g5nuU019{VKU?>I({>B7qkQzlAYqYo{tW|44K&ZAgB1kN%THimM})DXwBO6^;I7j7ATd zZeB+K%PGEFMq=& zlGZ=5jstiry?c`~%q@^86M~or^+ZCz#ZH>*|iY7C7SD$;qW3n2YKg+>Q_(!qSkjTpef&ZMnUq$_*sCF zWKWil7u0vwFAM;&Z#(VTbd%&pmz``B(Z=gfTgRasB?uS!=USL$NBsbK*cpck6hr&g z=1{1NY8FQ)Kq$JC7LkT7}Mt&wBGH@g>hqLNc>;VkalPq{6o`V^OPtz4OU8HOH!_2V$rp^#<687%in(DmRe8!dd z_20u8iKdM@_ISme@M6KqzuIrxew;ZHOtyJ2QPcfBtB>7%D1yt%L3A3n$%1HG=9-f` z8}FVhQOQmgmGu~paJrz(=@I|#O|OS}7DNu1la1tM-wo%k%Z!bVeslXX^YP-Q?LLXK ztMll^j8<*=KV5BD;tsN@Ji2-Q>_L9UG=;=20qAOTTK_{HWV8$@m-^@!7Yn>A;;dP> zSCbI@Px&_l3MOMZbO{`jcK?yzR4Yi1pw=z^k60iWM7vJTKM!A&OQ88w_V$Y}Lqeq| zL!CgJzqEtR050Ro_sU({&78`9tKIXQ3*tJW-vxko{;a<$YKdP$q@Nzq1^z`U>7*;R z0mLM_qU89gtQOXc8ZLf~zDmVpcr?^fi!T_KsJK84)yge4U?NdDps#SME(xSY#cj$% z5WDKfO3Ggm^m>xb1~Mv8y+$uydxRX71n~;k4dMA+;Xku)@AqxLm2<>@1uB&RBpRxS zy)B0Gx+9O1k>3N}6hJoLFh%ocPQ`AFeRWJ$@ViAD}3Hn3ZMDt5MYytRM z01E6OBPoknYxJY}xmy>~NiPh$1N@R5jVibuAr*UIZg6oJ;Qg2@_R&EXU~R!2EFTO&`;c&9Cq;e8|HfSq)C+)bc=+9=5KxKqWy#3{&GPm2RDz1@FxQU+5WP4((pVVRCFw-nxO$R)he4Qr zmnde&K@BHxYvSY77Y0`Lo7=UImo_PLNYBH^ABL zeW0|D(lIRLv}VN=o_i*Zb=966@(59VoRR?;Q%>2QNBs3YE+c;XV$Quk%n!@g@_tg` ztt$hHqh0YfA`i_`toG-SWp4o~CUx^WNq&0v#{&VRX3hBm;sHG~o(2{F7FvJ)9Dxf4 zlGF%(aE!pa0^unGyzK>|I#1kgMQz!0gwgeD-h>K8@F718kClC$i+5Lz62Blk%dZb}Eh6~7 ztM%lu=DaUn}I)TFo zMM^zHS_eJA=eM@R$yv{GbEX)t`)17&zZPuG7~RbMe=xQ73mFF+fR1Xzuw|2%$t`_VxkuGyP{zKY7M538e@rtqJ*>zSK1>{#;I;%C`OCxFN#r z=dmvv53$R{QnGG!K-p$Wr#KGXf0 zn|Jr9;2%I0r3j9<{oWSvIX-*$#fmdZvW(=b25V(L))F=?Es4Z*^<^3*jf5(J*R|eSC)~=3fpQg>@$Ei)WBc}d zp%0@|q|ga(6SP&^fP7C7ve&%oBI;)|4EHU#+kMOpou zJ?tCotv)|}PAgSB%-@*W*Wn&p_)P$stfxOHPgnH7g}z<4WLu6B^EdYBC+w6Sle!%S zlY&PSEfYp0v&%svfFLOvDorWFjqE44+u}(5RJG$mdpY~X_nyKr(D#L|Y3cXfyXi5F zIY}v3sndsPKVsCYG8U5y`JpF_f@!!|35;p(Ki);+eJ-(OGpe;Ts0`j~YVw+FIw)P3 zZ|c>Y<}s>u%VEqp>157k)h&)cEbb6WiZ{%q)(B!ATr<0ZbwX))bC zI6c^-_wg6sSc}1fgo8NrWHFzOv<0r>db5*0pJ;8|#+?-Mm4|V5u~^5VVh)Dj&I!&o zE%wCQ+&jp%e7?TMs?}+>qU@RglUb34g*3?s%=2GmtRnl0Uun9>sg`1fu^UGc6~M38 z3{2G1m!ZOL>8eMFHl0Ge&4s8a+RKcz!ey`KsoKLapR2V(UWvV4{CWV!!KL z!a;uZKLyr&w%npZyyBo*nj`X+L_bC|9<&qR;U(pz{{<+bx*u?7u^l!A>iXSbJ+uD85JnpqOvd?^TembmLvnedm z%7iEbdo=ZTft0C&%WMY&co{Mn+76$CIx!Movfec~`Iy;r~kfa0EliCwZu3tAFnE zQ;rrbm{1)9%V)j8Ak&YH>dIMmU{?ornS_+s;);AyRl^ag7y0uLk>K59{230>SPJ2$ z*Vmg+-tYg=|4~=D#1sjcpW|GHU2^r?>0_R|ty|?c#~FzedEL#LWb5f@N_KP4)O_AZ zwpei%G2a%`W=b*cU)WzUWB9%CaQr&X-}Rk=6QASI!JBvv312cP&8NdJD=(~8?ikwQ zEzIaUE2&bJ5=H1Wqbe^^eb4c6r$XR?c)WbbbLg|C*ouukvBSet)XIVQzpLDFoypLJ zg20F}i!PYdI^2br)?dts##3f3GZZV7nfT$#%saOf|V|j>;$@^Zz{C(=lD^ z3eDtTyxa3Ie&Z)ZH`A_8T+~8xeA=JatYlKLUlc%^F~VDhog%n`c#<82y_!`|a|ymK z{Nt|twDl9b1$VOmH3G6fsHxiWJ{~mf$c47L_QTA5C?9@?7hwt2UYhxM`!U)C5p^BY zJg@Eyay|c6_56KzsFP#9l&n;kmgySCNkyAz_}?jw@%UTkoB&VZRfH}$#u7n!fgowq zxb*S7v@jB4B7cc@NTkyjJS}Y(b{+PUBy!C&otQ>`SG=Hq;d0b7`8HIQtZvvV?nGsHXmL z(JVB|j6Gtj&qVsK2!cJdf$t9_cSqvHm_f<=6*5x`l<-y@~^KJ8AorST%t*|Yl-chbi{ZZd1ZZdpi%ollCkJDEqHtUF6ZH4!eb{=!=qf?5Mz`yO zH)&9#Br1+#lk(R$ZZ7`wu4lH&!HyEA*D^StPXd(~ugO{fa;@y)E_d@%3 zXDud8(m^-d6aP7vFf&4?YHNjyXSSyc%NiiMXUS<90y2M`2xn7)9+7Z{GJ-N~8Qbrw zZ(ROwIHrSEyCuf>K!mmV!HwTrn*=XTb+MX^hgE;x+u@F(la!F`pb=q*mR1wTO{EqV zU0Kcq1++6=!vWJl`C9u1kFuK?S)yni$h7W*m#zpE$Zphlp+wliz!=ha-f2N5K!zwj zdKtq?Y30g^*j64_(xg=0e(_b}n>lgRTg0y99*vA%M^jHkHfM33zxZWe$mLe}SM&BH z{4wnE-T!JR0067#<#7mgodX^FytpD)b7ccc3u`1bpU|w7gAoGdTAtnPajPgI`C_Pr zX`{k0OFk)iz?d9cTwpIJ&H;f&j?Yf)Up^C)TemEAFh85SXm53jDscGsNzl$p{uE(< z0}0aQijdeT?zC9LkQF2k>f0#iqLC92V>8;`iA%pLI1^|2Vw<)?pNh$SGh}uOk~JOSVs~XNYOD6D89k z*I^Czd}vS$o1VGe*1J912$G!(-RFO^%*^gNdGE$o`|#&u^0YEDi2^J2QF->&A&MpV zzj1wOuKoLdr>uV-jU0(SabOQNNh*+u>l=BSV{b+2>pl^mtAta+bJ?SG^WFOsr!{+B z6CFa_X=bSs3}z7H>&CT;*@0+mGmqrHpR>N%$1KLEA~OqOqKR{uuccEZSCsQd)GCr^Nj_ zJ~y*_x>VJ8Oi2ZrKIG1=X01S^N4;ZAPylbXGR>=K3&fU?WoQIC>%6nurP)`y)nogf zdmHs!O4v)$;O=BR!6Dj2#@gq`$6`&QysPxVgNGZBXK`yUGKWcOep_L8I!OHTk60Ab zzPU;CZW2lA3kVsK3Y{By&g}Tt6(UhZeeCC${_)`D_kUlC1RHw3<-!lqbgfQKz%cRV1-;CfI`Vb`5#`wdDQH{J1~u$p-QPyfK6AV^0?_uQ<|?=oHZc z)rG?rNEE!e^tqO$8D-~i27aKTXr&bQtn>}IM+aZH>$6YXniKBdKJ4o#67%*IhYoz2 znol~&np?@5o2%g@H;e)fMt~L&(c0+w+diJa;>1oNuV8y6byqC5`<&Zt&C^|FVxO!E z;z()6{QPm(Ez9WY%knZ3@#393mM8ViVlBWNf~SoRll|f1L?^`Z)?2Sf8ziV{isCv5 zv)|}EA!gl~SZ!i2{^K-V#hBU7T3Y8&X2pbV!z_t8k*=5I&V&dgLqdC=Qie%(*z|_y zREN?^udEBQVuVs7NGW_zoU-CzbjjkaH(6SNZHfnaJmWr>vCNU(~G1t@+rge_6q%;6pCfdV~!|wE|Zs04w-&e&v6#K&161G-~^xiYzQv6pph zg9ZZ_D{$-R#o$3Yf#g;!Y$BV*UF9cu>rJ$$-q|14thT2~{l_ z_b~$h-Q!%i@A-ef&!9!5i@50u>AX2>BUXz8E1)jHg(#Y!SWYI~O*Kiky#Df*02qGp zF)71t29Oc(=XC`qdsJHUl3}FeRu9y>eNG2wMWazdHw@K>!ok<7lEQA&+K(HS@iBR_ zEMqeHFkblzVaQqfhP;vFzuU^h;-W7QXusicGO*TjV68XdZ{^w0BgjcaKz>fkTGMrN zVl`l_3Zkz~-!KLIc%fIz+M|}O9`$jiUUBLm)*J?4L)R6S>Z~wskU65phB+j-NVE&< zcd@F<50Dkcdkhsie7*d{@jTAIc=jQ*Ats;Da8EPQD5nCynah&7@B#$j=q82nX+~0OypAaDPQ6fQL>@tlo5mg6LA^ zwY1DzRXLk9ty{-o^0?!U3%d=2{Pv1~s3rj}_Gmph9OGm`Kx=e=-${gFD!9-89VQVX zJ|Z50^Gb~(5|ih-?jSY3ULod9I6SwS&g;R&l8g}@l|(Y2nv!ntAcul1fo&grM#N8U92`B98Q}v}@UF%unxG?WyY`-=^!-S$8Mj z_;K%i%>(}FuhAqqAp7MoB0>V$08cTzc&ZF<$V6nB?DMd%=i23W{{@d3;XZvjGs~7L z@oc(&#?Y}GW!k6}^CN|wRlr}b&QA@nVjmEMXN21O9QriHq^sm}j;3X|a`&p3msmM3 z3&zT*MRQ~+h64u-L(|#b<``^9HteYbZ|p98M|in2nxRuBnqjX zPy5qV5duNu8@`5+C()cA?oH8S-u>-go<4bTrL&GZHfAEVomNdRfw9LGU1M<;oLHck zqYjvjo374iId(?9$*-c9AoGZ>rSW*vuE+rkImo0-<#Z?ZWB7cg>X~r^sGz+w8g4$@ zbnmqrqMFF>N$*(d9Q@$FL%52)G$DPhLfgrozkXo&|be zXr~Fk$NG1F`)8^)D7qEnm8KRlwSP1WHG`W{ z7x2AA0ws2$%S5qiJ=ysiiBDTWN{Dg(emQ%^~bzL#568%S|d(kZld==ah;}~Zq)DnTl6mJ{*f2xk8k}$Cnq?W zkg|S6BepA7#t(w({&VefEoFW*Av|u&nx`r+BW)*CC(I*)BjG4R6CHrtvtAm&At=CN z1!*gpr4o~WfrFk=NKi>aA;SPNkMv$gY`Fg1B{RKW^x`b%9xbro*zKMWN16wmot(>C zN`59E8MRTWB<(AqZO~x_5n^cfZM#1ss7}?UQp;(qrFp?VL}RNx!GT`D2zW!Tl*FV> zCB{Gnkm+A$iXN8ZRPjr`<0!W#)Age|u)`jPQ4SXpmnm{}9veMKwfz8zI@j#YNo6h0 zh%bn*tfbX>R{-g+^x+X@?fALT+nXrq47`+cx$q5DRmN8WV0DR0x=-d2OEvLk9f?&D zDq;*qIL{zc!n9V)MRtwBQH^e_VT4s-n`)y4m#s>lv8o_Wt8N2*z3Zh_(>4P3WFpS5 ztubE}k>UXHcL%oyRwl{e(Q*5Q{vAn`_QrC{vxQ$jT4)^v;|$&=4~UcsYr#&q-I6_> z6mqWo-|jVh`80qknXV5_a#%Iz+b_AF>Q$(PhVMh%-;oL6pgS!K_DE|G z0Aq7dGhY0yMzWmHsx@-T+{J&H^l&JHNlOnFvytO+U-5uug;|({@&TBYOv=RF@qm&b zkgZ3AI&2&hD|Muy%ih`fx2Y;3Tilf$ka}{&O&Eh!67wS*1)a|Vud95)KBDyV+B4J8 z)GcGCxV&jRAF?|_)P!N``M>G6?-$?_9?TMnehNTK9JH0m4QGv(maJQ!=>h>3Kgpou z;Gx?^CClr{r`sMAijF6Glp@Xz`+TVMAZ0i|xS7&895uDHa+q|`niDp~txbxj#50&4 zVOow!&Qr}t_GHzr7axe3NNMQ`QVZ2)ZiMJ~S;->yGX8`%d?f**4?fnlY zvic94E(}bd1XwmHs&Mn^a7hPCx-Wj0Hiu_N-Tem-MAKmgwNHA$Egv10Ka84;G)K{~ zh7;-fOnV;8^}hKmV5l`fqoiC}63L^aE0dp88uPq>C9`hG@~d9p-T%cocfeWJFxbag zl7QOp+r6*vHw2hnDejTilyssk3yhzp^$Cm|mdP#x^A(+y6J>T|)ZYfw?~ z?g`dmEh7r}C1U%MxY=y!b@e{|^wqD!haWSvYdlSHKtqmCJ1o3JX9%Kai0s6)3Yj{CmK^Api2?0ILj#%$0RC zc1Dz{F*}qs%Jwa&j;~%@mh;kM<-=#4&x`kz3t>=pliVqD^3-}_)=<{r02rD`g2Sqy zJ8LkBBa8lfN)}QASZ`^OujmP9*_g@uUH#(Cio_RWIhYeq^po9bND%4dHY*VqqL|b3 zJp7n7XX}LaFB2knS|XJ{3%D}Cj)M^;pd4wt^jGPK(2+d+cGSdN#!RJ;{oH>?ID~jr zKVHu!z(+}QKsz4FeYgfPFcmGqF9Dkc+;x}H2M<`O{m%?WV}H*4mv8V##rRV*d>9)m z5dYHWDt!h-yGx$eQIRghwvW;uxaU*i=4W|vAZl%=XaEJ`$v42^tYu4r7Rdf1if?x( zzEiQP_qhsL_$fg>L?D0g4y%E3+JgbP91fI9f1D%6d68O04T=3x;|FF*G2gau@QL&x zQ};D6J~Ed;z*}So#lG%DC&4+z?}|SNg7kL&T8dRS>ue=|f??}eRs@(@uH+K7YqWVl zKH^LJ3fNBO&fl`Tw2Djb%fjmR7)<-j@g(TrjrxA~fMV5_krQp#?)Z1W!en?Xan|nP z^n5#uaP}imw&v~JK4Vzk$+mYCNyG5ddtYs6Wfk)?YbnX(6CtRm1|HHUkg&GE6H+A} z<)cMDVwn!-*UE{i#=iNVbbIkT(wb2jelq^Bd<8)a+@Dszm;;g3s2S@u%M-o6LdAs3 zab6(!)yMq{@=L~yS6F-l8vXSLm2@Fqde!-*)1IO2d&iWqe(!e<)|eF08Yg=OV`2V^ zX5F!53)?A|q)#p-Q;>XJASf|pB|{H@otd@djYRiw4Op-9-3?dDb3EE^Tu7oek`IxH z`)zE!>^j`H`Sjjd)7a2*7JYB5iB&K`aLN*^0ig7*{UshoJpcGsFgDEdC(sB@EPKQte;Zt*O(66tY=<${-fOA6 zJIK-2PQX!^-#|m#(6gwoR#{R`j{9E4XA;crJ0X* zy@H|&4>_C9my};6dyc5F5Ia%#cCmHS;Fmb?_m=mpt(Ug*N!i#}p!)a&t87+2#o7X{ zEc;}+9ffO2ry9{#E}@k)iuEszPXQJ7sB*d8>pG6|s6W4n)dM%|ROsJg1{s{l@4l(- z{1~z|b^^R=+3_%22gflADEqU&7O%TNJlz9{IINHoitrIF>XK_SBO_YDxO|KC@6*WJ zo9ZGKVPSdR9VW{Ah|ElgSr2^e14Zxz zx^ilM1F^y~Xxe8;&>PeTD)t;Djbt=GeEKFQLh>u4XSuS{w2OrMP&@imBcadkEot(e z^G9xrwvQN{BCLNg9urW>B`z{*0tDNO6ecD7s%g)cv zx~^#`#laSzK9#riya0^fB=E0(PszS8qTd0Q8ukCn7$!aWqxMo~CEE6z|8W%U6dg$f z1yEY)<`>y(!eoM_-+m)``BG@H?%-BLFs|U@P4%U%X%W}2{|*ydCYCUFa6Gz#h3YVc zr$j;_P@WXC`@N@c0?^z!UU;NXUQTt%0^xw?aM(umo;+%DmP(@ehD#+=0K+-x-o|0J zEq`|LQ=HzA31`plN7Dn5R@2s=XG8kAtwr`lW8s-aJjsUaWsqd|J3hW|Q{DYx|3$|f zcy2ljUT?1~g-k8}+c}*{qq=5Y+}SGb2800dh$CYGoAp7+%(_vW>h~`y;;g9xxwI6) z&R;AQeq=86j~CL0?=?9FIZ*O@OhTN@s`1SBEkp%#~e<$(25Tu+BadKlu3h zShQEm7LI(LoD}zcQgc?5;9&eoNpH1x;B7f?w_=5iAEneBHd$wQ$9ycq(-!H<<6tO^ zHiNdr!7;{~Am3{Kjly~<*UAs6*4@ekpcoL3_JDU5Pi~(GM2Zjrj2Mj)vEQT+U5w#&|K{px@frzuw#+ViJ>G)|0OdAgRL zloURbw+g@v9}URSmkViFB!EDJ4$u@{z5W%v_mJ(vf55BM_sso7?;){*9hVc>xDQJL zJR3KLAw8I3;eppJ+U@WdyG=&hokdhAU#oAug?7BS^q%8MAa7IFvt&x19ec#O9%J{P z-)3y)n~n|@tyq^}W^R~*v#xg_3AhV1o7do8U0p+vNA+7+H@CVugs}oEW;i%-p}ZnRks^3FarVe$ zg1VdU3QQHOQQ9AOFI8gGS{a5Hb{`}e{NH@*(5 z8#7duLGi&meSkOHs3OuP2f4<(9BUt0PD9S=i%%QidWEjXQDAuU-p)bV%PWLe;})gq zV5#%~I(5nrhYj?-#tS$n`c!e(6<*%T4#F9_o5@3zUf*^F=?7X}cS;pXb&wWe!BZ$J zDL{>?h`N??&ZCF{ej51N!C?A}zOc7HN@D%*P(~p(;*-yQ1)(NtrBH3ExiJP%bC2oZ z)fwEtYenQK607PMgs#TZgg$4EIdOll>q!&*3OYAXR2Q{~TvOvD;JktJ2|kb-7YQr# zAXM6OPe5C*bYdefJRI#=@$R?b2D?)a&g6e#-D)?d0~6-Y{;e6T=xR)nF4r`jlQWbF z;A;{h#TKB?2b2fVCnkSQJUePyxSNv}QKwMYs7M@3sQ2^pE;a2({5p!o2t2hFMF}{A z0I0FMZ)D_nyH#^7*J1Ga_&#Qp7U%819sXQFLLH8ozr0W2SU7eEavnOS?&){6bl#Cn zrc@p@L|1T#G;J593R$;Qv>fbSh>Kl|XE}Kas|YPa2L=?s#r-5E%}=_ol=gSe{YYm7 zKx>qTXp%mejEMa8I7iQK#4Z{y@MH(Nj^v@`FxT02ovEn=CA3q2G?uET=u|vV^F}OBBhzwCeDQ#V8&A*=`)K%7Cg(y=(=YL^j zP2f{z(+1q*?)HuYD#q@fvNr>`bBW=at-3F~dA`%OYUT zLL++446vV?pg>k%N+6zJ7sQ6rf_O-;ALDY`F>vFI_5=e}00x#0Hx*{$q5{PykSI}X zoO+yAVOQ!@{7i=I_B1E~;3eH{^@uX~QOB|V|A5EAyAZEx0`N#YF1`Shp&$tbOR0hy z#D=;L!q|3wAf(12oWzBn9qtCGs3QwZjav%}lEQxPC<4dn&M9YY(1L4V7_nMQlQ|+9 z;;H^o<*9Q6Ukr$(>_IwUHwxuQ2sW2ks6T2bVKUmTrfnC>wBYX_{DC~Y>Q3%t_kWn7Yv-g zIY-MX*wD+r5xhlxR@VNl9or-P7ncI?;KRS0fCNUkc3^xbGMH|FIDnmcgU}O|Tl=NU z^1T5Yb5GFi>%olvQC$u%e8*KjT5vbszMJ7j#fXOf8!DpiESM?qanmr?tlE}em_63h zZ0cxPGq8-?qga|Xj}&9H_m8lvunYHD{L*GrP*PE>>8ikyc&gW@%h`M_!%@>A1H|&R zT5TN%_og%i3pTpxmLM1Esj>v0ulMyuPU(_^@0rp8VFPQ3nCKtf3qVcLY!az5$Qlrh zowJ(zZcyJx5*Rl^VW*$2N3faYH#@WAnAhB|m#yQMHzm~A=6D*BzojUqAI(lQ&5AkE z_^hkiXKHe4b@U%RMwaC?VV=ob*md7!mAlHymN@UQ)gaV96B^04i7|2?5+9@lIk3pX zuZKI8N1ttQ1zPvmuj<%ch zU{KGgWF8j?B*l(Nn~}?$Hyo=dEAEEzwU*z(J>7Y#Nr|3q;9vD{XTv*rnbGSW z?YbwaXpVl~){4AJJnL8hC;whP)nn&XPh`}$#!|)31J0tZwIFMDg<6Ht`O(N_>hE&j zco7p0Qs;j;bUW}d*?7&CzSw{B@p6(0q#VPQ2}@jcPE>4gp5aFpjIi(7TI~cr zIgo&27Tc9drsnL3V8XP%^|mz^zgVqb{T#+OqrGW%Vs}>_;yamSfgfKf2D0 zpLv6s^Nx7<_SsLcIb-anEX=2VIVk7VR-5BbB@Jl*xj7XtGGFkk2rO!1q>+6^W1~W} z=($R+<84)t5Z;A$yYA)pJ=^^pDhwqX%g=YRUSsyY9glHZ&A@N|aIe->)6x-#spuef z&G36kA*7kRhpun=Pq4bjO^5?)P@|2Zw%9&r(ctOpwzI${s*b_9ZW?aqVL1)s%HEXD zTf7i);rXybKzGBDc3<$T>@Hk#c=`PZQqR|vS6Gj%Fbyo>FUIreYqT_{4m#>F4K1Nr&Z@9g4g4Gz-ncG#d|43u}*AA5_!%&CE(rV&X7sq~Ee`NvQpcHdDL z(T-TfsE5`;#H_@6N?V*${131DtG?-OtTNKN}G=(%rWV_9bCmjkvdj_B5!$=b%uKSZ=R09w0YQG zG$JuWT_{PrWD|cm@Ji+MP5^wU!^t)EIEy)}zs$PqtaGbG8i%!7$2+U$kLKjce6)+` zJ<&7PExz>t$8_I^4Kv1n_7P(ROXt+#4SF@P9`GHS7QM2wh6h$8H zel%kinaovlCTjF9#{Q6Mrm@L6d#z&C(v8>Vk(Xt;|`?#y{TNp9?r?ATU4Z*Z2k_#8VO zYopRSe-q$1d$A&B>-(O^f+vUR7ob#f-v#`eKHR~aazcOldxgk(0TDUp@ccBmS+x0O zTo2-x@206QuD6kdwNv>wP5a{|HF)9M2cq~sA@IM3SezzUn_LwqvKk+>Rf1NZ((xjr$h zW&vIjC4!p0ylO`bUVmv`E+lVs+ptIBsh|iR<5bRWqj`Hp0Ve4I==Nve_<<`pa=RLD z0PC{S^1|3{Z7RaIqRZ1&^dP+NZr#F(mqgIZf`46eUWATCy7k;Vcy)gRD=cDrS2|If z8Sf?u5Cfp+2p?|xIVjca_Usx{Qb7n7fro-}5Q^$$eyCos^D=J>p@>-5h zd6&(vV78j{aO^^zHwz-yjcn{bzRY6Y`%;8_HXBkrNkKXwdu;lw=xtJy0(}t!^ z{RF~t>U!RYR$o7Px%P5}Y&T++=g0Kj=EC9e$xBz&1~fkgk~;2&(0^ZsDig<*Y`FTcHXQ`Qe~uBB#fo| z83elLcoPfXH^cuzPjEV@I@oK-Dl^=|W;`M`I>Kh9GHn*~T`_OFyPS)y)?@Xn!8@VI zDWZ`*{fq9!KEUj*qKbCdIMrCv(0txfv6S&cO@suC@p;UbywYbUMQqPr^V=#y=(wIh z>>Ku!T0}ywPp4&52~vW`;(o!CWNXSpc~RPG&!sa<1cELTJ=B$S4B&c1<@j8~uy-jb zZ)x|hFIqK@Ka(lFO4LEk=A6m7R{Q9+BTK_^`!{-%+h=fav2@GrzQ)*(*ooPRWgy8u z>%1)lJ;OPo()sy=qH(Wooqaf!mU;H3JZ3c}%yrubV7;a&90K3u*lO46o? zE9}gwB$&CBzfzHHxR zL8~kp;AaKX^cxttxZ>j5YeQjdn^{?Z^I&owd${h4M zCglwy#Lc*lk^)>jg#q1xMtJ>LoPwDP9U3h!N`bo_!1@I>2v2gg`B8yj2Vew?l1ULjRW&Q{P9_;N&9{$c?}JQ)v>FuvjPGI zIqFs^l&=x+SJq~om|(xpJK=ZJ=;&$7NYFqtGva27^{BPY}m zfAJ2joYlh6DLiSD?(vac1eR^qKS&%_5%8^RP` ztN^93pn*OVBlLFxXa0v;Lq!9S>!ZWV!;hlDTc?6cfk_TP&m7^m(`^j#=opnA-{Y9% zZ@X;$hh|(L0pe9sPEu5-Vmek&ug%y`3$`c(XH+R5Nx%az7K-hQ-G06jVEXS|oKOr( znS+Z$7`yIhBWeFo(a-prum*pYHNX9tD0$wq0hSO*4}0M!gewHe4%a$ZVD<#HnJ++5 zI~y;N*k!ZThCpAO;punQ@@rw!6$#``~EXXxWb;tK~#KwNXpN>TTZG;x^)laPsIJQu_Y{$Pp!=s}yqSc|$wM zIZwT|BU=15Dv2u&q`3xqLb_U-aDDm49T;#_+8bUk zuYDuGtG+5S7R>XdyK6)d@4kyzp|{SRIx{W5B_{K7pJQk5XNNlBJxtl@t$wPnQBUIo zJ#tBJ7z2R*n?Tm;CCVu}ZmER}|GrJPe2ef3)o~(+a{i947lSMUyiG{iSYU<|>%4X{R2gzQffexm>)i}^8SuDTA)jYExzI z0=pYN^W8(I(5Mg5@6eQMm@*Miy%i`dvQvKkyeVr&c{#sd2P8AH-fS&fK0;F&YJ|^&O@S2wd zS46{d2+D3vjzV;~@B#W83rK#QPs$h0Nw>Fcs1I~A10|F_iQ-@El<_oU+b(pHqH>3k zB)ymwBnqjcf@7r3ZH2IUzj9(9BhY!&aa&=~u965%t^A24@6g#6J@3uJo5W5l6<5#U zX9#rHk-L7W=>px)AQ#k@@;@w}iMtbEFWu`q#TAEOTL`6K7AbZ+>o=Er9IrXq z|FZk~=6Wrw!B5iAoSZ`Mim69C`9~(gbHx$rX?_G%IH^Iw01h>WxlTmfFL9|Bu}nCI zuhysm(9(qP;UXZ(Hy;v``>i2urQRbW4uIhDCV;gg;yE%dkm7>mcYTWe&endnPK@(u zf`pggK|FYFD?KKjz z21ZPc^zR9}JAxm{YpzG})44;l=IzuWa>2z}Zva4UH|;g?+SJdwIiP_cFDUokuMHib znOY1dMj;?6g04^HD1>Csu;m$lJ}Opz8XU?be=7=aBj&0q-%}cvDvkMK)GD%2=uG@L z36U5?j6w`4-g-GNlvS~J>;qeBBS|&|-%&ML^BY@90H4vqk+D=eF;%cjs@tf?z@N~; z)Y=Dn9MY#ju$;^M!opF12Bl#^z)RDU|j)cU3RPe92M#%ljkUM(Nj%#wRn+z8&E}cx$)X&mX6lUlA zJ7dIQDPXxs`K}$x&d3wbO}QVr2>m+)Zww4SJB=|TB~?F|?TL2C+RckX>wqLu`d}<+NzH7uqV;N0XBd7rjtgUbtd(Vde#Pu@;_wi-wi%Odk5x%^Sh9*A zd%991xuM*^LN;n~G8q|GhqFbzg{Igi81MfNby_1p={LbNAOaF-@;nXrcR-*F<3dV! zo6_^`0@A_JFtMLx@6iWgMd->k%^Ps7+xnkH#&CC@ogMxQEN)Q?lO9h#<+X%?pIkU_ z8TNIF^|dQ-xIe1~r89|={2yt$ND9l9^cU|;qaa7W_$h5ltahp`dTRsRzF+fEIp$rG zy!4ieBM=%yYM*NwsJP*ywcFMBj2JTKZW1j6v@tJShMtLZ=@--PB*{M7+AS_r2)0>T=TCbA$(eO0Rq^LI?D5rTWMx_PigVt$9KwcU>m z6ql5d+NRhpVMA)En*eb!VXf#qzplprc6y~arTMFMzne>8U@Ya zgMVQ6{p?ZgC5(@UDI^6b2U>Zg7>hFo%ML5my*kWwq{~HyL-jgMKsNS|X~G8lz6&M} z_TTyX=VR~zC-hdTa~kvj48JfYt|+U#e?VWG+KS zdR)c@mTO@HqM2SD;!~DhZwRFz4u=6twxt5gr>ZQTya;BD?4i`^ z{KO-8n^(@bP8)>0^5xqnrO2#ut0t^}JI+`1Fh#Fu#o_+*@>wE+oCGDrP1A(FB%rE@ zb2uq;O!njXqxM_EJ3Kw?xd4<&kQ+MO1P?~ssBuB%~OeGC{!N- z5@*yb-Y5*w+v(@ZZDU(cmR!HMN5^#ja}}6 z0)n=l8{_ssK=8=43wO(kv7Hx+?If;|bdtm1^(!tpE`9z?$VWogcwzaicJi}GFjHgL zsM4n(qq8KY~Sf!hR?~YON%H_bP)LNlWZr z>qg5ZhR;YL)OMLy-TtHzYz5SCH~5G37qNiIhl{FgDD)*!w!>yaiD499!o5q_!*+{6;Q>UTz`l;hRSS}wOm(vFpU&o6W z$L%iE+p9YM(fQxD-D8>gJD~29shk5^{>5cwWyADZUUN#5c1=epc~}Q)X1DLZC<}~> zIQKa?@M63oaY^gc42FvB>rsLOVO_j#hRLxk3nT1+6~jmC6$u|IXY zCeOqbaD%#+rYLzBhxn4_36ep!!ySU9ZbDCT{`2RUp5rcUq}=Gu^&uKiH_8Ygnm8JS zAIVoZ3G&{{PkCdgU!XoRJQ3B5<>7T{;@sw9`=PX?>^sPr*65ss6`oCe+t(cKZ9o-x z9YDB?N6-oa(ZS*MS$=9O705Xa7oCWnqUeoC5Ak^tw(+a?mSOF|d_M#P33Zvu``w>T z!3A{eLEp;dMF596)eMsf*jkAP6akHZvy;>AY3j9xP=vm*KeKSjx)W3z$n{DnVV7h6IgPf zzZ9Vku4&r_IxAd(3K8X6`lH$hDV^hMm@a&6!DjTZWobcZP(9ErLqSJtMcgLp6H6pM z^wSJWu5k~!@twGbg;U{V=ZAyBN0-Rno5`;;B!S6UPWNKUyZ07$DxJuhxd=^#Y&bAmmPOods{_r z7c40!!vF@>T8<2RS_ZMCWSMrb&;ouAj6~+q?6n*>*I8s1*a0j7%jjK$`mOCw5eve2 z!{*sX<9vGE%kicAC-N@V%(&FJ?;iNNOj_LzfAz*V(Tww{lmKVI@pP5?b-1Ebv)^ZF zgz?UOfo4s++4Ky)rBjy`*!EU3nxG(R8KBT8N?Yqk1FkkNYe)Et>;Q zW+NYac-ha2SOySk2N!OXk?Ha8>Q0Sr^UD^od=$@w}_>5o>4)ey3py45oe47}P_JG5QhJl8n zhT-;X`YjL9Ng>Oc7Y~gJx`E1N`t4tr|}^r@c?FXpjZg7 zC`W%|DR@EFeKm{o9wN>jn2^|(XFy;u+!TiJ9gaHnd87Ho!_~ZtbtI2&(L`BQOwRGx zgp0`xS4LbbS$=+e98UaKt0}Alo4x=n;#kCV2n>T*ZbQH*fTFz(LuVyJJ>Mu4=5!qf z`WiZl5FZl#BjG|M>g1bMjpc!iKc^v z;T8h|cLq%=Y{;ysGn(jjh-k4%Y@L{}P^bN2UGUR35tITj&oL!G4lAF@b|bl1rGS&C z8y%GtTR+sjr{0AyeJ1{%hB^d|a|NSpyjC(Au`f z#zyP@X}qkDNc}tgj=3hB4831zI0THEe4~Dko!{v%|NB>>Tb{+5_eu3q4GWL!DO}U+ zIfLgL%b#k%;XbqK=(*e?cpaH#D@V#q0M;*vn#n2>vE5QC7L;PEj7WSEs~;O!8(tQM zw4t06(<3lv>)YSo>hBZ6gCk0Z6k)}=r`{1@Fjz4ZT$I7MdWVnvd$*-!Rgc@rgQZQR zt+I8a7#~mkV(H;hku9hFrFOxR>d8toSSvq+$vZ@?9&v4FBxLk+)2mc{98w(DHPRX+ zSco<6@86StfA2uiAz87Xvao`cgWLwb*-%8@_Rq18yK0oRt0QwvezqkG@V) zl>w?fsk?C7eAG!rXIZ8QaFw~R;b#7E()*f;e9AjL;pWAL+o2b2+RL0Bed~jL-B=FH z9iR@@m@O)>R9)v;I?$QCg9M)i>?znyR(k@M=Iy(R$~PXb0gU|)VtA*Iyg(`pASMua zd>@&^c2V0DwD86Pp~5l=r0|W&ok#1iW6{OTl(=}c4JUvd z9#(|DKr3ep`0N*k6{)Ig@l>BI%Ux4|2Nokz1&E<)7?r^2PBpfZN{6Rhw{Fp5n_Us$ zIe11B7JwG~S z1=#@vK15pL{I46eXjPVY!vqPi%Pc3s2my?@X?Gf_h?|c*BJ-jjwv!gX8dJmk{}Jc} zknIzHDR`M##P>no`qlq}x>MBE)IOr%%5(d4y`D;YVFmVUG|HbD+!{%>Ev|L^R&jCo z6dMlL41y(?-3C^cF<|vK%`-6{9M+9Rv|ve>Vcv~$xmNzh=X=?SbFsv<4p&61jf zy3TIq<8q)*2*(j8j)1wFO7i^J6pK>CdU>HOsH3q#~{Jp4AKt$LBrr04%l zZ|k+mU1b&53wx5@nABXC5g4uvwMBRh8oWA&y{^_#H`__KNfMT9B{GS4jA`Ydo+!Zi zfX-y{G+Qp(4f8M$18NFjvnYRo_V2>C2UfrP`y2#)z1f+DTJWY)k0%N~(XG?8qZmmc zgp?mIL#862@Cq!)9ip7nK{qcBL6>3s-iV&DoVFQ8Dv#l4cOe`OAMc-l*X(`&y#KiW zI@9nl)}%)z@}Lfj6Dt#%wmbfl<7XErQ?Ol*ngLD|lo?eZ3Bt-s04me3x`|Cgh|(l} zqbrBy>?fl4$%D9SIt5x&6-~Xi5GjUC-svr}e?QWCQ*OQ!A=@bszCAFJ`p|vVNYQV9 zS2cG<-y5RQ;c(<{x+2c5GoT)lwFYz5Ni z(rY*!Vu*HU$(gwmFggebV!%=`5MHiH;qub-W8|e9^@8RO<>NHiud}$wx#MaIbFdkK zq=V|vB25#-vEoqfSbv0@ zc>2aO;feZcTpxY=-hNH)CEE3L$3Dhd|FWV0AOAa^3v{?yC~wm@nt?pL@td$8nezga z2+DnmeG(zN;@bL;*B*I?9@OVP&DhdK;TqyL_wO-3nSZJ|{Z^o$=6aX!gJW#;%Y;zu z#WVGTwD1+4;EU(@eL0IRANT>eR%4RAPf-dLwwbS>@AGg@kz4`p){wU2aij{qBs{Or zo6QKzm;fI)PdD8#+|X|}sByB<-Y0w35sk4^FzoQFqiaWF9?}1CC;U>y&En0eTo>%<{cJvO* z_Me7%sc)qtUFJa~Lat1ptAf)}kF@vUAo9=GBlQ^O#Z$c-GPZ*{G2aq4X+dV>O|2>k zm|Y*fqcwUtVwH{={g|Z8zJ2}sg*|(KnDQL1bT6duHp81+14$WAfiN~^n9IT zFa@n{Xf90b`O!E3DHY-JQ)A&u(Q4fM{~U>e#+pLh1FSnYiF-LbCa`=ov|S3~eY zv_Zes?F}~74=Q#dK}ip_YRrW(ZAU0t54H`*0ZHbHf{o`SvwE|Km%=fu(z?T(A3x$6 zwV!A{yPq?SOy}uWo8YRN!?u-HR76i6)Uzi^V?qHY)ConP9dYv0$5)q3gA_v8MVwhZ z_qQGHc>1%4pjH`p*7UJ{FDN(NGIx%eN-E_wiHYI(vH3F>JzDBKEmi5mm5EJP`@^;~ znM!4+D1Ia!y#5FJ*?h7^&_De{WAM!zQP-D%(h7x2j(Ct9qmLhbLnM zx-d?TZ@(`Ws28u9cl|~czP|6uP|?M_?xVhUBT4>-tB!0pUf1%<`dGtL5g%c(E_Q|Y zO%aBRUl)>sUFJqi2NdSSoz$=+cV>`1~f{6Q!|I zf*?SRk|g*KfZ1N+ONCg!w@O(v(}ZiZw0~&>Sg>8rS&ziIT;u@xCY4XNTi~;K`7QVt z0f2K=GNj|2SJKg~w?pL80oi+rSkS-|F|=J+e>ny?QLg2<#G z-=TMMFlzdPy(m$X9ug&XU!Oz$&pvG+&?opLbtV|zfj&uOl7A272 zZF}s-*QB24xKH_?H@dx;tnOpz z-G0pCkBpeMnToQlu`w35luX-V`z^O2RR@k_A{Z?Xm?njWqe)d~M+30ZHGD^T^m0qx zPd+alPo|Tmv%is&4f{p~(WcOJb#}9um2|`Q18+-Vd23TPuARSir)1dth%pJ%Aem+i z1(5Zq-0q!<)ZSK|nHpZ-s(D||hp_a(8tIgxAbmr2j=NWGL32&wV*dVHv(xzQb^sl$7gMm|#py zC@D&FJ%kl~Yx7@-rMZ@g`?=e?GMCk~cW4e9I?R_ieYN~^Q}&}!#c)(6Pi+Q-EC(+p z)D)L#fO#iUY_y@~kKa~9NcFz>lm zs+qdBJKH)>c>}^yPVIb9{4Xi>DUseS*G0T(?^$;9I`beYzeBE)R|RKIXQ$VuHVP(I zH-PlF8YnnnJ_>-A2lP$Yspv8%>p!QV9HaJr`E*xYUIiWe;@%#R^}hI&$Qk$C{QWWFr;h ze4egf&2}rqMgY(W-l(IJ`>lX<$IT}QfIU<~BE8qq)A8VR3IK6<&-9yl;Hsm`vBFQl zv9juT^&ZqM9Z{4XgYX1A45dP`6>lbtio$GuKg=Gu6{G|MvZFvzypfgxaOf2}2v^|Z zra$M7Fw(p;} z!lqL?k^cz_kNuwfEsJgh0DPF(at1j3?hKA;;v`bi|GmWUAH6yp)(m1E1>c;p?6eM) zPhD>8Y8Ts=Z_Bc30hb;2Sd;|F%I%6aBC~ERpmD1BJdZx(BZBS0Bcb}vK>aA9-Ew~K z@WX|MZ_PJCQRL?bR_^5>Ga&)tP%_y&AHgNZ4gAN#YWoPp32?SL{{5Zv|-nw^j}Ui&fBRV7~ECnce@(S^|@!G+61NtiO{9a`W4L z`+XK8+t+W{K$ySgJVZZ8-tolLcLjboYy$Q;^bAGr_b1NAO3CBH8aWES$+IVpyo!tp5(qG=Mw86*yL-rQN>`e-%^*JFo!w z%H#-Y;+%eLWbF`U?h+QSsdMxU)g?9YP>LAY+x&8UpMu9Df7PqStFf}9o6 zgB)~Vz4p_eXu80$p?pJ4?QMsmP&G=&gOtb^j4jQ(cAWwPZiIssSk z@1j8TCIVNReB?qaNWT3xnwpsqY&VQz-*6N!-&~QjrcKR32Pa(DXev9YCBst9?cxTLm1p3&vg)r-nmCOt zpM7CMg*}AI6&FCB{HDByCt<@x=Mp99cDYPF$JBXVBQpG%dB~U`!XrOryL5YWadC0Z zGW#F^D&e8q^>#=(D`O5ODbqwS6iNW8a0?R^#X<7@K^k8fW8MtNEvFC^U2$P2Tp zR^v|pkb_GKwZ7MUnI3u2^P)rzgWjseYwU8pTk^p)kt%T|@R1TAXV*%g=F~U;cavGM zL#l!=mDV)KtHf!?1u~ZvyAP6Ird5f1Oz&*O!6+c|dk8s9KO zZ=O8bhb<-ljOJ4YA?Uq~Ok}Se{{BmExjY5m>VLN%HU5yD)NDK{>fMH&xWe+Q4erb# z!NAt!MR@+EdtF;Y0>UCd!!e-;BBS+bye=&!1p6)BP;dC(xzX9la968IGvj-T-slPZ z=Qlv@RHFxg#IJXUD`sJIPV~wV?e231WEd163i&|IHJToBBfGruH|Zy+H>D-UW^Vcv zR+EqvQ=Udi{}hE=8vL-DJ3xtaDYxSb7Ax_h=~~=0tg#F$D5kQ3PEb;a;e~yXa;m)~ z!6K%BUd7j?J`+d8Q~yuUBNdj_bh#V%mm-CU7owK^P4dI#zrpURKREV4j=Z8)Mv!bQ z)~PXzN_A-z2WkqyzZjEQDQ2f3fQGT#-<2 zNE9{zA5aVnATluciGCKRVe^Cgr zeVThld&q{z<5`0G5L99#8a!}I0#|TpLAx9bWkSlntKW4BVT6`iw^om7P$&fg-Qv!7 z^_!zhLltP>Kp%yaGY0JKPk71kmk>aIB@%v%Pik*R41`JuW?WKKSf$-QXgvQb^e9== z;S%aD_+m4c+zj+8y_a#!jZcH`V44o<3hH!>LVCOz+veMISASrU79~#S>;3sOo}&CF zj(15-%?$K)3OH-+d?{t#-rsD=@|~IO@wS61oI57x1`}Y)dQrodn5`_eRvbARY3+2a zDNHrLqD{_xYWR+S2D))b6fj5w)FFash#ZtA!3 z;2X|l?ZXCQLAbd2Tt?`fgj+*>ejJWuj^T8mX7`ig)pUykj~4qCR@bL)*MA9Fk4}OF z%vjs~3XoO?XLH6x+q!37Nf zl!e`uVQBeB$g0pA(4^?0VcSn1R#yx&j96eRwx5iZ8_K=>42T934r56XEC^E@E=yE{ z;s(;}fQIs^#$u?YIWYs_0|Qkhe8z=+PgcZ%%qzLC!|ip#kK#)LGyH4o^$ox^qUu~f zvlJ-1`nf+c0*&hmwCt5ZT}twN4CrG%|hf%E456RMN|hX+94Qt zwp8VVN5 zWUCjbNd?;y2%6u4@8TJ>)?7jvg0R#(K>{#|#!GA9(6wy~1@C1-;&awGOc)byP&&7; z5~+;eJM=v>56A(Re*0Wh4@TgCRiG)d)$K8o{>6{1zkJ{SJVQGz@I((Guws*+2Fm!R zpqw064A1PFB~a{KVsNTP?IP`(Ejnw8il|^e4C27PY8$wc4eRpxV4{B^BvqA?WA6kCJB_omJ#&mPlCNI>>?pP1B? zb|7@!c5(BXB7xkVHV2YfnrJz_oB)ir@ZE{Ur{Wb4=m+%M-Q*lN;iFL!Zx=m5Rl9V= z#Yb@(sr(khH8R+=xM;w%WDE$26H3Y=ByGG{(O6CrGhm2S^X8r;a0uR}kw*&*AxiRv z)xeCJY;NTjEXr~(V&;5C(~X{)YuCYSAPsWRV4xdjaX4Bj1mHl^3`@@>C8Jf&4CA^b=+>$lvRuywlWSBV%utyLxdI-2~`fxBb*f@cS0mH#b(t!v_ zamLu7Gv3$6&nWtWhA*-C%(Z!gMogmE4(u|PFXiG3M;YGlBOTQbwwzaV&FITh0bm2v zQ3p^iU9jEy+*a%c4L#259yYHWLD~9G{y8b)y_jjazBU=wu|);dNiQ1b>Kd6SXPRRX z*lDNV{4CRPf*HR`JW%K#*p$MbM{6deV#LPYLp(QWaaDNpjH zy|Kw1Wv-IkyX8G^FBWA;qH02UdVs}bctm7hV3Sz+EPvCBwY!D`DrPsPCt-@a^18#pL8TTT~n6z%YhT8m)^m5%pPJ5ky!di3V zz2RG#F*I$~WP06q+|#v$39rNO=bX$?1b;UfvPSu@F zs@Hq__ICSEi)>wS+QN!p!IJgck1b_o_Y*uWH*(;MB;*qb|n@Y)-zLps>kD# z_Wg}ca#zsssOQymGJd>|q>vXThHsgsarE&do=g`{b6;bNFCtD{&E|?~r+*<*5}=_D z_`+aLM2cCQ(LnDrX~`aNHBQ{W;X_8>#3h^kuuST6*|FyJ_w-8r`s(-0z)hCxnwe2y zWqxC$i~>vedWCD%ufbn+maAL8J_gy;AoU`otW7b|nVsOa>I7q5cjn^1tv__4#?-|3I8&0J*1 zt8Xg5gZFr|YSP|6Ddm12QYqF+r-G(%Z0WqMfr^ImT=|#m<3c?lr_h#LoSJ|}1{^Xx z#>h=2U#}s4C{Bs8v$4S!H+F?57MBZVz}l%X zoCi5d2}q@!KTE(pr*bLXL#)^7~-={);$hpPG$c%+L?W4zQ})eiHbAJ|&F zN$j5O#rGtJvp^*gwT(pE5~|`Y>t7_W>|^oS?eYy|s$h!)l@#MgI=?vXR&~dBn9Ym& zOQrec3^dx_ljXp?C4XU@T(}*q{2o4`&rf{k6S@-4mg=eV+UqH%37@B5F>`RnPflv-in#pqe;WCOV?MQ^v1z-nIP~2&(_cz@CqlX5 zo?NgXLLlu)U~{I(6Onc=ym$6C$$8t5Usb~iFVA#zjlYN*9+{#FQDLNE9zN5P0pGs* z$*R71_#5;+-RNq_91&vl!rn{%+GC;n+eK9{|Tmo89kfV~JiU zQt-W@tM@bqo})A*^Weh|Rnv_5Xz&UWTpopoFGsve6KZdgs-k9iq#4&8_u$u(>WnNjaQ*KW-5`~cx(aM*|CsX!_8m3G5Xto>7R}tyGvvK|BR_g>4Hd!ESJ5i|m zf8h|i;Au77;BTWQ+Dq)qg0AJN3t_?sYoV+<_=Z< zjK@dps}JU5pwm*sG#O<0Oz*6Uh#ZNj0{GUj6tgc-ZY@ixH9sgMs9sRLmg$Y2^xIMf zoYmuDB89kXsm8#___|M@&#B?4`*W2>7j@;ZmTL-jq3|3}+yY<VSepP!tmzgQ}I~nm)70wE~VE+c#Z2)yE zxNuLa?O*8FmR{iarx4;nZlhE(i-nB;zP%@&k?nhBW3ZDR#4p}Xit+{3)JV^sr z=wk>76LNLbQU9|)ah9DT(Gof)&F76Yb8rd~;r*OdoC&{Y@t7-E`FXQY@)eD_xq`CL-4mGNEKbh z&QdDG3Aoz%E)!;yfK1|p@fA5(HA2?I_S>^^$+(#RmT*Te0pHm9*W($g2RGc`k_k^M z&5s->pV&||F_UJCc`0}l&=3V}S8C3VRFP_)TVUoBA+cwNAWNh0(4;2(b%7lTh{{k# z{rCQdy|;?0E7;lvaS0B=-GfVz;O@a4g1fsr1W0h#V8McJ90F`KxVyW%OVD2A^tt!% ze(UGHWAKnL7JIL%`sS>u-;6|Eduu9+fT$reOlEyb$33Z5&(^`KCl0eKG}Sb`7Xqvypm(*vb+3zjh%SpR_ql!|9yBO+?GzeT1 zE_EL=sYSDpGqKB$`fVMh+mI9eub(C8nLdv64b#BHKObAMX<4H~kWG+BgRGTx8R0Z3 z$qcCmLH8|?1ZhyGrXmxELnCHX50&+?G3#VId)g)*5J)MDMUF;HZoZ7+FeZGhQPz0& z1%_EWO9}E*s7t8Vthd3^IP+z%kOPw_w)0|1*zLBk{$B_@2MQ$Lne;~(>PJD;pmsOt zFz@=>zBEXT4b=>8|MtmAELx=beH(9Mww&OEi9<+}44a%#O)17DP%X3iGO^@5m*_DN zHz3AXIB8G{lJVb_^g7iqvo76+~gKkX*=RDhrplU@zl;&UbY;4m+iL4$Zao> zsqiL$^jc}?|7hw;-)ecQ)Gv$@X~$ezTms} z6oyiAJ-a1-l?)#&u%R8O#C1pH`1NV-oFUw1>{o!&9>};gU>_{jY{w>MV)|zogMBLn zfAeTEId{W|>Jrrd1vgT+~s@isTT?#CgoyYnHmMn!1 z+&APKQbJ|5bK0rVmHo?SSYY+@B;;qZ?jWgp+eO5JV#^;EN)^t;SRM`=^=AksCaxrH zwz!X!Qhq#L7o-p`UuR~za>3#S(#~eZInL)wH7$Y%>~@GySH0qyeyBp zVB%qZ7|(|bg&)X8454x{DDu4pjTP{>e9?HduP;G)T%D)Qc{xy(k(EnGvZ7N-(W5zf zsZwA*ZU0af(6VUy$3DQk#rC;RBPfW^x7U#=a{hJKpQrlv>+)k4cye-@f|Hu_0i%~E zn=p+1Fc@xjIWUy1ZG(ld?jo{{W0qqu$OF|fJ`=| z1`H${^bcUIQbH)`v5cC73;D&SmXnDkDm$kP!8(9>VD`nRg2?kIFe+WBD3O-SPpX;! z`~ey>9zE|4Ty+=RdO4z5x6%0YI3!;jrs+1pjWZDZ$O5Yhc3eYpc7B%8zKK*{L=Q9f zc+Q!#@#^LjYLg@!{Ixz$+qWHJ;7b-R#Lsw6(0#xBFjmpb)cA6tae;PxPap2|cTo&v z)s5F_;ed$X`Fy z*7!M?G6a@H<%<0%vi5T?LxV{(>7ICQnpbComqC_An+$xi5p?VC59gi~S6!3gDH&0k zw&AXj`nvt^lu%p!g!g7bgKY6Y^Qf_G$eJ#Hp&AS-bK7ob-Ge5OV{4Q*rFw^!eRi#_ z{`>{+MUONGq#i|DSqT0^9$=ut&!Q@6Qv+vlg(!}CIBrJ~jT7A9&5))a9*+H>(rv?M zdr19U>O@2ZF<>>on^Rtm66-|CR}9x}jFMLBU}w*{Ca-woF~W01x_p6cd#vkQspaeE zw-bHVK5?Mc`iuTo0>FI^1|J)zb5R%>x-(7LXV1D#nHh++Afebz&KKA}5-`fLBn_r{ znOr}fTN_letk7@5y{Voma=={R`Z`bwD2y-R!pBuwD`{P|p~iSd0`;fQIbcxnMSJ6| z5AScGTSR1}V=GM&U<6*P;WDE{=70gvbv(MT7s~~GP2?w z4(EP_U~sgNl=&p# z>vhXcSKgvNj5M?ILoIysndy&pEsDbQm=k$)X& z;CE1e);B^?=UO|ed^aCxuOp1-W3F|_%Sj$`#jtFElWEhkVEph&*U+|Re`0^C64%3R zk@PY`faNSca2+D+Aa^R$_a9spsD#nVu{N%z0bO-VU3K0W%8N#i{Bc`W!V$-{Tsqn9 zDRC>IO+c@>G{a?%Xaya{roz)RwDmS?4!C!+v4}`NQc1>KSc`~$~6RiPCo{f8$VrMtSkx{^}vX{V=lM}uIBe&@MJ0uyom5^lxwT~LuIt@ z&dU8}<^VPkVB<(hU##*M^5z zjiWEM^AITX5B0kqHg!hLCna{?Lek__KhwNeaVG6MA=WPMn0OWK6yX?A+pa(!)(TQD ze&WSQ-CPB5Eib8aL(@mao7Df36ivvGIg7VNauW#5Q;b|dRU$xajCFsON*Z4pNG}5r zBt3;;gW}>L*dfZLVZR4eUurn0lQ|;NRie9{?RQYa03GrvM;zK1-!ZZ#tDvX!r6bFtH1&*> zdbXOHo65?|MEBis2}2dAKijCj@`C~5e;G%^Z^a&o&u^wN+dqK06V44B{?jKWKLM=&o@Ox?;ILnl8$$oDL;v?AaN`r`ET{g@BR&9*nAZ?jQg>v5LGNzyFD^9|{1zrosbJ{`)p> zA6*p&?A8By&Hrxb|Jlv|kJxmeQBu}pe!Xdw+Ww*thA8G~*>qnT@Zl!<=5i1Bm8>Pk z*W5ct@gF9a-U1Z241v!;g42;c=-0U=;`v>w#hRDOrQD^fF2LhfApdM*Wfva+eoO+0 zAqM0A432r$csx$J3F!Z zaY(=SsLjS5pZDAx=Z)Slz7P1~^ZFf*AOA(^B09NuOP-U0qcB2LE6FI!YNf-YnrWM$(pC4hv0kd&Ti?xV!Afc;NYQ2c)^%@f$hTE`5&+L;bNxJitBqU2#7kgRlSCt& zns{-KFDqhkU23b^_~&UULcQFU|8RRNj|&tb`^aEmZEYQ$>U$j8@oO8=nVC;a5{-mb zI~`&B5MlSw%)|bc70Tb=KOu!z^dQHNzue*&Xe~+!RJ8L+Y#y`tXEQ%yd~(|j=6{_G zzvT-GpIOFHp%;vf-<*>C8yJpKA+hpz)#Eg&a{v+NWGG9zL-*}ct)Lw9PYet}P}9&D zI&!E;Y$_08H@^M^pOsup4tYm0KCZ-K5p$ZV{gHOk(+!koVw3d-hQyAmI;wRPn8n3u zYZVX(!=N2P2A@Ymyn^?Q6=*yiqDJA%XCXSo3$_~To#0W2OQ3#9I2H!)&kQAwJ#ZOU z!3J?2c}Uy)$43$IDdSiBH~dO)iwo?-=oidk9+5|Jp35rxlaqrLOK9NUFZg!jhBMeu zLxN}>A8dze!v8|Ogq6j|PxB&N;T}%>oLZAo6r>qyC!`Dk+ zOtc$Zc6H1ISYNrV*DfnZhi3F^Hg%y=PHz=dt8a0xXyir!%3o3-TtT4r-wFU19yN3P zUwsy&C_YbW$O<0;51rl7^q-et_WB>1`Y^|JBX2UxqoN@VwJIWf1oVMY3JSg?u+xw5 z@s8*x1SZ1_yMYXv-1Cy9Ts;1tUji_YtiUCoaTY5{7RcwG)+?&`PGur)>-Wd)_QDQ0YS~Fsj-z;2+-+hyaWYmfKyx!X(9IGH8lvo`dY%?gZLMfWHCP!^Lc^eadH_=CP6 z5`od<|?zGMnjVGx@zKS=GdE zs5SqQz*9@BdS}g7Uj7}%8vss&6ysY4$`AnE4fdfv3Xv~W5D8-1kv^#1T{-sAkrGf& zCvTR&O{oFwgR#!fo|)PJExvy8 zhU-?hc4S3j7ILEv{gC|ndI|O^#gg{Kgv506jra+6(|gO`m!)Q#r0DCMMDxl+2uTbp z442?luRhrj|fv3l**)K5l=;8Fg%LQ?h{jCz-ww z*VfystJE@w=GrYV^J{AI`@yx>Gg{}(8H=YO< zaI&yeo2J;^AayeMhS#v2SP|I4t@NuXs`X2l&&`?!LNfR}`a*-cv#ktstKN;@Vijpe zcL5^xM`;dAYwVkhbMGEkAH8MYcuhJ};sS@zSzSR^i5A~LIX(w}S$l-+6xKvIj#hmN zrDh!%pz9^Q90DF5i>d7{U8e+{5C>{ZXDS|UA_E>Pe6JsBFTn;cLQ{{vn_0jc&pq1X z1OV{AqA&zyNdk+R8o&BOwVnYpYVHd_^W$ilD@c29v_EK!%L-KX`@&jKHnLKW33tZ- z#%*SDcF~&sG?Rc3M$)NwfD#I^YduNXx#t*`-DALQH+*Pm>{b29YLEH1#%$|z@DBke zt*2>}t|mIoK@@{#MJT9{>z7{+-Qf=X8v;uOr8S=^lHfcxuXz@2hzM}!-Z*pN<@;oz7Xce?KsIw;Z%aY7R;{2-5B;5GRkHOQ$#5CJ*a+XZ;Xz z`NHM*)G+%LCM7=bggsZ*PRKj6usr?2X8P|aiO>f(&KY+1Nsjcg7-D-M>r$pDNIxK^ zYPvDQU2@ybY!O2oH*@FPqWj3Jeq6-z%g1dh$fQ=_&F+rHmv#RN0|!j;YSz&Af&QzX ze9CEc-ofDNj1ua#aCx2MpVw-V3I$qBM&)`GZGO7#yE~XgAr;U+^*uL|5pol6&u%PS_FBL-y-dDJGUC>oR3u6OF9{}7%DSJg#|Gq2BNIb~ znLK*PDmi{T)_Tor978Zobi^CSm!B_ z9wDgG=f6MZMNGp2l*kVE?6(@%WraKIRkUucKU|*ddFs!T&}f-w6zz`#(6Th=NgUI2 z1F8W}zx~l9DP%Wu^ByK9Uss>zwiZ48_HkpaL$Lbl>+1&?UhhLLt=p(oEk@3^zsn1G zY1zLFa&L$6opEPvHVQQ>;w9tsP+#_MisjKX}nFj_(HUH|@eo$HHCU^#h6{cn)tctZ>58@msLEGDqN$oHFhGr|8+ZwRqM~#1>C8b zXl3xS+n(Drm>^_S3eyn^(%Uc#BCD7RQj^k=i&AdKXN9&q&j$s?rGEFOMSwS2Bh?mW zH#vRMwj{b5G2~j;tS2-$w_^2O9SGn|m2Ek6U}sgO{e05ER{%wqKjkW@eZ{@8!I{dj z4(`Y`1k1_E)xv~-9XxDlr*jax)BHOIuQ4=^VM68Bg)Ixsr|ac{>&iFVdfJxV_lGv4 z0J+zHzdJW6e&z7`uKju{F4ax{%-?Yiho`u%W*4?u>-~Os@#qgw4lNa$I>}S@!|&o} zO=S(n9xtBD!ln2#&fcl!zL&T_U5ut{rX7tDa}&T(x0T2hyK9!jca$N?^Eo+mx`{`9 zKG^Z3F3F`$I!3Zc?wnsCtt<;SrJt3ZGlBXZK6 zesN!)leSh&?J>uP$PRg5p6f5vdqvy6Y}@Vl5z`C3oCp!|DxlolG>7JGVCWEx*Z)38 z=%aPRd-Y_|z#q)k;&lZTo86YnIQP6i)E97yTA;n70IWN3Z)62EWF{tXRh$U=XZ;+o z^TveN)G*Mb9gpmat6*eEwypoO7Y-7%8$^mcIWKTyN>ob{;!pUNZQ#68vC^76-~Riz zjKl`%B&6a*HR9y*bV1pGCgc|ZY#8^>j*n4u*|!JHC}n~59Nacc%*fhLod_@DW=z6l zxDS{0ulHvpOX1DyJ)AiD`#K&0`e+1PqNH3Vt17=w8+>@xA-jnreGLu6;qe_rU)HT8 zj`U|rZL8Vhox9y41uA?1H>imk(%Nvvd4Ap!vkE0hk=uP=U#iF}=JW;-%iQ8+so2b) zZfR_TEJz6 zeCXWrFK+Umq;kWj^XCr4B@V~oePS(*hX}7F^jRsKJAXO_E~96zb~;8!in__z?z>J5 zrr_M=<&#%qu)CC^ju9V19~{T``{=fb{m_&;9gK+Zt?MtL z`ZzjN$$A!NUD0?i1%yrH;9A6T73=UvHUGJ2ddp!}`e=Yl<;?{;KgT(E#iBo+1uUm(Vqb4IuXf zn9o1BbYD~}mP4_dvo*ZbmC*}eWx6!mA6*b5zj2gS?PG9bp}jtUE-%*tSjL22+KWsK zlUBXY#1`eEq7g7#3<}%5cPCy`xU;=JpTS2nPI?L4-Gk)&7B&aA@vpYS+yAz-UN^1{ z2nDQdfZ@ipkw6+HZT_~As!gWEarUMqKN@Py6^c%0sHMI|Uz$S+0xZL+yW_3tFpmmQ z`l1l3TZ)2@G4J22?_=jK+m*}*p{goQLL@pZ6*`xfhKj#>8-@SrOqFK|Yps!H2>lE_ zK80FGmW)k>gaFsA=C5cH`a&jU6R0D~ExTh-?;(=C7%_=C0wY;hjI*ylmi_62QRmC$ zii6*GKHvneVZk|rJfi$jeK(n-D@BBr*D_^>!8_}@7;W7d0 zZ#KI=)@C~UP{q(akLdkC^XwZ_ijwonqkj~J_s4Lpz(a1%`9+7{^hC(aKc?R6QG=BU7zQz6IST469414C#Wd zT^3~T9x6va#jS71_2>+KeT)~@gwonU9zqVdXs|I|oDQ@ygs1#C zsS(ps)w``Y+b_0R(*X=I?+}RqpyKwP91-PI`_5XK8hU_S)$Qt{W%RvO;Xux~; z*^h5>PC81!NLdd+Tvx@RyUOpWPpL7%s{c*UP0r`9`#U+g__LM(S-tnz&}_Ji%$yuj z@h!N7C6(-V=?P_aQ%EdnBhi3L60}FN%&il1Hjskwih-HR@n;LaDuimB6e?NvE+fws z5vAWPxO;PpoE|U#5jV_siq|be7qaLYxXS z5>&pt#{$VHX%!6K@lG(LRV7jxGC?Mux%IZ&QTmu;6Ac$>^qsN47fReh<*RWdc`WbKhv4*9fnq_kBdC!i%=adx%pvbdf$lOCEe|5jh6M%$ zJV_f3ucB==Q>$nql}qIBr{lkS!v0vHo>*sGlvu3A`|kSOL>vWY9g_lI&MAGMpX{1z z15BQpIb6K2d#-OzE4IU+hYj{}zs%FqI@LE^00m{?*+4l)X#I%WSROKTt ztF^M>bez{;>TEu zMYfNoie#aTFS{EE4p%TCkOx@p&1{+31NPR)+U5Knt7W_x&SXA+VR-0Qh=WGMKV2V@ z(3s4fKKGVq91N)TGT!1B@YhlrOIn-Yz_9>Yo&s25_tIx_nfzgFc**Ge2lO`gF;(%(>Ma$5Q zh!R0LIg6E}Fo94kgf=!X;xlw`RSB8Rv;P`ilkf!h-c7L1t`8`wB z9=HBr8HpQwc=!F?_Y`F(lj}S6ZvoP;io=vL9d9~r5rL|V94P-YBw)$kF5gx&KS*}Z z{YZ$bgIm(dT28d}}@*qb;AU^{o*07griz`*kB z&5!h5+^SMfq9kz{aWoU1Y|w&OZXjsM{&zt7SJfoshhVcok%DaUk_O9NFv$`qQtJS% zm$3%NkOr|?%hQmY`B!rLqSPM~ z-fX5A($R{lB_PQ~+Z{l+Q`*Lt6-54hPc^y zhFa(rL4fI|H{h{`74AoIW!5$u4ukzFW2N_f_C^X$N=9eXvm%+e|FbGu@g5E4K9Jk6 zr6mGNO=d9y_clIX&{a6tsjJ$I*T=}#{W?N$B)VR$eu|SysB#7qZoE`GMU$lbne-WM}G{>m7OZG;m)P21$ZdByRJ(IzK!pmJ z5G^3VV{}|El>Ef9F3c1aS)FzIj#c*=!X|2|Ol{76+Xn^tdz`XW;;|dStQ?fRe8)Ig zLAiNusl>-^QHvrU5=Wt5jr-1194-W+UAV-dbPB2N^XCl4Zna7q98A0qfDl7d_iqVg zL$o5D*JXeBMDuNw7?Ve>ms;NS8}~-SX}vfQJZE1k>oc%BHVi)%Grb}f+2TeOs5t3#uS3=*$W%Ml{X zM&6_zRW>cRsPkuVKJo10P3B$y?#Lq$-@|jt?k7gv{-V}ACwi!_=XD&X!P@fl^9O@` zDo}`>B{XZm)WOc?6_Gvgll*8fe=ox>AMz+;t4o6?9wHbkuI^EMgRRp!vfl-vduAdC zfoA1My3mJMe5DMF1u=Uu6j3cDS-Q6*-}#Y=!L+vyk$G8#YlKW`MT^=ZLZQU0?qfk= zy@ZcB1_YnUJY$yk_YTVz6KFcQp7(zES!g9!hFrzJT85ys!sa8gCSDcesT9P}49SoX1*>&ls)YXn zK3y#peZSL3y;G@ZCOF`|T2ZBiF#>yUq*uvHY$|6KU<{kWV7~Gu%eyNCsi|prSc|#% zB&szgm$W;VSd}Gl-LOn^i&FR*Q+EbP7yE%PBgS!nDe1%Wa467=73AL&cN6= zk!m0ytvj5uom!P%Cz6`UG7LyfSL*95X6}d%$ynKzsSaaO(~(I$QLopG0OM|x>#8aL ztc{)3JUYYAQ7bKI(LL621l%o!>93vFW4iQWeMS1JQ`gx8f7bZM@jIH@OSRN=4`|7o_;k$ z1|35W>5VF+7f)sKUQmw0+ccn3Fq&pT*Ya_V3letWo)9U+lk)m1TT} zo-GT@%;H4oK<7{JQL3OxEJo?13n`ZQEM0!lph}w88F~={v%&f$uNwssnG?T!03&2S zo|Q0O4z{>{!*La|Vz13;lZi_gV-zDx zguV#qPDuW-_bWyvCy8I3r4KnItcMSQY-`BqhB$gU6yY{lgNu$r+W=m=yT_)JuQH^~ zTL!v;h1Yj%3&C)wTnxBEwf*%sMYqfr;~1#f_y-Q2ZLRGlN#ISs_wLS`XIZ%ax}@pK z$0H=zyXI;(hOTlw>-^z|iH*sm8LhQ{F5I%3)Bu!dTENE^QpRV<+zPFe(k*L zCvjX)ldj04BE3AXMoS2eUupBi=kFvN_A7=y@IY<;G)eyg19g25CJA1i!+r={x zWS=9PHH72JlM>L9AsA@eW(Ko9^9Ks%d>`g9{6n`}*i@p`9EZ9M8UufY1ZQY7<#}b7 zAu)l#XVIC5O-2E%j_do?NWPmxp~NHw`^nrA=AHPUIm@HJ^3GEQx><{WU>XYL-EP%Z-F#pHNUAt|G zK*ff6u@-5IqLBQg8cb?7}0Sh3L*->Vq&FnoYe9pp*Y;^UVCc*v>7g_M|TKdB8t z!%3mI=|XqMi=k~fD$Xi!Qj?HfNixQH*1t5DC6BQ;9vs>A%-dwGa(Lbe@n6Igl&Ejf zl$CgTFvqkpAL^=EuvZ>rksaq3XTI!WDE}c{B@x-u$ZLT zxfXm)gQ>Jpb!L7*;Tk^e=}s8T`UZb4vL=iWN+dq>$FNb?CT`?x1DJ?T%g}7>A-XWw zP^1D#v3%7!-yj|4A!g?C^$#4xPTE*~t~$zXAVwWlz9 z?f{4PEQncENMT3Ii;wu;roOFsugFygW7dTV%UZ;ixuITvUJ{bZvNk8 zI$nN5BO%_b>OvpQVnzazHtOkrsY{HujEk;~W)0BK?Wh7r6$ryGG+BFCJB9hOhULmFWsyE!+LSbnb zZM^#Q5lF;uAYoa9>r=kEzjmMGA)uPi}i$=G%B??FH`DZ<$3v)2P1zL;eHqA0)7L#Db)UikgyA0DY z;AzYbcD3f^+ugkAO{ocsrI z?}pV6s+R?3=6(9s64<0mLH_q&z#T}B69aeD@n*GDzNnfrIv0-oUi;~NkNv?XI!#gTDt$oAZs_X#O zxcYUiRihHi54h#OxMlEbva2Gpd}8(YW~TyAhC@JKzCqN<@r^oQZ0m-m_D>2k`HG0d zrsX4RJmIH9lc5MV1kcBP!aXeX;DqHB7l#jr3p>btr>LEb+)1`K&7Q(7-#k+RcfF-b zD(xyA^@Y>QFJC(C#Y;lMx6?i*J9|Qfb8A$1)l)wO*H-8%8$y*7<%6QG@3uXr{ zY^y6mfhcd`D#c{9t7aTrhi6YJaukB6^@iOJnqq)dM?;@c9-=gy(z*;cX5`W><7gQQ zKI1CiZ3?4|-bRXF$VGHHVw1Az0a-^B?5YbjS(SsqtUZrGO5V*;qPn4h18h4Fts1H} zBSP>3rS#%vH>USe*3!vm;1#kUYrTgMJ0K)XT@4JVq?J&#A^4@xmnc`TC#^&X0u31z zb+%TVohyIS_Fe&|A-I+0x)&kiTpUGP@<5v>4y3C0p8{I2v+m0@m5an{qoEHyF=vL)%6dt5>E$<_(!NH`Of8Tf!uvU#gQiHb zS3_-n^98e7w2tq=M6*C#DFLN~xFoiIZ2nq}0cAe>mt&?FVn_eNk7^N+XT9mLbA?9_ z*S>c3d$ON1Mg?7#^2*u#{N4%vlX+Fdjx|#fwDo(~1g6r$UwBGhg`LQTdX}X0r%3R` z>04~raEzMFm+6_y7KX?+`3}D62*WNAI@&1`I#buCU3U#;aZ|3{r$j~bTH0*v7p6Y* zuk%5aEg7-F;I+kJr|}ssr-zm`X2F%0#m0pvU^wPT~3k$iJ0nX%uhoVJ^n+>;V_VKCP}<*VvUCuSP-sOMrXy z{ojXyNj6Q7#ZlZWL{<|K3zF>ncOo4*CBD^K$r#)lal(3)!mI9p3zX7f^vaM-KJ+PS zNSSc{k;4$aH&zNQ5Teclkmls^T-FQqS9(HbCPqLXFKS#-zrs1vlUQ3o#d!#~M}^Xy zhn_LXv59VFs>U-Jq&X&LAuhFhcjFldx>E~8;;ftZLLrg;zFc=ccOiZL5eb=5@_2n~Y|ZA%6mv2Ozmr1kA|Y98*eJ$huKj_ zSO7U*ufeT$;1}<2x;V~oTydRyI zGn@ETsB>r>LZ77%d&v^uy^GZ-kGRVi2*?WKWeg9Cy^6cR)8{xR!`pK&93QofP>Gi# z{^VT@NPR7@xC_Dt!jC|1$Yda+85krn8%d8|R#dISvO!Rq5R5zb{-EEr+_G!aA7)+e zQ)#fa!h4dbu%+W9Fa7j3J8 zwD-+o-~DpY((of7A^+1;}^g#W&$U9qirs@X_qd}?s^Dzl)iM4_q)QttH)`L7GV597aW z%qP8+h=>2#UmkmDd(q&;jv<0+EmimAcQz9(GemZ@0K!l(`xW6H$?#v{B|fl}UDjpb zSHNXH^Y~DK@1tTA_8unSuclGh?_$ABL%sT2 z@l!$f@x7R}NRrGAA420Ezl@9mxCBLZ(;Z7KhgAEFbVL?J*%+(m2}j*w!R!W9x^%_ z1)KdbP5Kdm5~n?+Kg+g)7KbY8Yz~OZova5ff9-Jw!hK8@JIOwq3Rsd1+mQVvhTkWw z1A|Q=CWC%=?;Y*SoA8n9;QLass^wbs#HJN`eA|oWGg?-bJIBhK&@6)%e$fdlKvA?m80lq z%knm>>uu^LJ5Cj#R9RwjZT7Pj&FeNu&*XOu2}fCV0q# zKGBqKTYsgl=`CF(hf~q(YjET&pji1=1s^h`b^=b8yX8Qogk{_LiDyUCLdb2ce|l2J z0RJt8BTE$*D!m%vTfA{uyCkEaLynpKRBe`YV`v&?%6|&0qx;%Jgx`DWu%zgr`$B|3 z2j4_!vFbc#7FXu-8pKIqT8UN?@h)xQd&|N*L0>I|+P!fZfFlqX4G+?V?bTy!1sav@ z(=R37a++SWXp8SqIA*`L`)`gs-!gllqqew(;v*f0&5~o|&b&W-uS;nxh3IT@*H~BI zh-fTT1vS6lr1sl+-Zpk-;Vw1ai3mk2fmWK!geI z8alKj>BeGqfBP2U-pC0JD$d97@Et}yUgNKI%R9@aM4j)Xsmp^z+MF+vwDld;#i8No zblvA-yfmXeS1-H~ynq};)SZllWG>n5aUFbSN=z(9-G}*Q)A;T^ibido)~BGgp%tge zz?0EGTMdSTSW^$IIvX+0UVr^Kn)x7 zL82BtT!sDY6~*t!#H3lC1|X#w-BXhm?k*YHz+sr2D?*wT<0(QzTDQImg`p@|d(JP- z)%}1Syiqn)SvznoWUlPNPzUQ zr_0XOk1b|?4nKm?0q+=t723CLxk~Qm@=>#}Z*#@x)zDabJ7LjSJN-8FMSXo&iaZcS zC76(AbrO^uzTEoAJmm|#PI}&cExA?!L^XMQXMBz6@ViWDAu_bDv^XT%pD4(*&u2w} z$_z;o4}&=Pj-?3oYKrM_C^&|qU<{Gdekg}wg5+TV?E{}Qr;hmT3O-zZF0CYpNUYDq z{8`)tvl~YZcMr8TRJ>G@ua2%hQ%GE7=VA(EgGLQJD?|z^FJ_KUd1=QnQg)S1u@VTVr2tZ-!0iB2md~ zVbb=ObZrG2Rzj>nI7n<@K1&9U=Q@SloONSxb_b6=-4}d6binNErIwVY;-XRjoL*UY zqn^+n8?PLuj4%#8wnK9Q)OY0o(WAAG9Gu;Rz3!|CS~-)|;;+ix26F)c2t@`QxQ-F+q)sP)5rE?qC6R)Ep+7u6dSY#VT(lr`G+}vx&oz zp@fZQvC9{RlK`482Bg`ZQUe_*@HVcXmYC@6Q}Nwea-jFH)vF$k$yV+ZS{xPP3&BQcACovH>*sH zJjg7J?3kXJbA2-)LjXG?7+nqz#TJ6`vke-=(`ItTP<0Jj#8rS5=wl4tfy)Hira*3% zSQzgi(;=?Sn$Dx10<@@ylHV=feV+tV7}^5nXXP{p$N{O12yN#S7&6$UE5u;N3~)Od zU?Y4Ld#KK&S+U;ASOM`4V~QYU7Hz}yV+?J2!!(;ScF0;gcxcqG zz(j@9#qnGxr|qv8o*hLprMSGF90_Ey97v6|UQGoEU=+`M+8=Q@3F!^KKUlZN_TdhB z6{@obSN|0KSQ%aOVo|oiSw)U;dmA-3m zFh!q#P*9suC#2?Bge-c4HapeV61=wd?%G1a+d@dg%{l$rq~)iyRot@|$rb&dJmlQn zhWscfC<=fK^Wx&-a?Nw)xTTL@%Hg8T&dp#dtYox9A_yi0R#r|tvhHhrlW3zr4N3@f z*kI{7A@dUV|3%YTM#a@MT{{c}3GVLh5Zp-!?(Xiv9fEsscMtCF?(Xg`!QJIMxu5S{ zi+>C=J#?S0?y6na=8Ma1%ZK)A2#$df9VqTW%@MWl*BK?YRwc#FU(k-=poww7E$N71W!o%i~!BuGL$RIH)N2AvHZ1q+Mn z%8+mI1f3We<7O)|17@o>z@m>Jzq(x>w-PJ|YzOt6)T-KQr}X`Ei8oyX<+NwVZREXqM4=iD(C zp(#*RpvtavWXxM$+`qV~N`i`?wlk4>Z>M{RY};o#%-AIh5HT_q#TmIA+)Godt?S!r zhcYsC6DWZrs99n3zNkh6rB|sC=)2Q`%Nemu|JWTE(6xMFt@c`vG~1w6CDgP{*w+JUamTH-Ae z6h5%VMr&r+=o}(IC@d=)u6Gt*05fyKj8Gp)aM7|bTf%`tDN?!yL!%?;`48#>PVS=z z@Q-ybw%5Ranm8wGt4#fndkn2!94QFD<945lokK_2A^bX3&^mUNu8FlJH^-_x8q?-a zBe19WE5e}68_wlzJkBQRUn;%+>I*BB{ryPvkv?c00T#xU7OczzbC0lZ3}NFJSk)$( z9uz1JQd)VFw!236ZDOTVZ{Y|hp%%XZX2w-tSXP$$w(H)IEJs^7)|HH0WMe$qN3J~` zx2?P)UASwd!ay0bjF;7WkHi9T|D9}09-;(?_OIt%5wfwVSl1H-5qa0o#I)H(NiUDg z^=oHRIZNS{`kE&P+0z~7Q9c7>kzaZ*cJME%kE4;Wn)OsTPjWGsD6})Ku~%QjAz~8@ z-wrMXMQ|Q=zitx~{P;wmrdg4>`^{_#=x_A}L+UbMN#6fa2IJ*&rd+i(bUl~gSC^p{ zUX<6cq*G|G*3PQ;;@iSiqmElX1jY5kTy>2+&4*1}c%`hRKqF;{Y=1#-w}U_;TSB^3 z{^K~uf7Ck0Ck{qR6>LUqHW&-X9m8AKyMFD|J^{ftdZyp2#5DTMjo*Dy8oWUu0G&5kTa>zvJ9Q8W5Z+J`h>nxxVmB9)%P_7m8HJ$QXUMPs&met&ekji%tR5{_wy6k z>g}3HMJkYVU)sq22f;=Jsb!$bp-Z<+}C`A28BZ-8FiE{4}e> zgO7npvH5}3Q@F19l8zH@<0TQ2@&$mU3A=ZjiL%d`W+!H zY9Oq{ReaQxeRVq=ycoLcY}5Hu>Cy(KnXFSZ>3u?IlVB%_-J}t#P6D;TPegtw*$3>S z|G-6KJaZ+nBkEfc`ILofYEsE$1r*1dftg;Yzqlbi?a4)P3}k~u@`qCXC?c@wb?e(- zE6F35)=*j$-Va5F<424$XyTc4;Xrh}UoRRj)#(pNo_&+pOk}wDIQ4XUT^Zb;?h!pZ z2q|YMSrBoYSt7%!DgYW*@^bROoxGv+pfXxHcAFY68?U(H^O{%f;RA?uQ7x`NoAiND zCJ&OWx2dmWa5 zfdeG8g_VX?ain<`7sY$U?bOMk1+2H*KQhYOp0X0$f=$2oXGf2?QGX=gl$0o$ZKLp_n=gQJVLv-*r>p>Vb{t?j?p=qko%470W3c$ z;oC9k^2@Q#(^Gerq^)b!w9%WH7aY4_=suOGMq$wHwLw+rH($%Y9_B%US;t3Toh}`L zJ~JLL3;{tE5>+|`4n22&JuN-ooTT%_DjNr(+WXYZb!U}h+GFD}KjyE8v<_s(8WM#i z6Bk^JX&)`>*D`zJirR6R-;>|=m+j}AEG^9AazFTKat|XCm-MT@I&Z7JeRdy@s*H&qk`xhnFZMW7&TzTl6_=1c zX6;M%ionLmt;~h@UMA}Qi!XVM|LY=JYt38Ub{#Hr$u2IziAgfj%Teje1R8XO^pTaR zw{gP{3TK((01JzY6Zd?ocCk@xgW$oDa81*pA@z*P%J2+qd;4{9YEEq;Fk-N0D6mBq z$1RcWJNL^B|9|8qA{!rekD)x`uNM|K45i4H=9V$qjAtlyxVi_A=~5lmHDpne>?Ml? z7_MoB>#WlE?H#i2RhZuHb{sMV;BAjEBz~H!8stkKGxhsYc#$3NS8uRx()Tfz#WZEH z0rD`hEkw$?ijYu8qKIuyf)(%E^9nJV#Dn}?1vX5TPupz2FwN?(mU~R0W3!N1M2EsP zm8O2DW}cU-JulZD+<**EZwF>U%YtT(MN&F!&9kPHZG1@<5BtwnI!Qdd0*~k4u3#_Y zS~EK&{l5D<#`%-_-97veQugO{nRIx5=~?q}3iNVv3jId#d-JWnQyXgQ zf32-tJF45g;y?@+okOE?Y#$CBgSVh~zB@X}`6ZD4-(6DtOXn@F--C9}?Hl|~HU>Kf zJ8S_72?{B<^x5v<%ZJHZRtJ(D2j^{QPR&Qe+x1|e_ng4F$W3G=AA@)D2NYNBQ}}yM zW}SU(=7Q_C1tYYhA5~FlL7s>}WocR5$g708xvf+Mo2LTDBVs0ns_X&+IMOcXehYOf zRe0?}feKg0fh%QCQ>#Grc>uwKj7rp-@^#Dh!$%sU#e*Iw#Tk8 zzcf;;ify}X`>hP{>s!n$jMpaY{hjWx>mE-=vGZvyY<|P0#38|z%i~5}^twf=)vRTP zN`c!JehkR3a64}{=SilW*JYW$U0%r*?z=BGLj#oT5}Foi>~p(8gTpz92*Z*O?PO;#r5hP1k!L~^Fx&!`?gLu1xpyzj?;IXj9>$<;M zc{C}eC>=LYe-Yt%z4OK>G2Kk!$@1ABOQ*x-<3`(;c|)`Mw$^m~@H0p0)XG8DL{dDh z0>tdt$$OJb!>g5KqlwzgInX~|{+(mVCk=7TvcA#zBeGkX&sr|rN8N*x-}7+xIf!sG z$MtGy3sq&r3F$fQf@bS-m`;!*W_tsmy)z%@OdmXYe;w}Sh-5%S6`xh@_ zrQ!JyAl{QfOl@XONQBzYMIBq-Nl!fO0>^zByQD7!%x-|0RkOSo>!aw3JNw7U&Hl#W z`Fv1sW#OB~7Pgx02S}9JIqgOe9QSB(8T|ZbL@*^K6P?@u!?D;6^UrfCn2;~i1OyN` zYMR!M`|v9)w__QTL+PytL$~v@PUz5)e#jw&Ej;h`!4QE?u4GkjPwrVsQC@E`ypIF^ zD`5~(OkUxg_dAOaL8K!)HuQ;|>yI2?xLiUtJ?29JVQ3S|2e{3?lBMe9ryi2!rc)6U^Meiqf`;Oh{_ zEo(3sJ7lB7Dc-+&ywH;w&ql=}tFezKV^_I7Ia$>giF!EiZWTuAJpPjGR zC<lxl zz(RW4dT3${M%j3|sS!9i(L07$B)9&N?yP0y{j=5MA@1TB`%^ByC*Pm(S(}s!o8)M) zzxH2iffePM(VKbkZ3Gh#@~NYz#A0pSxODJM+RR0uw^nIjvNn$nRyb0JY?E6|o0Vq} z>X#8bJs`gY7cbXpPDmJk%YXmW<^LrY<8u&ATMcYy^^{SY0Z&1Q&1)8JK1t#C%{#-@ z*bBOtV1BP5;5WGZR9^31D}n}zjr(R`mlI75d^h~ zcwQP1L-#NNjm-+sM}tAy(>ctczoVvBHlB3Tq8g?9AFnDaPnY993881vdgu!g=%Kl$ zV0ar=caeH}sfxN^5ghvlVIjBDwWD^S;e>X=(ivW^;B6ZNO9qf%$R&CHTu z#Zhk64eQ!3|KbdvXtc`84C_B$>Fdwy%Duu~Z<z*4sLP)ejb!Q0N&1@>7W582}Af{tzq#GbohmXK>!sa*^<| zcV2j&sGIScK$cH#K8OxyA_<#kK&7qd`X_?Xd)N5wVaA_vSit6`koW5~GE4|UOxF3c zIi+Z7{$b6a7xlVH4v%gMtO+&|bc(&F3x6bp_-3 z_=O|tV$2J7`ORtir+XCoz5zsjkR!us^HhkcgXdt>mjb&hW9UlkjuhduMp;^}S1U?Q$iR z`{o?7akZM0KcmpeXsKrvjUMZ^p9ep2Ysc4w8UGqVfwW!p+l>J2qUyFJ?*xAfRbu~a z3Y&K+wK*#`6cT1{GKzgB%zWHS$_q<%$=+=d1fn;P^_;c27C;3v6d(N5`DvEjl+W(y z&bzu-kGaC>hQD_fvKLWD`eughv+`B5!PVW}d@R<@+a(^$=XU86jhp4#+Z#|NjrGB= z=T~fwC=uLpE0;wt+`Fb#(%T9|8xOV7o9u3Z_P=!tZ6rt%%S6| zGbOrZ$4NrNwp!hGdSfj+MnSvBAU_X&|9)CL2z?4>vX!Y9914WO=22LuD`2C8}5Cn&V2D{DSh zB{nnNN|!!5clD)?WV`!%LYj~Jo!;uAa62;Ckd{Ky!S`_A-ScjaG^Z+dgrc-xN?pDx+IO@Zo225=zFYVf+a|zgV9dVCCB9j9zOiomPCsMiGAK$_Kkt5{ApoWO5Gmu^giI=aV~dB3K1o?h{8fv zFnb97%i-VNq5MsY>Jj9k)IH8oFijdq>SS$@zKFdjFkzu};%cp9OTSwidf}Gq*j{k? z)WvEoyO>)&K9z>+v^AYNnVFx3SvQ}&!!CfC>7p$-xZ><`=Cb&_e#XX|6U2cAxnB%l z2*GUzaZLPp7Q+|im@0g{(&@_?zvWRgXE%S>#@8+~!i6@3G!~X0$$Dd$e#OhqD&zO+ zpO?+OSqw?{jNrB%%RFklo+4|s+ALkediS$%P-bb(M>yaxp;5Q^rnno)ccVYnkB3Z3 zmN*pGc&aIKKB1sIlY-W}tws^u%F1d5HkgrYFI#8|_K$ey(>X`Kz?MSNW&WfA7g*k? z+0Qek06Kw@26vP5@%Ss%sHCYPIj*a}F!*3$EJV;ShV5IvpKFe~h1X?5KNXjMEYVOd zW}zJw{AzU{a54!21SI&b-?q;J<;`^Tw6d`j?~;)PFToxCf>e-nZ*4w~tje_mi3#|~ zG@zpNbX}dtVG9h9<)F<$UQ}CFZ^sV0^R(dDFT`R6l_&y#dTU{AkXU~od=`NW2i*|< zbinq1>hRb|`E*Tu?@aHsG3#7;nH!dk{o@0-|KIFbX`P7hgo|1ZvPE22{+k_J=^c%Z zzZ@J+hQBAl3jE-yqvfFp%AZcvpWV8p#Llm@%3*>j7uw>z&aK!R)K||d-q?@r3Ptm( z1B@VIDM;mD&w>w8A*oG5vi}Be5H6CW|7cGDrmD(1Hh=1ioj{OZl@U1#x-k87so zd^da%P*CJ&)5!i)&<*_M$zTDz1~9zOL;6Ft0U_bXg5wHcLvG2h8_ufwn@D(XGoX}G zZSC_9LsI3zp=EsYV5WX9qY#sfMllSn=l`DtzJx+AtB+cqUwQeY*3KZNJ-==xb~`<6g4p;aVWA> zf&*FMYz33;c)JK719#VVC30?W8nrAvB3uU;xYGkfUlFneg#v$T-uV5k6kn8?Nwd{P z?6hx66d+3TaqFHFxK)dW$<}*D;ijW2x73MNfX#2Tz%*|{`4Dh71tVB=I(z9%PMMCi zOi4*04#Bsa(Mw}D7E8Owu(r|Fy?A^6<(_6xA~9V%hYf-pid;qjqA&BI38Qx8AQ!hPd_4PJrg4Y5jDMmo+(#inI}l zP>Ot!x7Ls_9#U{{^w1gan1H`#P$lLZmMPt1yqDPa*Cp?;IdhU+OPDvSoh4u>sps(A zLuQ@rWO?&;jkSWj(>?LMW5|T$>J;DOZ6FDhBI~891lR!CpV?K}KVbFP$_DxeT%JAd zBGlH5CUGhj1q_#(SL6=ou$nw!y}pi2UInUlW*fW?mg6lRnyOCP{o<}5^J+L0Xdb;g zA|&jugR+iPKX>z-JXDZB-4msZVP`3Sv9_ALImm^~>7f zMx3gbm!3k!elD}s%i*5T;qA=rG^lk(r1tk=S!dor&8mjAcyel+{+`o>FCKcXbyp(OT*NGxnDnw{IJmJAu&=`q%jq z8_xRBm3$Q9tUMTH!3RbhP_1o`K%4s6quleHUJq(g1AVG2Z)K4t^R59Ai((~Go z<$3tA0jo90<9logUOP|g?>)}@;|ZF1BuN33VyOOFykza|J@zu}i)~ls97-#6S6d;X z->z5A-Ho#hW>={aEtZ}si3Tj(3k`dJ{yJQC9C*?7%F2tK#@+M~+W&}W!h#WsnN(?B zCCJy70cJ@q3!IwTnEdtnHB8TKk$h&0^v8T>bflq%&86LgT9&v@CP% zepvq6TQvSH->TmJIaknhrX;71t9{e;GvnLAjONbnepve7i~HYhq}D33F9v!)!MWnc z0yS@66*gw=$lX@lx%2z%$kBPcn=XQnZ92_m9HMIeo;I^h7N0UbyzTHiXpa%FaLo*q z+jlh1lIc0h!D*~&W)MJG#EKHceUD>ibAyaLAe>=$2`rgu|o8PE)!p8aR9E*%g{?fxj_US8fHmIm}s zR=Vln$4N6~J>8fp>3kPAdOc{|RO6c%F#X_ zxs~doG;Ch&^31ubY&6zLs@hkEiYm<*h{yRs)krSZcge@>NrqXWYFf z1zgrBzV7OJgh#z!h)Uo+)sVZ0^obKUCPDp-YP*DH|1e?63n1O@l?Z@ zcD%=OGB#+c^h@x`l9lZ#mhQ=KpG7nsa_0=ML`Zxl9e%Ir>o{W+UxvXBQXHp7a~LJ= zYA(!3RpBR^?R>g~s9+3`<$xj3+I0LaH*AD|xoc91x?v8txWT#DlVA5S%w)rjmS)UWG$(Eww_5xV`@LE_>ix4*NfWUwD%SPlG=H*( zDq2AW%u-RY%0dc;A=C6C{;VOmXp;%e(h&tGQc*j`v3`j5i+?J|X$v|Qcn=7M`n6fk z`;W0*G#pa&Uohq|3|{ufUG=U>RW4fI9xu5)Tp#)J#$_B*-y2*%!Y*s7#TBZDU+l8K zXdJ{R@ZbVwA3txRUai$|R4xBrWU_TKqPWv`dXo$acH;Uctm64^4DPx7J;#37j>NY3 zm4$PCG7|ZvP4SK5wf9w-0$LI(Ug7*Y|6`{ijbm6s>pGS)sc3DEZ&3!FZt6?*P?9yU z#m_1VDC`x?ga{4C2Wy@;h3t;Rd*&DJ zN`#x3*9_{-R&o>{7JTeobmp0%s_#WAO}cJ$R4{APMzTKHQ~d4w#XTc+_~ITfm*SHs z$cy!C=DiN-X+4`Cm2kyPYJ8tPSTz`U-S2g>FH6L=XM@{5a6CDbGh&1V?pzeQa{Nn` zU5}@psJco0TnT!H88yeHd zqd*^a@YB+wZvgfLsknbYqJ&4G$fyywu08*6*4KhV!=%b^JiQ z%;>jSdmI1{4A#stx42F$34uE4Y;#elWh&umK8!UZdOtTL;9^q{s`YCRX$#*c5L>cJ zT(Z*X{@FY{ENzR4*(x)zOF!0uq!lR|KZ@K*n`h_jD;N|x7&tOhLbR2&4SVz)8sJLW zl>jY6?8>?Uiwr|o!7j)wFa|-V-i8VcUM-_T zJ%c(?2~PrER!By;fu(1+X?PP?{zqQM zMZTP4k85ygST$pPK}5wy3beaIwQbw_y?(AY?Nk53jbjfve1d_D6+*k81WzpxNCBpa z9E}%_kB{&Cg4B?Zju#9}iWAIGhCKE9&pdX1FBFicpDo!(w02(sm<4881+$sSEtOX3 zh=ZC8$6~e!y;g)zpS^ggb%`_Ee9=GP@;7L#X&>|y)4U0hZ}V(ub4&belz<|#%`JoY zplZlb>^X8Y|2|CLE=VUm4gkdyRhz!^zdLWrTSQ2mN&+QS9SCk79%9IxX$>t~-o8Xm z`?KtEA);{6Ub9kDJvZ`@*1jGIBKq6XWqU@(MCf52#$ z3MeM!u+b)T1uPUp0flPl>>VU7I*W)h@kr^I9p8QtGQiOl+|esdrJ+9^w+2eVWO|yQ z3`ll8@HTKJkx_VeASJ`vz_tPh2S9>z9LIghpiJytX&Bx2gIP&p9zH8>ak{y}9=12&Fvxt4Q5LxVxUNEL7KBve~f z{HF-$4G9Un0l+EI?hnP*yy@-yGkNDhMgmtE8OEs}_DU5GQMAoJ1$whlr+nCsTf2Sj zMWOnMC^#_=LuGkvRb+%g&%{K`RWmd=w6~;aB(euB_HP7n z?CKnVRZ92Kvw!$J3JRcL{psB`s;#U3Qw-D&Ek@xdxJjhM_7$N46Q=_JL)(7xEoh(+ za@UjoJuoQy!`iqZG12xY&APX+7z{{tfXs!PqWHkTOuIPIZ%aDlY(@Du~Tw4NV+QQ zNI!F14ZRJ=%B+6$5${$7)U&2jo|CTwkAH+~R!NP=T@jzWp45sPhgyg*Ox4-%bLWRS zVz zwWzct{>;?j4Y}WkPul~f9KSpoZy3BD@SPjO;Q|y25DVkvH^f^L-bjJ42Wpbj*37kpZ7(t!H{f!%<2ANKgDX8EvVdVga zFHC%y71Zt9?do`EdX@ju8T5LxczL_&75PT_5Iar;^Bauzj+X{!jRWCzuYY%ldT3x> z@*I~;SO*9ZfK_>F0S?4V5LQFlnB`Op{K036Bd)Og^kiimKQdH zoLc#N!tH9K@sbQ?tGTcM#Q7!!8I)6UgZpNJhR7WB5I?qfDch0=xO3Is+9P*}?r`}~ zgE|8Thsa;1dCxY_DwFMX*JGAVA$L(8ua}Y8r6w1?J}Uj%bRSp!^?C5t-63{X)`i{s z&lYr01DU`sAw)q01c)sAdquX~@KAA+L7bNLx%BMS z%INcWRtu7V>y7{tE_$#)`yKDb$|Lu2T8ncFw+_kf7#1#G@IWHuT>Tu5s2Q%<^VNAh zG#qr#&R2!Oh6_An$(PiBWbomqVHL!AE1)A*-%J z28-b|?rj;!J5~efcW=AyWHN4FwPtnoufaafZ@%M-P7*qoAwj0m(=ciDTWME1Wz~$- zfL#tP3u!=)XkXs9klRIct3>a_`cy-U4aL50sK>U@h?iFuSa*h?8uK~y_DTw=SOrOThv5z!3tV_Li;!w>4&S^gwBMq~O50cG zh8_k|AfS(>$z~b+@qkqJ+YUZCyYp%e{n3$8ef|b5jXuQU$yzV?Xq|eyYF4FqS2cvh zafJ}SgI2wpn=3IJuXG@pMk(okZ~P$*K|vuiz4r0hy19Cv0*T?;K`(UXvu44=5&^1 z&s)N6x7%Sw#ih>l z0FE5ZTsIcAH};IF<~;}vT!p<%d?_G50une1 zE>-G+;_B4YLUge$`5V2U4#f*mp=a!U_#IzzA4dv1eRpa|_9NEaJsL-GY{9`~p{0KS zkR!>&1ml%5UzMyw@ve6WGEWu2KOLo>arAtOEL`w=ZCP_IusrcMuo0%1hd8iE4MleA z5-}U5C|*?u8dtc}wlfaI`tS@fs)hMG@vRFi-80OZ z+|^(5%773^y{9yV#0XhUmPN z#!21W8g#MKSR)#>&KJ_UhUP^~%&Q&@Gngjr;*2I&xaL5vCih{_iaZP_Vz6H1Xn1Mr z6)Pu~pC}CSW&Lj&IH|&jWjyUXr)8yCNrV(;{qVp!rj{E&C_{Mn(wPW}FvEwE@Co5u z+gac~Ybi{({T5bhq}41q<(A9HUX7;3`bM3gHeXh{B`>y4{lulb|9^Y#A5(d1T$mVp@|@{{UiU84QTR0=pld0Tu!f~L;X`~Zg@xz$30e_FNCf<5r+mfIG2E|e)A zFzQkoa+US!{?CJ@aDX6B27CcU;?xk`H*d;F%!4QlS1oXyoI47ipJ4OjHHyjV6z=sV zj^nDZF-z3kI7ITuL?*v=$96ZL*35(XlK|! zfynE@zOKQjue8~P8Ap>zV<;nmT5&unPf)~4GW%!|t!T4$jG(g{Mh~b585?Uq^ccP< zP6!0{XC65=DkD%3q@__~a zw98f1OckCr_=*MTofp^VqB%y9NdGt67k8;Nr`eM0&oG?9g6^g~fP!&t;e}?d-#NPd zv%;KSHV|`(zViy0fns9>cOd$I8#~B?YTm<4>G~f89!P%#{MdTp@QQL_?+h$Rq(}mb zMd6nk)lRkBoLMP#y)G(Jk}Z$ik0GnRcVE9HS`-t--s=Ut~Gv=zLXK)77u7O zBryo?MnSrm+siYj3D5ph?NQZx$&E>bgpCo%=yhEB@5o|Dae|iVb#=(;_O}Vv43b`T zmnnacjov0VCN*zdqw$2aVEpUMA9QHgYb5yc4yrTXn2M@@*89Wel)D8yBO@b7nU08 zT7QMEw_}&8sT8?M&7NQz3^|%r;e3MxM4FRNEU0YwF#!*#Y2Rh+eBQ6EB);IKQ*nkC zurG#qI)36#$|NCm<9t86A}pDcKB?DSo@~3v-Alr9;P8%3lo=hJg;lMG*b^}W7A_Of zZ}ZylDz*6OeIQD)m5giFT&2L5V8vSQl~^7Dm0WnW4o$r8dI~#Au0OrEqH4b<3Ay4L zqX0R%vUK;>uSURW>vBxM%fFt7Ur@5N`nu$o`!AtzHF3d;hk-ivU%JF2ht?1bNjg3O zB+VA*stzag#eF5IN4{)>L4$Zwg700EQ{Q3iM=HB!zS<#R#LRNV2wEE9bzDs~UKOKf zsi3!c-OVuC@*k$7lGk`0*A2k|*^KOp{DtQ*eAj-c)eYknI;iUdB><5LHq1?w*7&u) zH9odJusqBsZ??p|k67OF5>T))qvb zx83jZ5$;{)j$IG`%nmJmgjye=1-1cpESdT@d4oV!i?_i3YNLIRzdc06O{|L^MvI}f zh@CbfRR2+Ml0&e_8>_^;>;6a2;RFSd&eFgFWI8?nzNI{8rwmL0M9@|>TU2SCrkilH z>$8o(pzrL9Bh49nHSZSKRS%%uZMKsWog}kbc-FRH_FR)98_Yiv`x^QkbS^IFysX>g zg^@Cys|4-quE133RHRFVqlu%Wqjkgg2~eez^zJ$nBOReagaq@szuo#=(K{pi_~Qh1 z9o;gQeCip4sl7TsXuJY$3|qg1t1I@CAoc)xXA4{$GoY?E?9OO05!)8)(uUTeZVgYo z#|V$aJ!GKKEGu-?%NOS^@4$gy>KM9gT8ADgM;Vr~Ws?2SVQc-i)rEn4d3V*H5q93K z80Cow*fl+#jmjF%cEBUq{2@r|vl54Q2)SEOG+d$MD~ALufr_fY?DoVqRAKlGT2ln0 zZJ6rPG)9PRZ&Gk8&AK-9x=j(tnB13mmDp9bzK2w^A9Erh7ao5jqGre5^t#=wGMQdq zax>!by18bwBNr$YMSSB$XFQ!RS@W{-3@kmuytz2e z*4E_EcTvi%qG>Kt{v*Zx*@kgPL6V(sdJN%Ica?f*Z8#hDHPn>p>g@%GEnw(*5cnYN z`_Z=*Wy2TTDBuPTFAblFxZ{)AOERl0AG9XnnKuUge`6Gl%Yk_OovXKdg-9;@+I0s^&tLHv%!(T_ z-(M!}UVG%%&Hf(l2haUxHA>L2Z;~=7yGN+hTcHbP1g9kP^@2%f9rNqgs;s9H&r_X^PcUmMa{;~AWATLftMxMHn!bqma;zYMwck_*u zrew4nyT{s-yrTS14xYm1n!L2c75~G^h3)Oy_Bpkz4(~*r`a0`l07Sa1V2O3egYtK% zy-^0uO$bmuk{=|!g}iPAq@}Kr>l!hHQSYeQnP!pFQOO16LO*ejElMx)<-^@zVWKzA zrpF_-(xQsc7otF~+jJQS?d~9sP=#%KY-rgy*h= zECb`fKeV-yladH*6gr)OHz$AWy7y?QI#GLTpdIAt574%E?>GXEQBf#LR|a2h?y zd?^+HnTPy+JU8|BF|JiH*xk5$d@Jo_wzjh!K6Tw02b}7YDyp6gLyB z2kM%o;huo-y4Il=L0WK#2i*CeKpn!V#5yg}JINLA1 zartfLV3g?fpxW7koVp4KOvyKoBHD`)C> zBn%X5j`?=JcKKB;|DIG2W%EI6)HjQ*(^pzf@BOl0PEW_=u5NDI9mWs4bNg|RYNcIw z8wcKh^6Nk6&WNmPqD2C?u&J=F`V!vsSp!{6BRbzT9MjrxpS%@3j2P=gY`&^QW+O%@ zr06n;SRD)9b8E++0+wV~`co?2aOO;0Ka_eq5*R^ovqimvk>JmIDDhv8(QT z&tXOZr`3JsUg06)a=^vGom!rT0Yczf=k+&_^K(X`smhbYbUvqpC15}tsGO=E8tC-N z>8xi5s2=l-U@(lgSJ%Eq)=Y~4{VTV}P==IKT}JI$qgxg;x?+)b$zk@IHu82TOKm5a7*JEG9=rQ0s*(2 z-0bJnFXT<0w;9oD-`P>B5`B#D)LL)A(Gt-9$g^nVhnOmShbCK(|9fX~qSO7r|8a5L z#BH|w0k9Tq*Is^)IPo&;hen8YGHWSYC4ANCZ!P2NE)#dKeTaRDhq@PG2+|z^CHe|*we>l@u0E1IrzO7Qv$ETOi zf?v_ZP$%Yxxp#=#c+}a($d1cRGpmnm`AfobAS$qz>yuS`K*P< z9SQ?M*Ke;%$uz9@30_Jte4!*&B3>oo`;1hAc~m}n{?A%PW9iQ;+^cd6=AF^#fQMmU z{{V3(lP>o7y&!gkS)I86+lp-Pz2PW@$YT`2GmHpDM9qE`UKoT{qF`LZmDyV}A`{(L z{Mk6u=CaIt6-%3Q2B$$w{ef7`f}O=1&1gVH2AvmxsF+#m@AiVkMuAh1Zu=`H${s7& z$5oT`JL$YY-p0=WzV{PRq!<)mpQ=x-)Z_!k9 z{!^;4?|K-ph{u6G@vCWR4W9slISq@i&fcGYJL|unsWd?ccNU}Fh07x;vH-Gs{_#Z> z6eLDxU1s1KR30#V9!ZIWj1*EV&Z1BMSfP`p?%U6YQV<4J=a(Q#z`+6%(RYu*_d}8WIL*5S*gn+-a3%=AzYg_*RnbVeO4rKw6ZfybHZ|RT1T|; z#M$pPG{v>)X}eK6IO3+yFo;+oUB8cFhJNGuzBRYXFDh7&O;gkUbM}n(0fSLuP23fG z|0eZ2t1n*NSw}z-`kN?FVB*O=xSSwjB6VW-^AxHAp4qW=6WbsupvD9g3B|hZpv4s4 z_?CH)QPIL+VB)l8zh#*3T~%VKIY5>r>V-u>gi8xrGt&*Jg~MYmd^d|P!b#!YGy?sAzsu9lrijB@PARcS0;X*OBFcakBeQBKq-R9B8 zbP?jF_GtV9bRXU95XA4tZx1`k-c!TM0^eFqRPeg}KvlbcW-I~vvf%?${v^*~e4_x# zPo~&ps&pEB6&6|-CrQx2EOw;dh3cTf1v{kpo-RR0yb>VUUut#7}{X-f)$ zh>5`(q|g&=&Md(rx2bUGgU$6BW=*1tV+xZQn|b{MWst&PCpARvea?RVX|6@FQKyd- znCccXB5%q2pMefQoq;IOaS2bf3tXER;wTSl!ETN#ZWek>JS|EZQ(&Ot^h}NG7$NcA zVMW4rX9GfHj1Y`bmr)5rX~XgOxVjW6x8c!%x$jXLE6ztfYW%yI<&pH;ULc#NO>;67 zF#0kQB3%D{8}V!#Boo9m)A{;SUPn*?FuBAKw_OMsjG|N6V9Rs$OolK3pS%86!`+br zzu(Rj6nOY$dSfOs`Y75O4oZj5N44YiOAVq4Z7&-q4VLI8UuY-)t?H0T86b!6;H%tq z06KNMk)u0_uN!~77bM%qB3P5@7+ZjVG9a6LWFai5rr5%3z`(M#O&py&+QZY5Y>N(* zv2s=k^ah>2?#hsVA4rG0qrxq3;BQ1Pj#IqaVnOU-9IAYIl+5ymIq_;h&~-bZy}Z3w zN~@teJE7y_VuWcz@vU8y-PKUeT&W}XR$Ri9?Lzxbl$Fzt3%|7S<8j3pS}gl5Jbsze z(Mq7_wKp0k^WbZXJhCV15MO#&AtwJiy^*E^F84|cik<1$MAzw{NvVVACRgahy-eA2 zcZv0`vdoV{QNxK{2cR3|Y=}QT0@*>dE45=C^#%s;e$0m6Q`PMs?lDYGLONtc&Q6Qq=Q74gH#2kft>96z6jI+b81Sh-VVX-HN`cq|H=n-DFS1GW@M=w`{&Gm+YYj?){W^d1R(X2C zPa&H}cGm5U7ZBBn7$l8bFrhyCP|yuHjf!=VfZQEia&qbk!h#tU5fH-|3@8hc^`L!c zryLAKQK7G3`@fI>UjOxkibu;C4unHIde9-MZyI#CHZz9k2ESjs_iAZdm0@CpZ@`z| zR~n8T(2^a^5Z%+-ty$%N-Dk-Fral=H(Z8F6oRXr?eoCV+HWXxEei#C)o>Mg(T8})0 zfuDi$*JID-R$@pipvEsMJr1NW=zK*py&*KS)z)xTz+33V$D;1l^50~MBI1T}s@9@okFkuD;X z*sO%)h&!i82(_YuZGO^y&C2~4q)BLhYpXtfDXqG8rN=SEicOzW_f;0z^PU(s<7P+&ODhB!=)gH%$ll6eE zy_JOPM3_3AkMZx-as>bud2vf6f4RM8z(pN-_R3pu=^&u~N`Ub^akp~nsSJCiAe&yU zOvR-68Pw=K1tBBDG}uR(w~B_>^VgGEwf9(e4F7T)>RN5xVq|uea@z8+_-lqQ3v;Y% z89L@X2{rwRE2Rv1`yKEg@g!Pm`89R*wtsKx0(s3G0+WJnFS#f17@KXHHFr@UH~O>W z_8&o(aZd>PA@3Vp9*LL{OweO5;)T)CBX!$a&t;-SeS3aublSs0O)h^_gdK$?6v(RnD}XgB~0sTvuFtkF2B z-?QHqWaxQ~0VW%>^i3B-LZsN0tCnkNr1on6=u}+JLz~qLsYpZ>G?oIoo)ZF4eTBGH z@?67q5+bx|4cLew*bO6QYWOoaso27@NojK#&FBDT;qaVY`kxhs5`g6C?OewEAhJe* zLk$$ts|7tY+{#Wqa&yzN(^%9Uw(T}mV*=BOn>0=QZpkMT+TWL4P+6GKX=?e%1PT@` zPb$INLdh=jmOVxJM#N1qD*Dv+Pw~eovpa)rjR(dJd70#R^( z=@m6dgQbL~$a#@1lI1Wrfb?KPb<@w=<;IX*gr5RrXQOj5+R!{=wfkreyx^(iPiD%` z>2u_UY?JiA(y(MruAf>y&leqo%TB}?`t```{VFMNaYH|fzi{;h#p(?Gql?~0qnvqP zl+aEVe~8qIsRT(aDH>P8Tgx}#3vS$2xEf3fZ4_keFx&}^tk}U*m~|!(CmNv;A4n3; z%;b0>w|n^0f!iHVyGRO&uefm8l>DY_{Kgb#a|kE!6CY?ouTx;M@llE%dz<7#)FjAi z19Rx!PUeo3R%kUilW{OARMM8F2UBahI)!=tW?zWJpDe|_v7{@fT3YD>K%bv%81m1w zO@8MoS043iB{AxAdsh)9z`k-5WHG(DY|uY~-503McHb%Z-2LY@JgzA$5HL{55 zKT3a?zWr;HHyaDZAVSMa*pR?D`l~+$%T@8ZU7)Pm;Rv zW|48TTRo6d$_?7zZ60A$*a^dNJ%px_b;1Sq{Sa~gW`P{UZA2@6jQ z_r5S;1t&CMBDOW^FHxl9)H`XUaqIzUsuP7AkW3}2a@MOQ=o4flXXVCaM+*Sax|XXx z5+1*6MtI_FT_N4-;V4r<22R||GGM2wsJ%8Ms&M{t;3(BDIJPqY%WLixTzK$vIt2Ow zl3VrSE1E$CYlq`br=|eZp4Q|c#3;+i!dOC`NduU23zkZ$n zvj2z-oxSb%WZ<@C{~SV3Q&*S|lPaBKbT#qDm62EINr?5k&+5SHu}<+>Qq-nRuwZ49 zc7!3IDOu0HtpO1PL3DTz-QS!ApH8JP%*KNK_DA2<*4_%jX|{ur+BuUcqM^f?<+I8k z79RD5V&3D?M)mNtuac4oL=Y7Olo?mXCI+-Ujq<`UNKlXAeYqJhg%}V?aM+2ey10TT zlUfZuvF$lIL#Yr$u~GKX+f#&3+!V+K@%{a8Dkfd+Q^eQTXlY-zmJ{kfagfjvX^8Ml zdNywl{9DjI9@uH=peLS4l+F~ZzM6h1QPqr2)qs=%Z`2BkhnLrKK+=OH05aJJvB87+ zasLmW5_Gvq>mj?(e}D*Dk^?13fdi``G?u~jLGwa$`?F}9cD?TKRv8l&=!^R8>TjCN zCy^T4W>Zt5rmK=l^p&sHT?G6l@OXUIDUEz?hOHeP+^Z$t&EuoKNHXl$eMd%!BZzzS zgP@8%iseZ)rW-&Ph^LZ9K)>OHVMM7(*;|MQA3m;et4aIaZBCnG2f=&Bu=M+ZANswC zql1p$!#Ty{f-EVUUT-lT&jN%S-20=aSNL+gs*61S<^w;xvySxJbpc03N@_~AE|)P* zO>_Nm`kES!o_zh!R2aoRaz4y$SCgOLi(hzZ4XDKVU!{w3y`S`=q6{|1GMWqJ=qg}$ zog|Hl%`PMuWegnxO&iDlLR1ANs3`xU)5V(l%C|BL&iou-k0((aebHWSGXp^xN8do=LpLoZDfAZ|gE)xs9AaM?Pt7(NJ=Nid}VR2#FuG z%UPs>OiXLIjHir=2pkY{w(M)%b?nS>@I92xtY^sq$*2>Hb|eAjb1Iu3P~0u|c(?@< z;NAq4k9birzya8Xg#ogGy*woV2C#4{m(~nU-}ms9&val|w5DA|fRK}upZz>kUMcVT zV|G;@yFZj)pZa77an{f@qBvj9$K}Ah=5>uZz^PMSKtXg~59rr2y6$UQ6C_A{qJA}= zO#H553U^L^8Xdm_EJ>UvpuNK;^5Cy`TbMaP%womfC5|F1MznLsGeS$bz60j5+Q6&8 zqyuK-UsEzB?7&LF3L&nbLL|YCTb+$(;O>1cGbdMK%~=1XkTeiVs%)~oe{j|%9CoUUVWzS`{qfjkAF;50frD z7F{%2daV{bpo1hJvQ>Q^asw)hJDbr~#y70xkJgA(G5hTqAPi`+%O>+eQPCu^o;K)d zD`QIU=4^C8VzOrXr)CPYMWCh9_X=^Q@;SrD|=Fo=jeTbXzv3s!X-3S&rAC@kb&# z?`+l&XZzz3-4B#1f^K zm0Q>$Xrd~xlUaE?@#ChY*8w)_uk7~Oz9Srv#nzsLh<904yF8B3VSE=^k0{y>!ovNcRq0DMj7nc2N)?YW9n=X)O!?PhCp zSDz?RvQmpX>^5m7g=S8dH2~L#U?O#^kvZVaFA1ODReK!^!xY1x_ zk54Cj!UMgS%%uA5Wq1~5$MLqL%1p9u;UB+AIrr|vZg%$P!uh@e7B`V ztW6l(t{3a$-ado)dtG>v3i?l?s=bu+#I}S3O+fOljXo24|0&7LR zZyX1^Mv+PD!l!Z4&S>Jc(`??-vpr#;rK3hfM>+GjScF4A0&ODqyU+biK;H3@LDv&6 zdKEq6z-WT;qGuiT(MCZHiynfJ@>8JnKBf+Vo<4@?2>!o%T;#oErwBML`bu;enHxg9spVO$TyFv^N42RewI$)cG# zFm-avN!Q%@As|DnBXr0uaZj>Dm>8D?7)nxf>~u`54>6BO=a)07bd4g^akxX z?VLJ5HpyN=#Qk(gAv5>-CFy{06tG~~2a}_TTzPtM!Xl%3>dCL3h(W|P(#uLqQple3 zT2)8DnC3MjN@aRQTy%;xCq8kYPR0Yt>3?6@!O?bIBC%gy{IB;!@nby8m9XaV@LqlDQOu=}WJ@J6rilPyqqYE&C zhA_mcWHb|J5Fc#YeU^0ARjnZRv3US7tIT>Vkl(Le05Ss^wIJ-d{S+UZ4Jr_g;HULz zNrfIuwhwfoqkktbSD57??VXFn3zj~iqs*fMX=#bjobf!2A)Ca-BB1l_JyZOpBBYhk}oVL~f)nkWla zzr>XTkN;bu%7j3eM4S1&o2SF7tk2Yvy*T9~?w=^N1N6y*g=@kOG-ms|$@QurwZl)a zkZU1v+8}!z_}vG3evg`(^DuwKXH+=gh|bFS!y+TDctQrnu89MC`#c3k~*R=J^}q0@1yYCUq&r)GoBJIW?q+nR8oP}Sa7^2@8B8%cS)0mTHr z7FRMrj$g}&IgkX9V%PB5_c$wV0Nj#f1M9Y1Xz`FT2KVZgoggD7>f}3-9Oby_FdP!x z*s>I1QLVpp8N*)?W!qJu`B7OR6@KR#4SK1O!YzMGwLR^w#HL*X8gxbrt*EGr(zb0#HqL4m%P<2_#is5Ka z7<@*#|M$bUWhtMr%V|!$#UwHN`e^LCzJ>BU>5LF`dU}S^6k5-w6V#}_zS5f}Zu?m!JUZYO1i!lQNbP(TEkW)x>-`d~+>aQ+JZNF!|#L9zT{-CqOMsyFIK|O8{2l z_Tb#TsY)(ZPkO2Ds5%#@I|JesZq9wJ&vxQ{GVQz27p@Zqtb4hH74dRb%0o)Ph@SE_ z0zQT~74{?280CdicKVof3}It+jEzBF)0vqW=U+b&e{wchg(--haQXOL({R>vnU58Y zaT!fhv>>Q{Ri^`xqYrA6XpLpn|2_Xb^UCblL+HdNE3%_`Ss!wCpGl-~jcq z_Kq$a4==wOXG+ve6ly|hQ3GqwYR(FW9ug}(0pwLIOiC#Ltl>*bxn9d zxP~romwNxaD&?uLInIk4sbQ&q&d+m(krBOEL%p?f(4=iGO%YAr*_B#jfi$2vgxP10 zZDIG%qqfH1q|V>ajrOl5VjuX7yuZ@07T*2t78+rvwpk_10%b z<{airI|VDkG%o&CWHQG}+e~c(!oy|5!B(XVxaOG!uR(ZQ?n#QsJPQbBT(fRGyyte= zxAKtuT_SWg3sw-y05qATh6!iPDuslkIPdS==Bwro0a#8L<=MRqGnK(VOR9fpM+_#m zvm(^(LU9)9pM<_&m!LIy459fTUF=hZAB{ihVMAK6#G|l(9x&OBU4N9c(qyFu4(J`RyEta`&KN z*%ghZfim|gnOpFX6RaGS9a(E)KzvC?gaRtC_#XGY^gT&&GzuAm?UtHglzr5np|C^{ zDu$=C%1}m-erpXVTF9DML%&l$t91EE2IAabfv#0G(C4f{^yIn?FM+ER^m3gP62Tm^Xz(Z5t z&vBD%<(Dm{-~+?Qo(IkCYoD6}jK>oNN)E3?G^Rqk^3LufJE}G7|K~@b32z2_4+A0r z12&+c0X-D{Yf%TJMo>Yq%*KXpjo7=lO#wy9=Y5Yy7_|Wb5tT_Nu8mMV!O9wWTh@Mf zI%?frYuPy5tHs904jH$~nO2 zH#uqHON@qM$cPlbo)bD(l0cR+9mtF3d3_aySew&(7y9&~Z+{H+G9v4jf*{j@TO(4E zNZz?^^|8`JQc0URw{OJU55UciJQij)6EEUm!wS!z?;B9*W$3--;l>GH!NWvxIkVTbnJ??SPNHUNT`wU8`fcaQEC1AEmti|Yb*D7=^f4s3e3A+ zYZsdDO8h$99pE|mg8KNppStBw?jMz8^$>;$K^v~X1d2=vMOn;8KF$GS6Y0$&+b0N~R-T3wEbWE9Z1=0MU~&pP?_xaIh7N}M+jLA9ejZ4Lo^m8x+Q zgP;bfuop~#KZc^ahRWl$6UQvdQW4}6WT<0TJf6@#wxXSMklsj%_vdEoIs@Zp>e5r4 zBmW6XN&$omz%==yy}PF6f%QAoDwS|8;=)oBZ#^-=Z(YKJJ{I1_B~KG zgJ(3Y>CmDO(cXvA6j_3N?`JwS+DG8T1 z&zkamZPO2rZ-?g@9%;jVdKje z^;j21^P$u6dZp*r%v^1<^K{BjM3*+j-p#mRtu?O64e@5D1ts<|G4dF~#GOdRK)pS> zm5ZALA+Z(ls%`xNN@LQP>mx=jKPS3kwBncND?j%458I3P+@jegBDT#WBFW!Zy(jeV z@la^9bE)_HL6I60>vxXBe8htPV;9dy45CB}4v^BDVZenYLw$1|Yzi(enONd-jHw{) zKkv(O2gy_$qK9_Mc6x==VoiU9P(1^+|MY~eh59{g3++{qlMtG!_dvq+#X?8%vfjO~ z-b_+?Md4JZ{V7JGeMz_$))AtoeLu1b8`Q!ZljytO8!(XcdDWxOurUS=ziMf#CTH_I zmE88Wp@7gpGOn%Ze6+KmSp-_jb(KN@!%qyEeU^JV`P1IGS-16N8@H zR_n?y8+sFe_mn<-X|s{HE3a_4dod$qf3}9~ap6V7M?*Yy@u~UMAhW2xJjqu7)=zV3 z?@+HN7F_HH)=QEg`vkOMr}q3g8DrClp$ zOUgoD!()|9wtUsg0-JZz915ArzoZ!5sMRmc8B9cJe?DiPzd5+D%}pZgFjMG9u9@-k z(EoGReBB~XRF$Wj7dFUA@q@X$q^9yK3Hj|%k>l-T)YoMVfW8#D_TR; ze=C{(RUgot-JeEYN4*O`ws9%Az&0$p< z`J{eUE9DXPBlesSL@GmH1@f zx6Reb)06DKBi|fp)MNTWnBV3#HBA4^TaZTejoXICpM3fGmC*NBTBswjcAK?Kuax=; z=5U>HT((|wzg<$k{9hS}5(Drrex$%4aWHQ@l#!?j0F%-fnoJ1pQPD|=ZPaTj6UnCh zgS;X(E&sTf^S~18CoQK)dLJIYivN$l-&yjQL$~ymTP^s0UNIhVk8`?zu3HW7YtZ1o zsC`!}sBh<5;>F}#N3$NQa?prda?x{f9hQpoR3`IX#If#X9HyNhW_Ol(FJQM(4z4a~ zu$`Z=F<)8k)e>&Tugi6CJhsVAiD}ErAWdKo=O7s_Of`KgEhgv{Vv+g;m#yjz2MEg; zfgsA*fSd{z?n_q)s7B%Pb$4Jc`eZ{0`rM;&h=iJ!8;ZNY9E^@^OP3!q_-Ps?(4278+Mdbk-%0f$fvdu!?hIPM9T1MEqbku zZ%gSjB_qNHce5<$81Dlc1sE^RGrzcaj9yAO>@yb*^+f%gisLk}@1~=i7i&yn+pMZS zeZf?bCrdKbTeA{OrgJtH+~4`yZ^rX$Z1`SV+0K%Uknu}Hn@k&&6UL0Va^sd5Z$H0b zMJlLEb!`=Xb)&36vS25=7ZzpKw=7fiXIY#FYI3vI!}Q1ny{uA=_E-saT?3CpN-q-z z&2f#HNg8Q$J2Hh^KW(hd4F^#-?>>`pN04u+oFl+m4!6JFnzAjQU=(-z8i?busQqAe zm5IClptQJhD}T^ZuxHs7_a!o-UnXcHG9(L7a14lS57=tI*d2teH5loCY^kd5iyWMU zKJxKIY`JRHEN1?(PB&U8F(c^@O8;K&aQJsl-x#1<5A}`-8_kv3gQCz|G~`<9c)16& z>#q4JTdIz0kxL$yooN{{13UYmwiDLg^}eB-K{cvOg8rIdM%2U)JYr&kVl=dOdfr>f zGsoUHZ3Li*lwDFh%Y_NIAMZ^{|Aj5FGRBOj@;CLO=5usI_O3Q9e9g2cZj-r85@Ki! z#~7}0JVxPeJBh((+IZzBYbVSX0h@#rvh6rfg*|cCU?<0Gxpfp99{Ae93G+`mQC~;3 z>RT{Px_HvCc<8n(3>&`qb15j?EWL)8)UTAVml>5P%e0Sq+?8r!PHCZ%7vA&9>J)LR z^p(X8zR!NbgP$SAr^%j$Opflg`FAM%Uny;uF)fwnAFuM zWv{#NNFL&U#5n5Sl1g%s6KJbORKVEz_BSnm$|Ww(dSKeH_jym9PHzoHX!ZJL+tEVC>F?a5 zF=@ve;r)#JmDg6k$LxOi#}k~43JVmk`rb(Iy(SxvYIA1kDbg@>5pn+|e0WJSc{UKO zb93Wpz}8*)LLv^g%9rcu7F|y<({6o*Uov?Lr>WXY(;fZ0{8xh)8jBY3{R<>qIH=elJnqlA-2{ zDCz_32PGa(iccG#mA>HuHA+jkJWg0Diam^F8!c=SIA6qp-iL~hNgC0dzdSEqkS)+xoX&W%h7NZeNA3pzYGWy(CqD?)gnXQt_OU_Q(n}qe zrYJFQeP}i-8|aTp!0zx9SKRfQV14C)fZ}tXN785)W0X=X`G`kS+q0}$F#?+J?+%L3 ztzE;vSU>56z1y3kB?h!RB%13E8T6XNR3c$oEnJ|0#ruGT?G$nZ8i1M$PYZ<%oi+ zWINt;wIC-Mhp@NPc8an47oF7dcc&+sdHQr&T24Fd!dND=#m-?QQ<_RzPKT=G6Z9ja zqe&Y&#m`S2>&Z9gf+}u73RY0sx$$DvlBdj;7+A*qupQlu(aCAe3qz|9YNWV?aqVCE z8O$z=51327zyR`H^zdiDRE?yu;cB}sFFwHbj7*Pea(sR<9(@u(r>e{f<^E#D(?Wg%NS=*lfu4^&IyV+LM zn!49Sm2LBW^{|}#r;lg_fcIcR3sDUiZ7~04C=>ZjC1e}qQ>|^AReR=SnpCvzoPH9h z-gHxktxv1t9FHv0DKY~b<<8ECsD2QXN268z_}=efUsy{&X^^dW)g7qi&29OR8q&%` zqJ`#B*pA{GxG*r}DTQbeOL}Ib6T?j5j$(6leQs6RXyv)eOZZ4Htw70`^rJR3JdjGe z9Lm|HT?~ZMPWpp>WRC|7f6}Ox7(PlSr>YhQxWJZ@1&DS4#vPA<+5ka3=pK?u zD{H9&bywmhC*MV|l;;q;-J+yZ7$`S!kbHx_ZOI~yKYj!f4Y)7qAj{EQ7R+%jHT)I! z7V|pVdL~<-i^gI+l=v&`1eLP{F!Mj@c*KFg!Wc@}b9b-9$MF0r*C3Zpfv34{*T9%o z+BvvD01h%F*~`ns&_=Wf^zvw5ME#1~!{lUd`HfMt%kC%-GhI|)@C?zD?l$fL8$^Be z$~{MS<}3apo{76XVCmfX-P-h&y!RLwlsbz?OGeAbrsrS!u?`stM=R}w{)18dg%a!i zPQnfliVASMwpD=uK^ua^c$k{h_M10-JnFA-N!Uu&gRx)H(FTK}E0l=OX;pq`Jb@$44~k-wc=7Ez}8XXfr9o&dQ}BF^Cyj-4`} zU7UT23)s~Pfq-rLD?%j9xKOU1i^i>PdbZYx1qKJIu@Es~z(1-19hB3s?cH!OjwXlm z?RdZ_`nZHRT)1}K^FjMC6nJnY-o2!#tdCV!#pZT{JPK%9pZ3Glak4_~_Pjk_6;Mb` zOAtCoat>g2Bo{5_FvHk+ZHPmpEw(gYg!Zois?GWUZ{o^`CvVyTO6ojIcfvbd{tVbh z{wU1wJ#~ybejIrDYlgUo$$<@V(07joLd3QI&@>C6bEt0sQt@or=-1$$mFDbo#B}}5 zH11X$tZbgbtf5qafmgjd6jwE8lAaa7{;Y+*cp&C0PbPMON(J&!teU3`V8gIL)DTSd z4i!F1WA?mnxte@w%G%wf*bcaQajy(^r|I*Q-L=OCw|tkDJ>7iPd#5aikKpoNgE#A; z(#52K&|I{HJ-ub|!9~y@+MeiQC-?mo984cO?Eh8K==uIW5nP~?Q6d)N_k3D>u($1m zf|C^_kj5<=c(K9DO7P+m^+64}#i6P{#q5wD^cCH8Qrg;?=eN3~69r(w;rMYMYL}b5%eOma8kCjEZT@>Zs z6*GIveoNLkjE}lExHEWV-23S+qE!#)s)>PWCW4jqaZB)BhFgh*C|g$h*cIP}<&(3( z4NAG@T0o=u4nn#({4{e0fbgEo^iI?C-m zKcv@US{;swU%9)RtpvOA#QS5j?y!zO=w`VMCq|S-%LW|CIb(dmkgxXz?X^R#kC@FDUO9<#gM5yxTS;Nhkr03 zaln#lnoP8ey6HWDJ`~ZKqQlo}x4$R8(q95jfI*~)t}hdc^0RBKvz*k;+Q| zzlX2I4pIrr;<)zwIMTOYg@hes3K_Zn-C?Jyzr{;h_ zZP)n`1#^YHopgyeLQXPx%uzX~As^*!fU0dQjE>#9T~93B_jX|rp}1nWnzDMRbiGf8 zJElMFQGrA^lI8kC_+DEyfOMtb=ceKuHKPK1`YE&ftx~(di`(J=S$a@wjjSM2N#whu z;gi;|qCO660{f|TK4)GR)ZKv)WO^Gwm6O@OWdyyFfrA7AP^xDc zXFzpMC%ATFUi+Zxv3yBq*9RFe>7d)H8{zN!Z1Ga^kl5d}tEp|@o?Ru{IKuQdZ>qwX z--(E!(@MFEb-Z}d9aX(0z7WoQ$E;}xhWQY&6*b2++Mqk~PAtHawxdhD@EMKZAcn`1 z5pfW2s&+c(|DpvABef*4^XV0}c`e*%nI&&8iAa=(V-E}c!cF@gojivluBgA<@@Lpl z5Sb+TB{}rEQUz5#Fu)xtZZAWcPd}()V~z0vx$9P{t9DCBfR^*9VE7r8IYwzXlmf2G z+fAT>K0!~}9H_LSjiLc?z`msc3$TrH-q+5Xq^lT!xxJ>P%+?Q{MlN67*C-hSmpIom zBd~ZPkaKEme8T7?n!WbG<1WTBmeIqWw%CMptK?wp<65P}5U9+eB?MQ9GF!c9snb!& z^b1P}whltL;q4-!01l=wMAnoE=r}~MlY!-j+ufzC@g6yg{8*dN6cgkH8}I^J`VGE9 zyKIXfNM%}}_u^Qu2jZ>bNP!U+=bHDJdAy<@0@VGiNpNtUst)MHk@FgS4pe^z=3P%c zGCx(`_1UOv^{Y+L_Y8octvsN<6*k~9|Kb9WpToAS@82ECdS@ch={Ddu*vA*Fw~$&M z019oZbE~1E>p0&>e4Ftn@m-_Wy9~JXv#lWFI5;Sm8Oh&08w~26*?(>!nyKb~|DkGj zh{M$b{(gWRdhON}}$8@`F0Bcah)-kLWVY zZuJmcpzR3b+cgI-ri-pVN`1uC^XqQMA|_+IUH28-xTt&a-l}rnv&M_P))I|X530>q z7}|#ih8rSOUBGo1m`7AO%V)q{S!HzuJUId?$5Jt=e1WvXlUr@r}e zcZmY>eAXqN={A1;F>Q`81hLVgLv!N?AnBDLR5EH);x=SHC_d zVollhOcLkj@OqdkK7MAWfC-PwxGz>aWzAp%AIkaQc^pCeW<{a3w^FA+9W#B~RKCmRhQZM& zW+nx+irS`7Tng`BTYm3;v{VC9Hx(;&XZI&(9`MDlsZ=!d!+ zPK3s0{9dA8@2|TS7VY)t{$5!@`pcjg)5p6B6794jk|Js+vb);hCH0cIh<4}^q(Sg+ zn}vxK8WV!5bsCzjfmdRc9Nr|{Mh2mvyYKUnYfJnMMMK!uH1p-(9fFac-`{|oc1hl; za@N38UIt%)vV}IG0PtpDc0C1cFzEU!?Q11pz>2&~2-n3TkX=cU{?RPJi}%z##33Wi z?dnyM66+mte?$_TLs%kwaipxOt{okQrt!G~VA|?0%kG z%CuHPfykt{|0V5p91brjAP`Q!pmtoYMwP<4ZR5BrP5F!xEi!~n+*6!L37@>@bRc7{ zI}X3|)j?F=nf1j*zX?}a8zGXSZujrm1T^2Wqx(NmzRB z#13GF1=T5&d_q~k^EJJbw>#CykGPXRr#&&Htju-+AHO>%Jp%cC+1vs~wo(t{wMJ5N zaGN6#&k(=WP76}yK5awOC$97fh1&qY?o5IaR##8`t{b(Z?82eFN?v{ImK2CmZCcWr zYj~Bg0aqu6&?k0P6@Zh=UDPs`!SuuWguY|q-{5pAV;bS0<#m#kl*>R&WHyYG#V+@A z-4(t)U(;y0iynsuPg6^|W?}XEv*uGnt{r!Q>QxT+%%GqIzyJU)$ibIQ|KiCB)Pscc zE$&|{3b|a?uUdcu2JRGq7wx#B(GYzDIFc{R&oHo7ce;u`;1Lo$M+7JMr{i@%H$Z|( zi0B}I92WA{R@_nMjv><+QY(b)*@8%dhEa)xq+yrHoJSUPTZ$5R1fX;ecS*~bzRE49 z_%1086GsXAqbc_@g323|t|jdg4*Su?@RZw8fY$h9N~rc{7mo{18cyz-1)*K9>bj5P zhMEQQe3X=%t3!suxovTBm(7BJ;_I!lmU<_UM|g3waUAOFB90vW6s>soq`@uxvFU_| z*)l5P{U;B0xM}i|FJLyp{_^H@7s^+1IJ9!c=DF}VE-o0`T?Rr0dl*s878n?YznSA+ zgRh_Vp)5iVfb4buVMaaqD`&!7pyUJ{SqxHR>tPuwh*%TN;X~8zKpBE z)kXyf6*}9J;jZ_>-k|K!ym$7+xzQ9@8><%Ieg*>B#gRSk_d=U>;^i- zBiKYk0NzPjK7X}zy4qgqg192$PNh*N*M~o3<47j;8dSkJB9yd{b&3y`hRwpp1}|4c zP)wX2no@x0MJPW)oXdHlH(y)*n9CUD(|RRAAG`XtLtJrBb$Gm2DEB8#j0F-^CMB_S z4qm_y3kJ8wCWRlLAQ=zR5b6?S7au}UC}ASMptGZn(c6M@mPDdPeFyo?Rrg=DoE0J$ zyyz$U@%MakAC(Ir(!SscSTuA&168v%Q>d#hDdTBV3Iz;G|(F^RCX2c@!&)!dgHD1^ng)K5w9BL>13_^hT z;ZDBxt4F4NpfT)d<5yg@VvAzJU$=w-pC1DJAJ_xPI{qkKQWx=@f9Zofr_U&e+dCv` zNdNr+dla&6L5l<$(yi^51kl5xe^rnKNy7(&kLak}{(_vbjCnMba?Y2ll@8Y0%DZBF z_ctKK5BI*!O+@{CoHikG-;M@`nasKKZ&(Bf79Jzx;HSg3bZxQwuHbL^+zwQCQv!d( z+2{7E-J2!myU(p1D=&os^fPmDYh5OqRU) z>uaAOgWK{CJhIGn9*8NE$$X-=<_7B4^n@iNWYQA#ni@WUM5&0cL+ku`L4D0ZQZyNH zYWrAX-xYg(=QFa*$^GxhW#?1HF-oqa0hM@#N8?O?E{!nv1l8uuaRcZ5A(J@?KPdkv zn|a9n3?rhM$zxaP>`^Kr6hwQONt=iNWP^>OjQGiZaX&AJcg3 z9)C}u^XWzO2gA{(jYe-hH+p~lw$qkR(x0-j^H%*P$Q_Fw7yS%ba2)w7zOfeJTy#@- zoqm|aB&Q&6?<aOG)mr7haVg3dLylNQ**m zc1$IkE3O~hY_=BEE>Gd>T^)^iEq~75uVwD1!hRNzX;sHm=FMi2kAfg$mQJNchw8jcAd)^#`>Nb+HDO2>jSg3JG_Mj z6=SMc0XgWY+TBLQ4TX)J8~0jwNJwa5t*UQ);cpBG!zc8aFk!ts>2U8cuFMr-X>0p1 z=h{Z9u$legWX;lOPWe%bXsimzc5ODxuWG$J;|5IM|sk%zD;9-j+nsxd$H?EYiPpp>72tT z14Kt(0V(tL9HU4Fyh(%_^7+ZuW=$vWdjD?aYdAL<@H+}^D(2hq^L~9)ep26-Z*-xx zt1hM&c)ogU;}$~(-KI84#`@a0!c{%Y*!(Iyw85SO?-dk2AIfd$GlmyiUkezHhhpbP z4P)o?LZ*=&ynXeXSJ>rIv~DJB)9jJO=`r7Wn%G}t%xJn4OJsX5xo%m9i#8`YV+T;8 zzWeN=#2|mj#o=9Npju8@>vd!~Z5-!%^qVo@R*{H&MUV2gZ+Sx6XtL%b*tA=S|0X8w zxfEU-kEs9ke764IR03iJV*x{ot7RGoqq-@+lt~uV?3;rPi*i%(c1Os(RE7Jy`C`!` zBH}pC5v{q%V*65u8qL^W|8&)J9G$~?Hn)c39AqEMc11bljT&7s6~8KjA|&;)=k{*n zwz>7AO}~jVuuI4{1^HN}*Icb!hixUc$j-Yj{kL7n48q#4^PfACHN6r!+VjwH*S(k) zV3`eYNM2;SJCTMD!*<9^jC5Olnx+NLl9JQb)pgORu$co`xYqbaju*!#uUG5|T8Qz; zm(j+a2f5!KH@F(iK(iNamFzoUBA)mD$m%VFU^ig%Uj07}D`E{*N?!lnk#v;KZe9H$ z-EX@Ar+wO4>lfbt^avz?&neSAbEE=7Hz~AG>&j@#hS3|5J%E1e4y5Da09m zLj3LIPP1D8OIAli6hrf$%EX~n*=G7a+R;hOZ}M*0z`2f`7qPUgAav+|ZwMe@=QXZ> zcPVk@bvzN5H5K5r&#g}hYgH6^@;`mvVc5RvbMjNmR;~MIB_SH3y^aE2ro?~R9zvtC z%ryQbva!jv-NP+b7Da)^uRIB1d^krV^mY$`f8Ioj9DNU%5BE)+V_`688jQr~u+0C( z_}`~4S_RCaduytCOD1sS|7~1v*XkjgY(H1y|L+GT@&WP1W@HU7Pu!|dtHSHH&80J` zVLLJ*EC0z8CVo$$e4>ouIg4Oby~}s$H*<`rCY_G`zu5nK$4hTWBf)7=%el)WE15Mp z7ozbE-cmEzpdlw=(y(IZ`Q)V<1W*`IRE$fZwO{I=CncOs2(4?fbVLV zwn+jDiiyKBoL5Ny)9p3w{H0%KNZ$ubvt?gnwke<|7gYVA8dnUU*cxgmzmX9JhMnPc-T>OFhcyhc9$!A2Ck z!t#poy8KckT^sSFx9Rw&xv}6#>|B0EG!>RWzFqf4&VY`Altllh!#16VHUcH%zLY5g zd7BBA%ul$>JMF5^F;u-=_OOOssjH#m*=ITyV-{X7S2ytS$nZ{sz=SWDULf3x^)0Q6 zj+d$5$AxErG$+1S@Pnqw% zrJw84>%>VxRe2+S)q5L<;y>}yG4BT)rVmVeT4qR{knN-{jy2%5hjiA*3f*47lrJ+y z%<3fs|4bhBld?;&T{(xJlFhl!=?YQ5)NC(Jn1A<&fldTK^O+<%|+om zH;fSxFY$5eNugST_g1UhUg5Dz__gCWO$@pgwXNC}r%u8%poa|l{ zV70>rBY3yB6SRqz-F3EAZs%?B#KH|lQgX^xUuaf*qxit1G)PvHoCDM3$I>iuE>4G3 z8k}CS#av$6O;5M37nl9Fkv7GZh#RI`I=@fW?C4O7 zb|3L%@3|3&-tBnz(O;Yne}A5Jmdq@Ge?CUxN0s0b$yyCPeK%L6rW~GcHTcSY-_|k_ z7>aEZQFph^5@gV(4!4OR&T?SM)puq8ThoMi3YK-pVxL@$a^pcw?kTDX8CNC-^4<2; zmA^0Z314^N&AMBK<=}6i3t}h)>`cgnyD+eCZws}1-@JNXPh3i7xI2wpdOzwF*zQ^v zK;#%MiD{|ua%v?fJcu(;eX@ah7=pv=YO#kI^n&2_)d4aODF?M)d^_L$#Q&6@1| z3vQq`M+vm@?ZKR7;;me89*d;2ZYW)+4HUdOTNYL9d$)V$RxvS5Ess(@vB+_EK(HQB z4kmx1pO|&ylE44V%hF~!hiQm-R#8eE!5fcK>F$^<(jq44*=_2x={o)PIv=eVc)@Cd zE)3VDLIN#n&bvHw&$kgJr6tE*|Gwp#-oNnsZhp6lEOxbhIxt;*wl%c@5O5U$zI4kTC*x!tj%c75H7}O=0o{^wN5=7 z_Dam?hkKD+p5HXCu3IhiKD+MhXv<+D67YEctvNNGEIB}8DAE@5YM=RJhxCG~^7pq9 zw!7&-fJDj=kS-d}!KryBe;?TVKzsHK*M@4lK`WQIjE`@Z83FM#I#}VokA4*+0yVWu zU-js^80;={Wc@0HXPIfyMH>bQuWsER5lRur$0;=IQi|faxO%Owq~P(4x~8sIf5fAg z4DTV@dkj;w-s$}I3h={2mI7mDjt35FMw0~>F61g+5Sw8P<_FGy*GeSm?$j=iIE+*O zG`%PAyTFd{XKg%WWqOZRN8n6UbS~2TQ=Bp%r>cAEa|&(x+LIHqttGPgOYhfjkFt0l z2hu@LQ}27-849o+DDha~|F^pykp}cnqh0Ov-Y^3ragdacpC>v*mE{E$N^akWlDj!T z)xjLT{A?Rf?&GPv{z{w{TAOb%*y0|j%nE^4vOe^=wVi9sm#Ud8uCo-)1`WmxE&RGfH*_4 z>D`5kjRI7o(C7Ubx$Ue%x{A+?uG+~vr}v&1@K)>;)LXy$Tm`KVdg*e1{61~OKc7g% zE}-e0uT1A}ddx%8?|2XVbj@A5>z=e{4MXxY>CS)s3f;;U7-_oLIts=yh(q8GWXOJh zcMx9kgH>!IYb}}Z+G5W5ZqZutv4M6jLr{8D^)h>n+PQq90k#?UYS~*<)@=h zpEtibnn105v&e_>Po={|Chp06T5F!mzdYeH_S13=@mFFxC~7m7k(GxbB(p*O?U6G@ zu!heIX5HMIV;+&$jklc2ZhIf0Gmc8Zk~$n@c>XhI4Dt5?Q_Ibv_LO2~W{ke$ys0w< ztN&7|FZD^_M7c5@qW#291MZzVgHGefY$q3Pkb4`YA%*W&)@p9$NJ+u z(@q~P?^vcvsA8(AX)X1Ka4G@5{gcT;;Bj$P|6we6@O*KF23@p0+~&?c3g~^<@N`hi z^gvS3>t;nxan~kbTewcSDWu<*FQ6ZXyXLQsYb9}J@iR7`(^xd*h zirrI4FAIL)C&`cHI05Q{)gESccXbz4RjK}dZP3i|TOw;<%tCJ>@%LEi>&$rGxo~le zzSLeZY+QHZe`o#>TCV&_Q_HCMFv6wL!13zHRai^RCvZiC*^Z@V>(~7)oQckUp2Y|k zUdO2S#MrM*#Oaiq?~axPOvJ&ktX3hXuvBGkxp@;^Q6A;VBg0CdyDz)*-&|*J&;Q~H zFFt4X^|4QCYHYSyADN=|IN5vn33>c;*3XcC*53r`W8dG;`-Kx;&%L#y$o=qP^~9^z zxyNgc-?Z*qkg?KasrFZLeFxU028Iiv`D%Qa>>8Fmf=bbU3t5gP@OMN#-G2{a!Ik&( z8Dhy)`b|1lrDaKZ+4mTvnbP5IR z>C<-%i$$luROS)a$QLR)J@0F38#8Fv$YahU=h2`1L^r;$O8a(ismU`VS-e#Eph~Vp z?v?Qnl3av5*2MeCwgqTqT{bme*qv8%hF03pVxVcHoPo&cMY6`bq1si1runW^rM18a z)E&Z4c{Gu4-#{8x9IJNO@$^l||V+ucC|T&kdvv zUHm^8qr4JrhgR&S4vAyE*4x-<9WSV9|9cHMm6>SR?e6`Z zvUd`OYtUv??DlB~sb3HN8w@5=0Q7L?;s03yEINUMHF-c0Dmm%Fw&-@aWiHzeyels| zefz;)O{b#$DPN^;`nO+~348X#XKCt^yI-WZfE5G$4h2LC*ng|2>O#t=cua zFa69P!&%=CS62Px*3Z89$#vGRD_3U6IWwkn02`aha+0hN5A_NDc(_h})y3O4jOO*% z|J-QyuWYfHZuF8foilZhKdwG zl7035k+{xlbx~=ve%-&g|ElHY&sVhlw^|-!Oy>sLRb3}33$;sZN6Di}_x@hoJ?rNi zpQmR|Y&`bs8+ZO`mc7FBXRP4=_n7%zJ6V_;vnz1<_c>v|E|m1nFm*ECXXmW9{j^qD&Sk^3zrR}9{(JOT z>y-YbYogJjZ@adATfe^je50TOw_`%gN>ONe+N2N=8ME?je$Mu$6VIL%*`K+(&3xNS z_45BN&-vyAUt6Lh`b;$IX7K4R*Xw_-uCsgn+-mdpp0>NTK1Y{#ZT%xZE&A!~_3^;L ztvajE%dKWtU(*T>j$CZM?_=Avy26!GucX%`9c4Wlwwv`f_m@Y1^|StZzYYzJyBfn< z&#ZGm_67rUuq`x6vBV#^6I=UddVlse^YrN6>G4|Ezq8lI1FsSe>JHbrwxo6Pn#aJ= zl`qXb@)_@|t4%+83zvSp;rjRG?)pvdUevt|j}Hn7nG%}!T7KKBtLpJr)%X8)G2Umk zIMv~_xbF2k@6TSn^Zxu)>$VN6Sk(^51B>H?gjLYKaN`4R;FTvA*E~<(YBJ}F^+~VI zQ)kRuQP|&`Z8`6fz3a2>i&h-6+V=1%?=|L>{y)!^bq}t#Ok}!uhI8hvFQ=AG%Rbxw zX7}c`J{&CT*G4eyJ8Un?ND0_eOaZpjrim%PcUp?GE<^YuBD(_W%FzrCE7FwvB1ny_++B%$Rm? z$77DAQ3g#v8cjDi9$5x0Km|5zKk$Oxef73YVYlP%Ut6_!o>70?i7H3^|9?-f?b|FY z#rW^0fK+)Dw4uN#%{9Yxt@Co;tDn~EZ8s9)*Mft*!=qHI)4p6W&bL_{_`Uzt6E8njZ-rHx zHh#Nxx~=)D7HsI)!G%Ld_wUvu;pD?c?yIi94h@W(cxKl1DU6!A*N*&3X7UYdzS;Rb z+rwv@{i^O6R=`*_3_j}sE!-Z2u`F7=_^S2p?`vb6ZYpqo5D1^JAQRdz0+wwD?12T= zzrrPA@Ny0~^d)eBjdNBSwABc#?Efe`Fuay`&w@6nfVqZa60n>dK?ddh4{_FoRSv+m z(~1=uSHrHH`mKVDpM>e#Jgeg4Jpze~(pB#k)w%O7?OtH`wJurkn!~)Cna&0eYs~lLv-EaVkuuXOl eM?_4G_z!!%ZTs9+d&{mc0D-5gpUXO@geCw}1plo7 literal 101018 zcmY&=WmKDOv?YY#PzY|t-HR118njq(cZUK6io1KEIK_**7I&w3ad&t3Nxpk$&6;0X zgf~2L_L05M3E@f#Qs^i|C@?TE=rYn0UtnMWl`t^CU?c>{JN_ZucQ7y_FftONs&25y z-iRqA1CC$y#`yNo)_*u5fMkATONaA-WzkTQ$_(M@wBC8tn2j`S(~YR-W+pPHaWTYG zg2MsfznsPpNm%!OP3!x-+&N-+hN2Rif2UKufOB;%C~SE=TUbgvEa_Zgro&yx9Y<~vYf{x?#7U%955iCIy34zBY>4B zXXodat!!-wm2C5}TK$>KB?lunv$#b5f&}V)9 zH1)(p$Ub2q=G;;<5@`~U>?8z!KjRa1K_4aAJ4AEyiax-!EGqten`G{Bniu=UQo8D3 zcO2Us_%{Z!l6eQ5+bgNmy1D3w41TA?soK3ShM07}+`L3JVdbeNre|ch6Z5YJ$IufJ zUIUwN0YDQtQ(*==RhUdS%!p$f&N8nk|ixY zT{Qh;1v?X(puJ@0HRMjLA9eg-&v5|a0h%v68f!Gd%0cAZOyern=Z#T*Uoq*~n&tg{ zK3!jjl1)XAmC-}sL=U}@VUuAY4#0Pc^Olu?qT>8$j;_7KJ_B%z$d+UQIJ>=BOnuVh zHC2j%H>`9l^;U8{oD%?meeK&`@&ld205mG`ryfv0(o@l80RVx5*{~QxNc)KX6=UB^ za2+ei=Ig_!zF^b`*TXG11el+TQ~6Fw(guuy&Y_+A z^vYmmSa?Ll2^HV_?9*$DU_w|)k@vv)tN#HCI}O18?G&j#Tvj{G>1LC6j^V>*V3Z%O ze{;!~!_#B(-X;WudmW4`Cdj3vkU(r8?*n&!y%O3PJo0CMDiHwM9Njv#4*+^+s07MGSL*IZ)m z9hwRKt%N+l4iDhQ&Hp#IB2VUdt9*HPm!1nxW(?VAEfvHL3uiEy9VJM=gnR=l)*9sq z-LHnmg+*qn8=WLY&XAnzcvQsY;e{@`$`c62H@H0v6lb$QN~lnLiX}a93jw%YqOZqV z2Iv{MBfF{cc0R~f)1DF54@v|5@_qgE zeQxwn&|@)Ww6zY=y4yGa2%?70&OyUqih!ap_Q#1W9W)N6OazEEih>~!wWb((Nh9+A zxXLfm*7m}o>(`?z@6B|+eIk-w(wf5j(#A11Gd?#L!_batzcDzY z8(C*J+ucdG-4)c6>~sW{3`ba2tx<02dV713ocUEk6UsCtB4Jy-80Tq=0gH^na=b;)+5Kyo z(m%c)OU3AL0HXX@An;tw?K&fbMa1NF-eyirn!yx||HMzHt~fHBXJ%74et&y=o0B$- zg*dTM5Y;IGOe3}~1_zmXZJNZY{4V<4*A200lwU0*3?DLo^U5d&(X*E421<=7Yv1ak zyqkH$sK(dwE>mGE<#skatd#)&shFy6vmepyzyGZ|Y)})?aQrBKd2x|{I`;i>?d{H& z!TuF%@pXP0yV+0lYBy3i*F}dchor>7thsavSz0KI}qW~6<+1<0pUlh%PWlRIo}KK5>~eOIgvk0b8J)Mih@T~ zca2K>j)&`COPCpccV-(7(NM_jdiIX423X6wbyi~H^>*Y zl!_Bsghjt6A;=bf9TrypVqIYCbe=JHFq8EvkdG=(8e%RkpU9+{ z+4}@ww{k=6zBECZA<&*R_I^o<%;UDUrgAKX=W%S!N5MR|2b<*QkI$A)&g1s`<8KFu zT&^467Eb&7rg1O{B0WVrvoZ6SCQ14qOTL`pe-!>yRrkXk z&naId;!yL#xW~9zxc$GMYT74QT?j&I%g#V3(nQbQEng`7f@ldU^0wsHKm1mHT;A*8|C*h_ zcS-tkEnrZH{j`?D68j8mw|oKnIHUL$cpBEY@F1rVd*^wvEjo-PV>p}pt^+T`Sk6Sq zdn>eZm{*PK7#m@$<7ISjpYH9J`|%FnHI}#o46ktzzV}HgR`_aQ>&5%1@)HYMKiqXV zn{UP^7Ocv3mX}aAA@SJfuB$q&{f)4H$l}*?9nU#jN^i%+Yxr(KgCjOfBl`yJmokq# z80|7^&lg8iEzkYFZz4n$29J-fRu9cAH7jA{8@^z-Ul<)vHXCZIyj^fhle5g|FXKKp zSvxn4+0t&dZW5NpE` zh0Py9N-Sru{qAtMSveXSHTOxo#i|!&cQnbvA-E3wkaMya;tF0^s>kp>VJSP7Xy>iV z)QclWuQ`=0g14xTBXjnNSsTbMYVSIpXrrUhrfajF4jVTjKK-1owVeA&R@hwfjj{2Y zzDNEz>5BMB(4;lD?&LdU8Ab0yMFwiFU+WnSxNMH6dA`s1TOmwi6lKOew-^)UOcwcf z^9^nKYaeGdRCJt#gs!fsaPWA{pO0Kv%RWMmY>5ba+iGpWqKO?%<*@ga?>*o5I1UMT zmQs9$EykttOcyy~eDbKy+bi!3&4aq$@U%-TtKZ&d#J=ARYLf~)4i8Qf29!BmtLGIu zoDbQB-spU(+6wU%-^cearHu-h5xae-NFFE2kC!n*ojvh!`;uHnl&2|X%VG?+>k}qd z@*%+@JgGp5QVF8{Z86?dS*mMfDRx*gVmc34?YaNOeEImR2}re~4k_x-B2)X^gY-z2 zo`|go2I%q^^mAM8!Za#am~u{*X20miwD08#oNF0LcCJIK;B)xh1BO5mDXHY&eG%b` zzvu6%&G=`O*nYm0aS~PXc<3;D*s>rkNp zNPhQ~@(iYE>KYv!{luq)9rHNRR5W>$@^ZB87CN3WcY{gw@splNlMM*yZw*UY)z5K_ zwZ*!BtsXjw+YUBm4FYK%gw9qpcvXqOHGAvnzBj% z{xV94CH9da3Hqi^yR1|S({J+Pvh*zO&-o<S)P}qPfaK9dT?u;N8!GqzO*8}msS|bjTiX1k81D)IVVlN z+3+3HtbLvy+3WqiV`~>tr}Xt5*;SbWZ}C(punQga{f6kTzx4jS%t(vW)P*B=VR0F< z6L=wiW*or8Rm^Db-u+m6BG9Q6r^J^y3-`nAo<3#@n96j2G;&nbvKWBYnh-J(2s6;i z7)W(zYzSOk@eHs&N(ZCubzhiLP|4#HJ_~%hR~{ZUpbUG)S1F=*jHE8H6wagtGUB3ky0**G)vVmC_$BVb)zj*$ zfmm3#VAlmPmEZ+Ey4RO|^ah|#)3@6cLxZBW1?=TmIz(8#$j`EI$ULmXsBX z;g+4$y`nTAvkM5`P3qwgan$c=A2}JmKtiy{5>nNk+R{VOx3NkeT68Ufx`qH?;oI{Q8@47K>ElE#4(PqOD{yBv^P@p7A8HaZoM*M&mDXBl zkklZLMeHeVxEWP^Kco7D{Q^}fkC3#J?_0?4wSW&XShe$Gp&Jndvvk52;G8wh#$GAm ztK{pIO5eEGhjC$6tB!R(cwuH?Z&4mnD~A2euHB@ z`-JQ3>6LN%k_EHl(Tkmd)}Y8SY@jTv)`j}Ruy6;tn|=oq3TmrFhv!Y- zENXIPEzi%&g@An64Ok?3%?`ULv(ARDT-+m?@q-aIuk_cWwO92_vH9*pL{ds6M?S_6 z#w(=r4-`XL5tj`}DphlCeVV?I!(j5-<@Y4j@3K6wu7^^l;`l4<&Q8JPETa0MsM%~Bv^b9N(-`0P;}o#w&tNSbMVb%o{4r1Dv(d4d z9aHg1>lOE(q&e;qJFKDKc*8}yx-Mq&qN)K^Ysf1OF19eIiT_P5p}wbQGtk|)c0Nh5 zZUj|OT!S^M``3p*B3(B`q#$k+-=ubZnIm`}pejK+^V;jtxRQk@>dTShYJ7DodVON< zH;u4P=C9vW^t@S{yfY+2n6LK?Jdy$em+{lfn0{z#L1_Tc|LLuUaf-#Y8)=`|f33qc ziYQ$N+%&0XYTkoe%A4wLcxiDY+mkIXtsaBLz)KxCwyhZqbpR8B*;o-ND2IVq`)yrd8PVZe0?!mjNkJ#q%7M^ak z&B;cxQxnPAwJ!3|#_W2ZDxF?vMt6g|Ut1;Xk}WNQXLT-8`@HRnsFV2%4juig(5bPf&UG{JY|?7UnO z!uJ<8|7-}cQcp&}jWF1%&4})#;8>*W32nzNwaXleM^4Zl5dnKYYS}@3}lKSn`SsIS?r)i)GAwb57ovMMJWUO#=>X-OU7h$qYXnfp2Ffei zq0AJzNyezo&Q9_+s02bn!YJ!y&a^DxI)Frg1`FEP_PL`C4u+o$&6+o=8oF?X#wa#O zMl4BAG!{}5T`ylF#R^99pm!Pzc)I_rM)*U(4CovsA`dP$l-)0g6}qB-cv56ODc!QF zK7`&3C)B$a0;-#bDI89_J43 z+FnTzoBV@T_}r5_?c9+f@_JM+rz?30^o$h2Qj#|6it16I2><}@1+Rp@U3?^Yv@mf5 z-%^xj7HZ7TQIMOZ)%U7ZSeX(Qfyi^L!9gJ^U$h&vjMm*bArcitmlQK|wO3Y^Ab9E8 z(m81kA^rf>PP1vDN=P{CUAYW!)$Iu8=K8jSY_CKHSBpVXgcC_MIa$f{7SrxtgjGw@ zgF6qO3Ch-X!G0q>X|5;od{-(K6<@keu#`n@7#{A`y{fl!@ot=L|UFULm;a|QXf>Lkfj)Uu_W~`Nkdp{+XK=RTBuk` zB<$Xp!4twNQcl<(@hG1rgG%g(6ev^ zdbK_5_5M8SGvNf7i*=S-IE$zjnHvG4qDAbpg4^%%G=@e7vc6p=6L{7LLL^BrP!b|Z ztPm&D3sV1Rzketknj5eXi|xtlQC-oO*|9TIT~e0+wEAf(Y&2#lzY(*dksFrz6)Gu5 zK#BF{k8h#c66?uq9pXE>gqu&%jMmdz?+Vb$%Yt(V2&(6J3X*U>m zDqtJJ#0DG{8++7X_z>)2F z&xWC$+x?ImW{2`~1yuAbb^u@pxBmL4K`XIWa-}xJX_D@gp z&R*7WFOiW7ez`}Y2`FX~P-O!FR8&2xZZ8~QDd05zr#l>XPo^z~6XN6i&D|E&~=9SuL3_8suwq;Yi#Clxy*K+nM2 z4kH>VR$lV!>)+@HX2=stW(V&nnHJFuy3xJEvF7>E#f+Xyn7D zlXdv^(T_J?9!1tL8kipG6X^{Pu}&ghAV$+;Ud zc7hvZn@B(WdRxqay9Q$)k&Z1QAB;!ztt!pMxuApnF>H|$kz{IgU_3IH%_+DDQO1fw%AK!J&MJSrC7X`d^ zVtbap^1r8F77yQ$sgoAetO1qY&6vFq!ajl4VLm-oZzdr~W)~O6y~91p6L-B+9vAuZ z%gWZ0`6TU)w_H|dB;@FKy=(GDgwav(MH+Pp2Orgj|He0=>YL*tW)KE*TcD8h9~o+H zD-q)WlY_YD0eXFcYN-f}iJ3*YfLjCAE%uyFhqbsncDt-kwOpd(w%pp zcek6%*httmX_b;QhO@GpNRT)|9sp-Q4f9j{`=3V%dviIkCzfbuS0R59P47GVN}M># z;B(L^UWe)E7jt3;(*1JQyZB<2yfrdRY3E({^kieC<1#e~o|qwaG5iei7VEz*J6fXw zA{M@g8T0~-TST6S9t@Q9@y$;y5~DYK*YnSG^9~sZ!mdOWO0X1gqCTPubbFed(=E&y z(|;{@dr-0PM-bp|KAL~s->roN<0(|s*h|kpYjF~9U)4obdMNM+B#&oX5#c#Sg}cIp z^gV*kg)E@)WQ$_`s@~q~g}6T*Z#)c?Lge4xrQiI;DQ-+HGwGrXnNq((xwom?sh-cG zCCcdw;0AsDd!C^0m_rR+FF*rkp42S=u|ne#*3CrV`uZ}N5y#8r)r)VT=Y zJpxL`rKuQ^v6E+C8#+dP>e(7QXzLdfsYsrYKm2P8TU~aByeJb z)~o!(-OlL(DPhK>-Np_5Hl%#pvlqK!iAx^HRY;e5n5U`Mk$Z4$LVBB8|29oCnGX;W zaE*EBPrm$WbU*vKYmm=4*#3i~N}s z{=@I{feh*eMh?c`H%#J++Oi@#G>xKvmSyNhI|;d)i$RqmQ+8BS&5JBbOf+@ zl1V=+EqXUfkdLy=8KK*>Cv;oub_OsM|Gf#VPLv&$hUw zE+%x_swKed8Wk9EU-Q6dNWy5_{c4#2mI~Uf0cp(qb`p!Cj;~&2xzG5^yZT|zNISJb zyC2ZK6fi$~;w~UB$Fx0Yw_(+Ti$N~*vcejQW;n*K+UYxM<;ZeHvYaa&)-6=vB8of{ zy5}wOE&CJ%Mi)68=Liug@+GhvTcu4!?qsiNS^rbxu`n@+DGqHd@ry#zdImF2DIX^w z>|)9L$d1vDAqkoJexo7GPGtfK9Svh*tSR=J?a^P%n5>(+4HWRY%gxU}P8jXvIhjS*Yjb|W!zl^Z-fDisVqyRU9E>^VGY7W-+b-QVm zm=*ERwGV(rNxubzy0%YcE6hmb{<~f~gsEm3bjFKH{kRFd_$6aslvOBp`bHB;HOSbq zv^+MaMMO|Tn68VkFNTo=T2#Vo%EH`%xJ1SVdbkPEqfx_nGv%x*CD~_R#Dq0m(hfWf z>6?iW&L1M&;|6PtV?S06Tg9j@}qmPB!8 zLCXn0NTP^EqiI}mldI8d)WIi>g;Y_{_U_!mhfO>pahT<@Fcml#kS1IeHxVul3!lzK z4;m3k5IXs3Y~LqfopGa>4z)y%zEGoFB35r(vskiODWWm$#$*S@W|5vep!P)LTcH~2S5^9-{nGcFKnQmD(;5vl&PRA8kEY6btKULh0@Ilb&V!ScmF`t zY%}T}o61lSlFMh@Y^(LwK##F(NYaRezQXZ_IE2tv|~))EiM?*3h8 z-e?>lJu9Z(5g)KkGm;(LDV-ycszl(w9^JO$pt9=Z5kUd|bsE3DQ*(Tp8u9a=kyT-| zmJW6FulSUsHnwF>Lpsq!*5?yg`|HIh^vf=-Gba;BE^nji%+zSB%k5$4*M)9T zFi9MVya4xC`AWdqRSsW^rP(2o*ew^(8GPT>hex)l11jw|76xc@>E;Nv>OkmO0%n@! zo9VG-c#4K<8o;;g3jpJ^`!`L^GJOR3zA`0T-u4}$W0kaXnJwGg7MJQA-ymB`!}f9Z z%)yaqs#R}y!|k`J^ux|82n@I&WgR~{yVtxE5_!Xo<`4zp&tzf&sc#<^FTDdyuGUxD zaIi_3geqkFB|W~*o)T-7hA?4@42pb~~52P7yq9~WB@fhQs5hfukO#neWD)tbFEOVKW z%>Z0~Xi{=ddIn<9SlZBfs)`!GMKBYf8xl?Mj~5hl;tpFbI6y{yCe)-NG955 z>hNaYV$a(HYR)eT~4t>2k$O5FVDJVhS zj@?4etU~37xSw%Fnuz?fO}qT_c59LS3VDawf}Qa;hwhyldZuJpT^N+iPPhvb4oi)eCd zbig{P6!Dpu_GrybIgqY_(=a?bZRLI2_-x9?_26fIWsih0jSM5%a?kT~0Z9>V2=3I- zwNK|sfnT|oe}q#GZhK4oBUD6I%y3v3+z)$J7U?TP{MMkX?mQNB^=$MVcx~8dj3+(& zC5>_f?>#}~`LT8gb3-Vud+gtr{dIz7eNS-QtKbT9RRAn8#l0iQAg24jlcV@ZZ!ada z1v4%UTdifd(!9+7D}VTJn8+V0f@BYR^G3m-h4u73nSC3qNl2{YzJtI6J#At zSk(AQfui8pfrgC=F}@kgnZj$ae)?z{mN7qIZ94`~3t;Pk;XK(JU*cIu)HTjA2102D zYjPU~Zk|bP0F|3l+qgBuIE?bupJPSBvPr~q8NP>MO8W77n^UEbvOxxwcq3tI(Rm|D z4EAsUNZ6gT*0K_5W$K)2W|RaK17Kv$-fqfL{H1E<7>aDV9$Zd1u)3~gE9Fe3xfsgQ zhAbXqiXdKVB_hh5l3sEV)~ye2Qf|g zc;a95^}v-M^{FB<{ZRF47z%GQhi!dxQ5ajmfGmbw53Uga!_+>%7)`^CK>;Ne{&l@) zYdGYHxl_SW+a5xfl-Q!?exD~WnCBhmeCYRAFcYXhKoKPZ)_XdLlEYEJ(WU#)Q-N;*Fh2%kNEa{|?81#0oW`J$0xLxu6TTdmsS6{xiti z%9q;yz7^9N8tjiHXlFyslhA;J+MHUBLzc#FC-CV5H^75i_X9Dt4NVB0qR@xn*br2V z`SZc5^Xps2<*VLPyhuhPJMZ%NCznk^6E74T8b{Z^?wab`Vo0#YURniGFpgD4h9SOT z4~X{Mbl>D-@w1G8=Hk#&0dw=}9hF~Dsmjj4K|#@7OaR=Vx+LM~!SAA`A1Fj$hk+HQ zAA5xq3Pbf{j}{!j%+4yVc_q&k4M#+b-3a6vFNb|?xo=^e(lQ@h#Jv!s7~5Cdr}B^@ zit_ppi(AY86{)D%>N{=j%CuYK)bZ-p8L01lDRVjv1F?eu(>ge`0RO@g5&%6NgV2ZW z^pM7x<#|5N?=kHDk!pL?QndKwDWDPt#Y0HY3{ncgA*FYP)Xstq$WIUV#lyUQj^vO? z9%h=Mz-YrlC}7z6){|#h?%~?}4E%t9d>o0!YBFK90c-8M;rWkqLg)#+8~~}1Xd#1v zM?l{DKqe$-%|$b>qt0``&xg`za%t59y~7ze9}FRcv5wqmGRp2@`vqx##i1`sNe%Tw z?&)t3BxNvq>VNnc6^K}qelv&O0Y*arJ-4X3=;M)gJwxyLVg+N|TlMnY8Gg9#{n8NB zeQu%J=ZmJ}2qC7(2UyeT0934n!3|gtXB}-o=&jojXy~Mg(fj0omLuqL=qvkCmT#_B z{uGBTnO{_w|9#%iWjRz&*^6HVGlxH>SVkMBh$}@0}y^Cn6_aT z_`uI`qiN!Bb*@c>oIVy4Un)Zc$rL|GSjJvo$B3bH_;c_MO5II#F7nJ2XWsWVHr&lAe=zyJGhl|F=N7END6si+S^T3wLi>Vh>@U55gC!T*>_z9iP+G}%bU#smUcCssCc z@NB8vG|f^;VV_5V`a_^%j*HQ-=g#Tz%yD+W81$S@jTdSF*d`%*H06Ini11ND0o%B4 zX0sMw;|UjTb$jgkG1=ocA_)B{QCB@&Jy%7Oxj8sC8zcz-wTy?4=IPvgL?e5$eyPai zVwDy6_U1>}--S!S{Yi!H!)nehr0x|0f`LS&OYe0YfQB{ea4*_~ zVa{ojP{8jXR5c3*VI$aliQfxO$ba0)VDkto*v*N8jDBv@`e-Z>BZsSEy7sjhmj#DM z%I|ph{n}6-Na8i-6V2!5;t1+t<|K!)7M-xlnIadQgUTT6+>nJ7qEX2*U}06e13thj zcG;?G&VD`bfj9Su@K^1$nr@F2gAVC|zX` z7Uj+T^A7h4;b-qHpMV+9%f{vlP~RXa6{H#0oOZdnJKT|6q%;Uc=^X^6M=vSoafFz> zDvn<+)j$2Hl~*52%Q*;ANiJpk0sdFHmQufPveb)>Gr53B1*u>GL1+>{@R+8H{Am6+ z*JY+%QmV#qszJ9JOgF$x&0G3)nW^zPnYDGH6*Uv6c_!-P9LgIW-}vIyAg#PxHT<;kP(q zvh4W{%k)ZH1#1H~+tm;}&zpcRg73;7h2EG6RMB_sE<@tAY0NdDwsd!QpC0CVnnyBM z>j$D6HqN4sZ(sgX<6`q^-!P9V(O$ooN0PsuU+xCy01|ebKVmSnRxW&KQk`&^J~8n` zvrYH^;Hv|NH*>FPE@>$L=pmB_KO6F0Zs|YO*$@koFt~@r1O(&>vT*IpI8-}9u}uBq zHKJ-d@a3cPB0j4=V+Vw#1VVXlJj zaKTvTZ5(;?L{*`Yx7fs0wQKOwG2ogbP}ibg1}dIp=P{Do>0Hn8;F00$oHg5srrPs{ z^FIf$`UQL!EGnBy?)s%^>U-GI4Qtg{CQQFVZ2cWbu3vyvt`4sEa!f`PL#*=7B+Otj zF;M%JThNke+sT32^>-u~Q3!`aen?7r@q(hJ<>gqZ-k(@1!iZ%Zg~jtl+EGi zGDbH;DX>|FSuxWI5vkmr0ef+lA2*)`sPr%9pkO#SEy3|%qeGJq2VoF;uCR-HNN+edY zrJmVrEJ%q2HpX3zT=1ms4~93F33ZLl}(X=;l&gNp<_k zfk(ywNH&S+EP7KAA$Bgi$LtI09p=@rFsI&92RDjJI^*>L2l%Jq4NMGym>laguaWql z;k19SckxbqoR{mNT%A{`A$R=1?fOw!+_kMxb>eR$Jxyd4>@uANlY*5bTRCF@iY}y~ zcHsPgY<*-iQ<7GOPnC@V3A-hL(sq7u!I?719*-VNlgG?H5g~+=lirFo{-3y9#`PjU zC#^vq8Bin+7oVt-aFL79RX~!gcc{r$O!lsdOCBh1FKzR`-9>dlXx4)$ZTWBq*&aSWt^p~a|H6plweRw zlph{_jjan{uGHinU_>|OSbPF7R6$a}neB`Xa4Luv8Y4yIBNgT>K5=pNAm%ru?Fk5_+A>EkkXb4>4KP| z9bdCNpivo;K!ekmHoxG{_$12aX(e(KU6B25L^c_5&3ix@&@Dbf4wMehPG zu?^Ubn#IJa7yiYyfFN)UrCw?p{TRE9lbX8Ekm+(t6ZReQu9FTbueqG(uIs^^zl1&Vi2x>UCbnDAGK zv}n|~s}`ZaR3cyb+GyrEe|;tjXC^P(y~Yf=FOflYb?2CMU!g7<)56g3aRMM@Y689H zN6cq9{}(#&FD6-MRt4?d!F-j5HkJ}c|5qgEkIYe>robPVNGR0h=Piqe_@wlW`a&c; zhBFpEn_`eM)TpNiW$>voJE(iDWQ&qvNCiwoTl{+=wi6}%4;^raVmjLDd#dIKc<7J? zEauI>U~W;=3!ftnp5`)-I}!Yu@Bfs(F6$;gZN7^N7ZKOUc%}%3&E@vro8hpeAehg@ za~s8U9b27y(lCa!W`O#uRFE>;zUb@~!ApYD@3NT~4AAg2ARg$r6f&}eUfA{Js&La$ zy#~_F4ru-CY|Gad4ly>NK0)b+FS~oQdRw5ilha2^KLWvC)g>Lq`RJ1H@<#{0Uvx}% zT6>Xy9fMZM@!rF~*F2^03q2VPK<`@`(YaK{h;G!Ee9%AZ_rM1c$c(6H3{CZrSR0kq z?<_1sNrk5-!Rz}H1Ah;MWj|S(u>|B#{l8Ll3!9A4G58ggevHjo-)rf+6*32xruVw6x+^?We3>EdyaTgEo{zVBuJf8Rv`o?FnJQ(5nLx z)UP2oERKbjaL=dpV*JJmDD((IH$~k_g!l{!Y%l%owd+VqU-6jdBSC8hhm{LSyay=aTk+$w^XPq}h@pHx0udhRA%uMpU`Td4K;5U2 zCc{IFUDvzq<@cP+Azlm_cu>U45)k}|TH}f*X@~;+Ptc}Qjh`wZcNRf~#8t1$po<7Q zN7XVDNNX`6Q=bW~w!2_Sb=m9gKQm$O6Vsz>K4(p2i7R`8~U);kT{biQwT{-A_sOi2u9D)$5So^)o~u)uxwKVmx>e z8|>_yfXN3JtO?VT83BO+$S6mRM>-xujV->k93*JB{b-yKm=I62+nE)A=0}Eg=YVd`mj#M{bIPFb`4)e{e=IZ_fDUIi-{X2sA$mcvW0+gYc@xyFY?7KRk@+av() zyP=!mZa=M1b)aO+bsIFfWcbtIOjJ^KFGiTVbSX|PCZ zsNk>gd4~? zIaW?sKs1qu7w|W%k$*^j{5C4L>V#BY&CmD$qVJ1LNk!;~ABH(?x369dk)WQ{%dskq zJTatO-X;NQH!|oylvre^^FJ;!a8sVV`mcCxZxZ7lNX5Q{tv;82JXkK>+wb@MxW9h) zZb~up5xMQf)qYy3*>kWvnkIu8nPr}x-J+eNSa*bR-g4NBX}#$_5sZ6KZMihsJ!J(hw%s%%z3`{DQT zHcu2-;6CD0aG0isSr0rrnVVjq1hUuS3NJzmqQ4NR)+xw&=2#oQpO95qu}m9})q zIa%+bo*gMGtCf^jCPI=tcVU(~4ge0ep?^FnHduxNFUgD&KragyW1kQ+8Z}lBG|?N5 z4pHs1si4z%d3`k+YQ5YuVJWK59|juIrU0?pzp@7}e{lZ{wp;z~a;Loki@5r^?OBBKw0?T(;A@Kcdkl%I zFs+VhO?Dfd+K+=VWBE-N4?(=7q;12Df>gOs9SL`>eC$pVwf%YD_f*d(E4Mp#Q-&E{Z3fw7LXS66`Tk4# z)2z`tHtEA7iO$}4E+uSbbZ>^f?q5fpZ0H4j%2eqWyVk+S$Fpp9`HRH$st#3t+QT~Z zj4TneLLD(vNd?NI(M0@jrB|vNN>WLzk!UJNifDx=2CJV=U%dC3FhKoMyn)HF_$SZ9 z3dJEp{F*m2TTG-5RXgSVJwJ)uf)GRG`1!aWzt@kuBVU32kkTqkp%f0}@c36w)uTd{s66LGj`Kme z+;wg2bJP!Vv&4#iR>JQNj~$BBnZ8Tn?XT@P>F?$8Eo2Y!g?++_B8knU#ax~Kc)cei zrD#2~J4)1Z{^v4y1X4I3#@CNCL355Z6JII~RKss;>&{bQ+tO2U&f^!X)ZX-{hi8R| zwBI^v?q={3i`N1b-Ar!rh=@oNNau@nQlIFJTlXS$!ywvB@7iT`hsLgCkVT~w6lVi! z>5>*BKY}5nCqK1@!(7$G(}L%?C)g(PE-Jh;y#Fn&${mcn-%7D#wJ8u67L+)Un?5)= z6PAaifW@LD@xA!THg#%$S}@-JszvBy*jKS{3i95qA6L>`)|B*Ec;N8ZxvG>NXW6vP zdONtf?l)+qFwwv8scF56cC}m?n4Nr#txP60ws1~aE}RwKA|=G>hihoPp=8$&4GGFFMIrTc+vQu)nzMM6pW`ndR^n6!^)qPu~}Xv>NeUeZKFZkNQ$zGa1E$4U$-~n=m6$9VJ5#9%m07l}v+yL0D*FZPF=7#D31VHf*Q39y@(wi8Bq2rpyy5 zdYCqCyzZ&Y69J(NN#RKA{?OihiFGM?g%S^l`1LmiB!12e@Q;?zH25kZ;3d|1wf3Xs zYq5RHh4n%DA+jkWs>|@Z*66#qiJL=;WQe4xXK&df{KJp~oO|c4ua#WCWry~#wrvbn zpK0Ic`{SJ*a64>!kk3h+`_itw_5Dd{oi*Ad@;@_Ti@u0W2{SH`mTvu?$D}a!awIh$ka|?oTt>@BmwLkbi9!Jnak-$ ze!~SFN4WFeUqxlnLkIc!XC}@Yxi3mIdSWmVrVW3FVdSfhO$UvM`eZ>zp#iO>{7g0N zj?Mr5UjVDZy#}PD`x%<~h-e5E(wfhjjsEo*7S2`rrJ)+C#bVuP)+I;srpu)WgotIl zW06^GJw%L6re(PCS_X6GUlU{kEru2UF3v`wCq+(G(^HI*dUIB^x9=SQ zD%P6%LfuwjZSG2?g}U=e=)}FRd&h40C-Rp+7%Q3I-^V4u59y{d801aoAyXh4BBU_0 zJ5yNZ{U)CrQ5Vb!Jh`{-kQ#)EH{XPx_FDz!z-gls!z4-9$wYrHT#Ej4jZi*tjed@t z9j%p4^Ai}arA15WhuFa9c(w1bf=fg~_{%L`_yd3lD(axK#k7^=yf$4o_m>>GoK!gv zmaDP!a(sQhmQ00l*?$Thw02wE4E_;-Ei6PKRJR@w+_@h7%l#nAtNs;Zb;;??Ge)E` zv07{;{*U%V7r#BX{s6DvmWzyX5J$B(v#Mc>0z?2UsxJ&q^B}UrxO#$!s!^}Gzw>Kq za3X}NK1;yVg=T9?h>@dGW#a>Bn}e@&xsUi_U^0_7T=ag`efDVWS1dX&P$8(?@CPwo zAG;-k!a>*P2Ev~`+uK5Y9>NW#BIq=fhI4UJKdZTx#wfg8;%GIqTu+kuBmjB;((`aBd~@zM8-RQn{0K6`fS=u7nxqhU4tw5Fl8VnB z$(Sa9e#XW4A-C?U{<5xCs>uan}Ev$*s=N02> z-S%=sj7Ju>t*y#g=zpS01(r3_uzgPIx4V2cJ)(Gsu+|OraR{q;c}-)chz!i`@;D}r zos1v&#gyOjjFE&L@s8x>LU{<@W`s$===@H&D5pnttMN(G#vpPVW)>Tq#3Jd&PE1Hh z5j0>JssDed`UQ_E}gOmn#A=YHgf@csbX$5 z_E0u87*6Q4mWi><#-}SvCOc`YJH%VT)h7Ux@DcJ&?+-EcTMxwf8gQPD(D#>rIUV>H!$;ixiAODdhHQ@C&IQ<*vr*zIkx<#@VJ0(DMznw^L#P>)V zca9Z0_~d8JV3B_49F9VcCu<7SV2QY?vYL`pG`>fN9ifl$xRXq;`k1H1u_pRIhI5=2 ztJpW|6tPU`qbn)bm2r2=RM)L#ZJIuh)yev37F!tg!?GQW>Gi4Pibj(-%k`3 zx%`3(HM|_ggHpQVUBOMBZBTp7(k>BbnKCs3!o9 zq-rV=B!@4=b$)Te#gmLuNE%|=<1{Fz8OR9*Sa6#@bF8Dm>@-`S>&Z=^eoQ?ihm>+< zxCeFCl_zAk9RVn8Z{~{Lf7%SJ#OqCBlEf!&;mAM+RJ*EIo0XovEmFra`-j=71+~nn zC7yb%YkU!CNZ_%TE}^c~KP0q<+-Dcuv#f6qs0Aa0KdtIPmECB(N(1}=52bu&5dQwi zJurtH*69`o&ur94*|w`&hFzl>c%HccgQ>#nlv|$rTHvZzTy<6xRN3GkQ%bp0pVBfn zVHjC6G-7R8rs~SUfb@gRg5W|r*ZZ>)OMi3MHWCd+G7PyvgdvEHk#`+AC{&+d$Vvd( z{@p_`rfoN3|2vmCvnIh2pnN`gor~c;|Iv; zkLJPj?6XWMRc-I2&B5%r4$i|XMUN&E*I#rb=MG_N<@H6pi=wM$<$KHb?F;{`0aba@ zoTtv4^x3%qzZ$|f08QydLy`3|D>_LpjkKy-Bg)I&m#aU%L4K<52KF>?;bb_-*zn?Z zL?M@83IicLW-F{C{Dq0#0sgkR&9Wnl6LUetso$XRyL|%YtulN9GF5@42ggZ*`J4LJ zHj(=+MWSf!MGnsr3`}g)v#Q1j}vl^Vt5v5c3MNA)& zkL%RuFbWDZfDk8y{ZH!9s>!ocI$l2fHtSCW%@IN)?QZ|O{RhumepA%i9Vb~>unWW`z-Ug5FXgK1+acg8@ zhK=OVh`Rx0U=e17<7hY*HPEys0iS=Z(UcLqm{9_-Qq)m$u`i5I^th zzD;d(zLJ|4nJ)zC2wS;h!P0ffBFK$wBTihIw^!7C^BCF~%!_$nybO-5*deaH04lWJ z1U#lX3AIy~kcITuRa+YerGca_8LBs^Kyy%s;Ql$62st=NM}xONlD_6Vd(>Y;UyW;NcN8uzy!p5JP%93zmz zm43PO<{%An%(^*Il03KuBdr$Gx9@aoedu{wr1FL%q-WRlXG-{&(5x>yAX+R}tlUdl zZ#b&0=Zhdri_ywvgLuMm%TB<*&Y;7^(-s_qxdqP@(epa#1nI+A9z_2`WpAZX{Z-=# zSl>r)u1*(i0PR^@8Lu#D|83AklF^#~_YHWvo2gp#$b&Pnzu5R>ILEl;vn2S~TJX~F z5#NUD`q2vDPXazPMtWXQtgJ@`$lB1+QM#8Vc#INmQ!3q|qVLPC>5jwF)KRl zrUQ@~uKuAd3I+mp@k$F9I}hC~>eJNzKm@o)IQ!AV6VJ;__ruep#uq8_dVt560{CkS zPB$w@9}LBxN{>;MUWg|a|A4QKAp7irqO-uV30pj+||vPgtB#3 z8IT~7i>~W0V!o5{^&)xJVPI9XxG%x)p6#j2Pf&W^8dZTN#aGpvDIx7%L>zz}!4Ht{ z)_FX(_oAiH$#oWVokaVfYE+5jIjlZy)c~?0r0LT^zC+a3JP@5dG zq0si(w@|BNN2nYv5!8f#%H8_u)06#57V^Qrx#ZcUnFBB~Njn@hd{k8&AT!7T6kJDp zyJu#7IA1hIT#^co&(g!;B|F-gldf_2Cv%S$o*bXxGf}+8O~{`O_#DpSi8;F+U1W5W zZFlD-c|>`C6=UZD%xEb@ws_hamtiC{EQ|u_y=&pXS|ljUtAR)4&iBDVL)b>=X96$A z;zZz+m>ZQcg>k!|wW@|KLFxfLJq^TC%2M$9D&W0`)up{TIGAO5Bn39Z-vS|90>I&n z5*|17fF*?c0-3K^l%A?k{=^dqf#+8jMdVJD-+nBpY5S)?b1yRfY6cNO#&_ zxp9c`ZS;*BNPN+^y+s+IrQwlbfjNI1;rJjGfnGxlC#oNUiw;&ns>*e;F zf!|Cph_7E}VETE+pD}lGit#_=&5?rDo^JRaK{!!&b-q6A2R&O){zxpQnqY+tRiJ|b zkqUob?Z(s^U5H-vwny!`p21w;QqY;#9=c;lDzK!o;0}+sA6UAcxqEPui6b zkJwdlLK7DKon6$U!)VX4Dh{CX(^+W^CEd5YB)4hGojC=<^w$xIhU7JvwtJ$M*i0n9 zK)sLvF)2rK9I}xAQ{7O{1TETogga!9{otb}f}71(w~G)=TH99|O$6gB zqw{t@e_!ojbFzKneQyDb2y#x&0Ic8N_Z-(BghYf?Fio$D3x?`CwkkJ7!_J5#fu=5P zW%VEbQiv_Ya?kPUj5E0Cw?1 z^5c!mc-HX;_ZI4C8LG&p`tVhV9JUm;W-TQ()KCHBchX5OQvrOwpHv;LznSSvD|IxV zGR_rGnoOqVQW$c38t6$6+w&=YlVUX*I2NZeD?PR6gD^!vR4GFY;plK~XKBSZWM1MG zaH2R7En8dE55ve;h$uN&#z$NanK(bU$N$Az3^{S>>-czC1TWCTtEveDM}TvLh+LP6 zi0`Ms_NR}XX9MvxM;KVWf2Cjwhd&rE)d+XC?v&j6SH<)Z?3t772f;NCR*hCC*?Z|a zy(C4YpK6FiouTuliDeT&I}V;R4pN`jfvH&wU?YsdRyIGY2bXMXfzOFu0ea+@IG){KmVoB$Hq52Z*)3<9nsE* z2~~^b8tF6Ql1XKvBkkFg%D{_tLZL#fuB%%8XYg@9*y#q>>;on68kQrISB$qJwUnvh z^(Y9W)Oo-$!^aXK=6F~u^?C0&$ zRqxivbyS2p1VrK*Ug14fyv1|UDFCa%$##a}*1S&fJn0=isXCV~lTBF2*h{qn&9zfg zm^38|wJsm5LUa^Iz(#LZE@;AW~Xk}5YLI3=)9#sIpdES4MYKtaE#-g-v7ix+HukyyxYJyFi?`Yt(!v;UVeq zj&+rPq5?MWqIlfCZ$IykZyw+6mX3A5Ij5~2G;Io1otApu`_Y=9q>+LD37*cORFsVi z0*I`~E5wI!NIXL$)Fc*Prn7ZR zWU$IIH<4+Ym-n;SbDa{tN2d|4`!gRO4TLR}(DC4dcjp^;lS~>niR1RAogmDbl&owx z0L_>XfPi90GBwz%$QU2CW?->pI>Bh+Czzaw zL1;H+rOpx0o-fCwoRQ=EyF(Zpoz|MS#G>3!FO^NGaSC#23^+d=_&+{YSTkDYG-a z2~PQg{@bqrw?Nlo^%hIrzUyYUO=m+(U@N17lBD%;m)|Vu_QEDJ3T9E<57~cch}XTv z-NQPS>HM+oeF993fBiq+8NjUkH-~6*q_2j~qVLCrV{p;zDn3ApEGBjT4Ywv{_Xpv| zk`fd3uf)2v`uzU|Sz4j@wYL(*;x{y#qiL&(H%rs7Vz4e*Ie`&hHik91J@+MSbsER; zI6pAY|N3dKG1FI5TSF>>k!83UA_rG$Gh}H?1U_YXrH|0^Tc?W}0)=8=?F}^)2aZuu zPfhY5qZIj@`8YzKu7k-MZvzS&f+P&FiQDN+be#@UQwh%c;zKowk>5FqWZ1@Vmt;5d zU5T2XY0`CXxqe^jh-dov*^=?f^f;j5W_`>1f74P+6~UpTRV>kklo4UyV8JJ*Sbv=J zzgLChtc}x|MHdM1K`L43=!FM=H!QL@pO0?*BO*pMO zt{Hu+x7!$04vBZ@>f7YM@a7$VRiM}Q)&c*iRubCSki zRZ(vIG~zV6FAoh=k3jzyh5FXKT&zeHiWz5g(DDwTFi@r76bvM`l5{|2Z1_bL= zDXoIQf7`y{AwRf3Kom~THSPbbj$0%Al&0DLmHU9k4pa$pa&r3vYfhdW=PbYgx}>I? zMg4_oXq&2iHv`BC9ags2MjTQq1>_vk(+(3a4%_?vNJ&@F0JB%_|*b_T< z_j5jq92!*?;f@NzA^MRBisfE~-dadT_*y55or((jVn1TX!JpHF5cCawZvB_<+o}_= zJ~X2f;Bwx{pSk)MOweoLS%W7Q6?Dqvny$MR(4CnS&Nf&mZ zrgI9o?lsE~4wfLUwjkFo%K*UNW+(b3dZYI^RGZ4Y>XTjCB<%U}yr-Ue2m)mtGHf3f zZLOEJ=ds01u2bA1QYAdZ^p6^dXS>z!qZ&XU;5{i?`TT1bJuz1t<~hhF6yL_?`APe= z`7zHpjZr`};rY_atmrl)?4ky?$P)CW64BBNFknlJv9@p6qL6oQ6ogI66pv@!rDa^O z;ThK|Pp{jZ_RS+$v|nRid3-*L^Fhz^M1VDuC-E@$!VTUd@Txsg%_wuKC@L>j@=DDr z@(F^%w1T`Kha^{u;Yoz)YKoeNL?2YiAv*IKLg=64^FUpGw5ILexh&ojwHGY$g*{}k<(8{MP2*(M8nq8 zRPE3xcwnN8i7XI&^>KC1{lFlL_GMO1@=ssTSX`D@Oj5n6d#GnOHaH!A=nzZ%R@<{< z%kO$VU(g}`J)Z13xwKNEdqA3l6#asPguEZ_6S#Rv1ek26rL7@FUMZWu z-<~*EL2Q}c934m@Zup&ibyombg%ezD3{l>NqO;Vauf{U)wb)em>`PCO&onJE9P--~?lqyX0|FtJ9 z11(HY({Cc&fD;rOj+%z9vMmp?K=e2e&61RBdsJgL-VAAceUb1-p)lS$CtEeJm~2QD z|CLeOpm7bGQ7oqBXmXS=gfOie81ZAOFdDXZCVd@#X^s=|^g9eBul>uIY&e#|)(pj^ zXuYonf8d(p-DLdvPhQh85!}wfqR@|mHN8XbP$S?5E0BR3G!O=ZaM1xp3>a`D0mB7- ztpZBKrw1K@By^cP*rH-4K*+@JYRAtGf(^5)@@&5_3n~THFcXq@YUZ=M}D0TaN zu`4LTJ(jDFT@8yp{@VJ8M&RjD0B5(wv*z_obD9cSsI`gQ^Rl)MOKw!TVb^f^3FOkq zqFzyIYpO~0ME=+l`Tq{nTR5P@>T>H1!KDFF$P$1WuHW2@PCAdPxQmIr`~0ar;Js%z zenr&;UBKKyntuZ_>^sb%kG|Kudd=g2yv&Esr*TqcRNken{!IOhpF^m6Cw$4wWI{S@ zrK?|2+EtSGQDB!v#Q(%wAN*azkIr`H9WQ^%blUgv`LXK1wtY(+=C%$%A&%46f%FhQ zHf=;&A|D}NdTvi0E)%YCfBBC&_0Zv{0I-YUeR|KaK9 zJo8Ga@AQOQBXi$@%!mI9T?hGdcHhBAM59r5#neghQ>!&J3Iuc>g62Z<+li6!<>HQA^GK+^@>q z{5)c@K3mmK8L06XX*2N(@%hcp>p$m~MPYVQlN;+BS(ax92V+H{jjZrH1gN7qX;?8R zarL78XT%|fuL`MN)U3m#g+uzVQ3w<1K7&J*W06xEbl9M2IHE!~Ja`g@nA^?cm#QP1 z!}q?H=y`2!tke6`BONm)OY87GPG<4wd6A}wpF4~U*>kI<1%IJAB?I5&q_Xur<#?;W z9}xkQB+1!mE`Q0(Uj09uv&V{@%hosI;*w8mPxYr(NBg(x3EFEXLYEhR3w9BQFAeNf z_Q5ZjcSB7@vAHgT4&i#}Wm**%cx4+Oj7+!(DJId*5RiPMMu$o9R9dlWW5sMENw^E9e= z!lj>RyAgoXU-B@sjARrN%}NWfCw_IK>Y9}2-(ioJY$x}>8X@;zPP+WR{(RH>8*A!m zdH*b`359esW5NH3kQiTfeSMwVAr`QjH=heX;GAb0kBdEFte_3Ba&xRP6TD@$_XMY; ziPky~Sm=v(#E9K(GcpoY+nI?ea|ZmOXA#wBf#?y%q_4Nih(8SdLO~VRD{D59eEECu zQjm4Tmr_d;I-ORec`|O~cZCksx;U_08q4LkM+rEiXmdKr8u^v}CeHC+0(h-YFdCnT zMB%MjmqHi#`t5WSsjKt3K=)<8dIuAil%*bvoi8YcE##U?y`R_g*Att*2|*%!^Ix?&Yh>tZ@?=qZZLcTexG0jQ!AP?!~g9QnH= zve|?#ZA(n%Qzt;-?le+M%4Mei_zOevB}9|#=8!Fzp(|b#O3@07b9U$wQ#n6Us}VkI zJaZe0R_I*^Od}KL+@?$_#(Fx(DdJQuDd~FQMGT9ZI~cflD}D#~od3AoA0OqVQX?${ z1p%`_<@OUm_q2Dkkz!4c!-RfYp`1gDX=sC zzZ`cUe77E=i z1CA6r(K`eT{>Iv#4j+ z_cu$#mFd4s2M*0D8wF*&6ulC5Nb{CTe?ll8_d-XkSjcyohbj`<(Gn=MZG!cC71v2h?MG$bA{0 z$ESYshzhc_B+X=(?C)__sF1`YGh5ef^7Jr5R^=)Ek-TWwDUGH4*|(>#J+LFnrhc{f zp%GYGgY?Iik6t@%CajDj9;2ySECm!dU7q5W+Feo79-eG`U$7r%G&w42M>Ga1!ostc zAN#c-6Uy`ylM||Ov(gnFG1vqqccGgkcJpW$D z>Yek;77vJ&P%&UbEB_lHZ{c{z4aW;IjhL}F37HZVXQnPLPB824GBr+i{COI>n z*51HTHU3rp4mGPy{#d|Z;Gee(illXb)(Y)69QIySqAsv z%fZ2cJ^t;xm0bDlD4E})>D-E1j>_BNRjQU4W`UbFSEc-H{1}qdlJnY!CA3yvEt9*h z-_P5fc`fU|v)^?bSLw`$cf37c?uB|)X7}kRYwN<)6472$@+I^_glyEJ|wDy2^xEwMvQgOtf0H45SFL@)`HRbdd!Kb^~juI1i zzkBJq%Xm5e)OKIm1h&q~5R}hLsT6s}Fn*m?C7|`(a}w5_*0cL%Y$r%(Tj!B1AD(1F zz20(4e290$GJgNQwyYC15H^q~;L84reb1T^>ht_WPGBc^_h%R-0rjjYq2apHWTQoQ zS6>28sB)nEDqDmwo#Ew6R!8)nN!%>%U%AbRlGEdD$&+jxn%coj6ax%hF3aD>EJ#C? z40wNAf4ds%JQ-}!qneueUb8w@DvmOgHSbzEC0INrU7!K)J~JoC=ZWs3&Iq0Kol|m{ zgu}LOj>k(jhoe$y%wrD`KFRmo{r{rG{tVSop44^mBR#ih4-Gl0Yf32eR7bi%+jBT=InAX-o7BsoCtn~#CL~=o{0dcHBah}icG8ng)*&lWP8GELT;HXaoX!)8}3B`V)7G_j@TE8#% zJF;kqn6ITPHaI0{k#|&b(xH?eI3${t2gyXpI1D)bqGS9~%1JGUXG9mBwzgx@)KQPA zZuV5qVr+L7;6TVR+slVQ=NYVMTy1fnTsZmnFnkW{pX=cI{QTj?5`A>}zRnkJnAl(V ztiQ5m9V*$YbQ=ii1)XL;YqD6av8*=t@6_B>bu3=rd>DBU@}4sI{Q zV6+vKJQnkpvj!hY$tt(3uj6lVzG(S|XGVK<%A?h_Af?4%XA4$fh!f8bs%S>7K^+!m zlb5`OikzNSGpoj%kvGM2W~qE%2A>P23KBJN3@M{kt654vSDqv?OQ%S7{n2-#8s-`< zOr{>_qT7gK^x8F9@sCmdIDe&XlF$@%-iSuoeAMKF31<6I+fas8m;nUG)?{V~4RBgWK{&UU%*0 zJe;t)?h@#v=CrW#D3f5Xu(a5Fr2ygKeHX_M>fUZsHQt!C>kY<1?yTI=N*bFUs+!l;#< z(`Y;Q8-d4LBwfGJT(SA1)|ZrwJX4HYT&KixC^pJLzb+h;FR zmRq+4DffW{`}JRV`TP8ESjGKd!`Lciatp=tq}Iwa(F@Rnl4Hc-Ehi*Whx*H(p@1c^ z;FBO5JMl8|howAmX(>H8i2h23iigfmm&=Mvl|pQS3k=wD;)5Z;(%QZ|97~r8PaMlLr4)!kKfLL|dh1=x_xdN54(YP(l?>l&Okpjslf@8I0BK z*DVK?(%E_@q^6>u9L)H_F=ulo)OIyo8ke)BfQ@D2Cw_kux;Z=Znu?~OLoVKoZVQ|? zTY$|8a+%|KubB&}M|FU2?d*!B3)F2l-7Y&cd9M7H5BKH5h|REMm;XXT8DbLco7Rw) zWCZ`AnuPVto?@U6((98h%CDj6>7Ek?TlM~yo>*9RCg?N z5~5{gPKr|%^B&qbf}^oe=#?~;5FL^)f#5xER8nGa2+EKgr7eemU4GHKxKa$#^Jy(i zv<0*vr=?bUK3Pyw@Jtd6YI96*80M_NTwzPYlCJbeGQ4wQIE=Q7%p&j1Z~u!OFtTBS?fantTE`NZ!)JaqbOjZSPiJwrch{q9;g`9xXEVX6M_sl zsk4bl!9~zgC5RT`A-}-$uTE>4lj9_AnTxGR!4<~8N)y88 z@$8PR=q^+2_Jymvh&rCB9StJA_J*5VfTmGh?f;_&y}uee`7i!RYs z{+(Y6x%uI_r3H)JTenc!F<<(b(NlGa^MV4ruE-a@4pi)nnfUeBNcK)G&&PNa*_jA& zAk)I)owLC{?;RPgXOEk_MtKYPKStU z#2pG>FzECt2WHgf?w))z8yBo9=BM*4$KeV}7rfBq{b<<*4++?M2)(nD*pkxS-j?@dTY z`3$%rJ>}~|f5d=={pvfi5%J)Eix^do-y40L#o5I^;;5Mj#V^+_2l|oh)$fyk3*fUi z%eB8?4qfF}HL)h^>vl{3f*TBT>tv?7oUUp77zHkjanW{Br_bq=>H8*9ViK@|-v>bs z#jQLQR)31ek|78Z59*Wa!`#&Z?a;$*_24@G5mIK~?p?iUIi|mQUlogbJJ)EqY$C!C zNlF!Bq>QO&(-m3fpf9m)9#Bh=POxIB_kD;C=LX82KXM?tmNU_L6Yi z89~0LyeY8`!qO;dB|NOHpNYX&z{i)%-E~k=3ds$QN#`myXVds#Gu$#ob zGYZ2HROVzZ7WcSF+g)Bc`tA&U@~Z51H- zt7KR1nn-U82Ux5ek>LIx>5DTt0U|OIKas|AYf|-7V*l+@Q_ze!e2SI;YasMR_sYRN z;pSf~Q&MB+QRZ=@!Un>Q1Ye!cwq~9xdsOR(Jdk=<&p=fC-dI1yHz=qQ#VHmR@2m3W zrOA)jC{8N|{1DvHtZ-z}G&NS;nJYQ_c*D$D0Pm&hb^0}!_G<(T&~U(RlNf}ytnq-TtV4~Y&{jTTP z=Re2AbT|}bYQHL&_2;jYrodjJ+9mR2V)k@+GH4qNqDNubrg z59+3DF+0#wF$vy-JjZOTNF1n!2|FFOs@{J*#F_Y=%a^q8l9~$AZ`$@6-kd7nDu(!v zvriN)%+dRxBG<}FjM&UMS-^K5*4Orj4LZNczi|8-yO5x?Js37S%yK<y|;bQ?^OSoX1j~BH%)?8{Z7__pyoC3RY zg|YEPLLVsL+Ab&O+T4fVP=T^f-ZtJ=0xdRkP2ckADBBW_XXlo|(nNly9g=Gog7l{i ztv#PeBLSPQG+rao+2O5d59={4&Xnd+Y61K;?%y>KFUYI?TB{~V?T?7gZ2C@3qi+aj z2LNAS z1cya7y6!k+6BK-xqo%_8I@ztk%j<6&DO>1PkS&Tj2|gK?@sXFKi<1J9I2z@7|2VqA z>vFn2U1=R?bX?Hsauq@$AWFNaO&-<#4U0vIx@jmF&h|<&4U6iv2dDs&UH%L~&-iPB zGRnrhjpO7CXs3x#EJ#1}F)5Iv7z|8u3$&M(wAQs#CVuKqK zgK@@w8AWYp(U6U;2yD8?`4w=cPkdXL1N&5g%_>M(A&g*7yoLFNZ{NzZa~D9j=QT$t z0R9$Av!#e0s0h&$5c`BN*>39SFw1{%rchzEB9jGcm(WSr z$pMmzBoe@QxBh9{x5{oY_>8?hlw=0%g_F_Lh8;^8Kc#3@`*)i^Wg*?ym@MJfnN!h^ zeXpxq_CHCeMcWd(g}%TU3|Ls)e-BuGTEnoBsYBLLqV^g(Gr6E07h4zJz^1_jGpZ74p1cJ9`TChc+CFI-Ee7|sXW5kGi!2$kGOsnG zOATX6ONc-rmHg%zSC?t|yr2%*^=Myucc)y!7Z&%Lyxcp4G{V?7g~VrC%sF$b({Y%> z3&iSRWV+&oIamXsBNhwzI_*Tmq(+@OJ+z~?!()D)qW9arA3l7zYI#{uX~}TWi5aLM z-D>sNr+T4cBXh61N8I@2LTj{Qv?iESJ#Za9ZTPSC@G#B8fC!Z|(Kh0P#Upoud<6s9u-(Qxhf@o5L=Jfhn`H~CA<11-rdon@{v#`ezlhi&N zH?L#Djj`8N8Xov3y#`Bh*c~c*ZhzPHfdLbMoF}+Vz;I!bIeQB=4AF6*f>IP3VWuF! z3P6qVU{S75TGrfuu)C17#GG!5$a6Ea-g;y-coKG0w>EcV6H3MD^YpcU^nfdIMa0^q zf*szocjotzst#3H>>vez$67Ga)d$B}^D$CyzKOKlf%R2WIo)21mT|q;y&UNZy9P2( z`)CLpW&hYry_!u45@@lh$y;W*ojF;mt9L3!V>n~vjW8w3YL-6nu3+>3`JLz^LyKI5 z{rF!ar!p>2Zkx~|*mPE<<{xQJE~xRAj3I#4hxg8afLwU0W@QgJOIgQyM0mnFtc6~D z6REj9@v+9L+IDqN8*b$THb2t`Bb%L^WOmC2GYM(N4~hZ$Qz61m#AnYub$yoY_eQ7+ zw9s?`>ynE@%tM!49#IBqdUsQnuHdRY0pjJqR@f{*QeF>M?i{~b-u@&}D%f(jSzmEJ zCN4@w#k$hHTU)F5jrn5mR{G=#8c?_iTb`*S4NI6aBezW8KPHas^e#h*<|gF z%H|H8!_@V@nzLfOXfI}*0nhC$XBZaauv=7=6%CKEXbCU^G0Y1uf1NV{n=*_O(g~|t zrE1{wk&lIx@1ALW${!Y6)NZn-j>;eCKPpK8Cs=RI zzP)*2b(`!veUWgHI446$2oB18Hv2S@&^sKzPIpX*{)WLGhmo2}X55`2x_iNKw$iSn zz03Z`^?lbkt=C;&xZ&_L%1JF87#KLf3yS@inETVZ>RCTi)I^nviAn);7=mV^Ixn&_ z1?Kx@B=kLc_H(LCb!)LurC=>ux|1?lE9fi-qVc*?$A;|0=-%00Le!>$4sqqSgto_< z7$x*C6WoRq8wg}Q7H}cYA%`T2ytcHk#=tpP{*mP16Ozq5CmOKRBnJ@E-x>bXLI6TfK(bg$+>w|ILrASkQW z#=+}L;kCTRnc>iqE%S9TXQ8Gk-j&OPme=qA-(?5!F2WtP8UBM%KwUtDpKQ*%nzDd^ zP?@X$!?oJ#bJ1lM8W4{oKxS_elyiff%TD1t|G_;H>Ibo5RAQZ1VL#Vr-ci(1; znk!V>wf4NakF9) z+4iU~*#`#&Q!9^Ux-ol};X(9GIbv_Z&@eG{EfDlEGNX572Wzf(z!osBdTcTQ|G~7T;DKs0^S8HwCJ7Xa_Adlojv;Q~$|_xQq+5ya^Gv!riJrUDL8a=0m@e_` zx1s(=bfvsQqnr7SdS2tn)zvr;U%RT^?$Ysgwy&UfrkRIY$pV9 zNH{68LiGP`WVOy~BsJQ%ANobNvyegbq1y{LyLc5%)?-?6YGkn;f^D-CCVXj*0qAS7 zo?1ez{4{Fl1O`=pCS$_TK)seF@CQf7o8w%eASqwdgj~Tmdi}M@*c1_GIk>>{&{_W~ zeuWmcXdg^eEJ;g~B;xc?A>y#$%XaQRb5s?-eYqUk{c>9hY*SvqMR+|)Q_8wy>+(xy zJJ!SQ())?pVAO}YoBYmNkZ-3;%e6O|pcjh&51+!u0=QQ^O(nJcTb}rn+Y_1B0mWGOLR|RHcT5LE zalCuv%GH6P<@)zoWI4$V2|i}p_1b96NQ{OKD2%zVWJ}#uEc9>79)9Cgrd1-5I(9nf zj9Pz~fE*9lWkvn_n}8 z5D#jJ7rh*W^(bFpv@D&Z4y_Sx5#NjCwsAJ!$2j4T62~^pcZMXtRV#Hp%7%pnY8$Vn zy@sjj(&ChyGyrJ0YbfLYXbk5e+={pxO(C1NdUQ~O)6BZ?`*X>>#=5Y)j27*iBk&>Q z)TSu-4+B*^?vX9xpZSz!%rF?pLWsuS3#aXR@|J4SfI_A1BnT#lJ;aP@}!>kbal2Q9SppbX`%AtgrPN`Cmr|M*&d3TPnlD>gc0c=N5$uInUbQ8Qq4a z*-xab`rD5CKYn3|5eIq~pGWUldfw&mfxOz}I%zT`1c5QzRdinCrH z%So`{t-gQOqm!YtoXNyo{B){4YvkB^b@{w+0w? z>q?C7aY4DGt+VALWVfgMpCJ(y#$`vYZa|nWzWHEDZ%xI-graycd>^V1{ppptsjqkU zH^?*cL{0&6CJdYRiK0Zig1*hf;Lk6ynsXY_*gswnmsTZQ>DBsHe0$q%!28v%v?9D@ zc>GjVbpzuA6-A;J${6|=wofYyfkSvQqOPhcE_` zb$}7PASb%8t*U#oD^73OdQP8&H8rvmVn#k5i#5!YUEH@eOICPNeXzgRP|4$kXZ-l% z;qY{}g&LJu0YZ5^OVqv?fbdP)%Ja;3oOPlMA+~gwVcf z0H<8hUzPU@eoFVF_yBfJ)Oq50h{=SLiA?Z7MXba}BKBPE2a;r7tjr6;_t`-(E1b@n z1IP9MSgDPF2~9haIG)e0bbZ{#i+qTn#?XqqQm1^J+HyMSphDuRu{UeuJki^@1_>kv zT5%6-1LP=oqOvap8AZlX_=7HAYg21DkE4Q#x1^$&t!mcJ*t_<5a|C|MCk)q5_gOvP zudAEQ48fe$Unl(5)=dyepzGgVr(0}dmxzgfaM8HqW9^>y-zPiignwpsn4u!2T%4&y zRU@9^#-TV`?{d>lv`kQFJ=OLsSLi=KG*Y1aKkR*lS5@8DuIG3tK`9ZW;{XEE9TFnl zNGKg5(%mi6-AIQ>cQ;ChfV6;sv~)<n7l`eo7hmj<>~ zRg@4tHSEQ9Wc}|w)OD^M)$~y^Ts%`ACINkQT!n*UAuv}ekheXv!cGQAL|dD%#l_;- zoXUa?4mkj^kRYx)4&@lDj|{Bs370RlemrJOSs(|XTj%6hA-xRGTLx`dqCpP?OxeQy{m=RL+EU<0CHFStyM zs3b`msLPihnI2*AR@81}N@?&0_WSwIgGBfRHog5sPfV?U_CkXgJY-JOUyDIs<^0H7Ik$Wa61 z@=<5h+P<#ul${s>ua+za1GrKMFID5S2*^SawFdw4$}`S^3>mDMgCXR>-8SIDI`W9|!m$+b zP^L3NJ~0M49UY#uFLZE7sOae;HozV|M10*1Pl(ou+GmLVy z{aqvnK|@ny%0((g-5hW{EhK+Q89SM9X0v)QWv_#Hu&3MXnJBsRVUu*BNs+j0p4$$m z8)sCc&+_R*7!vXiH?)f=Z*3S;dTVI39d#fHamZMCzN2=#JY0_%Y`ksiPR|A!a=&@? zYwMhIor|$)L8G6owq6qRi|m~RRzGss9JUs9PPR8zf_TqWQfM7SgdL;M(H}^})r`j1 z5;;vA9N$ZHy5?-v$QFD#e-+_Gkv`;V79hy;fsxt#d-AK;Zh5dHw_%-@T@==yCoU?F z)Q;)*GF1ayQ_v&OyJM4c$pxuex_L>-A?mo1b(5wf<|=Ke)mqIJ#0sdfE3v;8TS7o7 zhXt@E#?6|ec+Y-FZMnIn&jBdc5wV)IPaN5ieJEZAZ1lDglJUqUaf`-iMV z^d!k2TyJJX;AWne&;a079d`vpyja-yKG^DE-p=Z4f-JQGPXp9R#9zr0hz}Cw8)y}i zbi6G4eI}Ra*v)~F%JmSb8@=psJoVM-`txP7UWsGcA5WSUgX<`45$qx?ynNY`r~2asOr5&ztCbcb7!? zSMT$VoNhYWjz2TG?LuYE5zo?RTqF+4F2a63>0we#uid>`a|@IY>dM+>+@K)BTu*4j zck=Tz`t%0AfaTNMPm?W>OMDPyOvjH;M_a3S z&g$mEvB<^tKGgSaRAijD9sl63I22tUPBk^YgtfS(oYLkGB&A!z)-PD1C1|MJJh;>@ zXYLVeyFC=k4QUt{Ka;a?Vbr;V7)k`rW|FtF_)5g761q4YZ{=I#{9J9@%f80atw?R+JIU7-XrQaC zvskFLy4{Ul9=7lAXo<5?eeJZ#dKLb=&zC^~;}gki57_R_vIp_y0(pm_nZ=cd<^0Ly zMQl`x*Hf~Cq;JkJ0JYC0DJ^=U2C6%Jb-;~1d3nuO%jFUq$0P7Pvx1SmMIgJ~I?e{J z#sz2lL`6|lQ>6CnAgS!l^MGIUSI;S*v+RCxIjESV5ij__E!*g!0lrQ(x-2BC*Kon^ z80~*62rbZxKfWoA?LSa^$#0fpf8T3A{%Oi2F|0tQz~=4DSayCM$0HI)mMQf#5;IRy8>QiR^>6W+)mCpBOq|M@f4X+de)z$lmg7s(X zk51};)-&r8#oT#T+NH%U{Sy;{MlyA{N*Y+JD!4$8PH*EmNwH)Luj$YSqLg$?U4a8=q=G+a`HD(>N z*tUV+UcfG|F0rwRe^=YSR^^P#D=8Lt1<5q-PcuOiPcE6?ib@EtC;s>WWcoW>RpNo}%9 z9aPllbXPkrD6}IgDW}w*pv~A=yPN0prRqBBX2)jB8EN7MR+BS!S`{Way(lgxYW5rG zLn0@32Z9up9{6Rflc8TyF4pXSQ_g7xCEvfdGANP;&5$ zF&x?{8{$T8JZ0bg7^GdgyP^1^AV*Se(gpKd`}+WVd*a%b$or5_St0OB={X;FM`^%U z>6b=I94x*t&UIG{rag|=ogJYz7cPl{GfOo^(;Aq!hV8xwCzZM>;(4Eb&n#-CbM)!ZlS|3)da3o;WoG-*SLDCYhM3$QF?dx~*g3TvL!g&0J5e6}P*)8N zv?@KLcYv)hyXAAQZ>+BE<`z6CI=#N9+9eiU+M((CXDPeSj^3}~!Zf;Quj^z^F{ivE zh9XV>N0#FS!wdxxljIozmmLn)QgvoNB>oZl`@8R-6}++3f3PaI`ng+@?c}g!V~wJl0jsuV>zd>aY+x33}ufzF6krgJTu@-IqM9I;n0bA&f2)8!CKY zbH|Rtyep> zZawf!!m0>*aN zGoJ+2%lL6k@{hGMx^pF=Q&m#O7OixuC~(gSrS@@uNr5+apq?5Htsfq!aI3heXyoq_ z;^Qi`k7tkZzIK8d*Q9-UiJqpcqt0v0sg?6|;x}V1tNr=#PD0)ZP62hq8MSd+!)u>! z;R5eWp`k9EM+eHh3RsD@hYyR+f74$s5#1vpDbA1@3Xv-33n<^TvaF7CPkht>1f2Cq1b|=7k(pWlg=%EwzY37=lVdVgvPmEv%h)4o@%6hI}EV&@^6hJN^EN##xyeAH_WJkc;z((D}6jl|@kx=;-lUf~J zx9=yRXNLoRT8l`NxT=T~5oyvn?Lx1GfohJ=Lp8#6FV!hun56I!0M<8L9G)OpG{4a# zXU*bJz9Mh@Tjcw_`-EN+x=ITYze_XO`%;M9j$C6VuMJnTqi+>9GVMNYx7NCD<>RN$^p%qxpXv!qUhR%R2`*HbY17~gsaA?A z4tWGoVVk)_*2h41XfzcImS$jG7(^jT zqUmk%SIK3b-Q81m=G>NA*0`TT(^hBn>`R)RbwD=uvG!E)+Q6hN_IKEl&NQO5`*Nr%EVi zM*9OGlWG(kkdxYB1>TGJsx8y8(hpZ3(E7cmX+w zR$p7auFyKna;iR6P4tsplB}p2wR{jsXb=zRsOvr~i0hSB(9eNrGc{nMRWA*5g%Ysg zpp^#+e&SShzy~Y7L)mX5CEN{L7K*f*J;!QsxuAN|q?I9URy@;!EcZ>0FMfJ9tCy?1 zB-|zL=qWwpn~zWJ=J^^XOA0>YfLc)Yr0NNS5i#U3mtIPB)+hLaE%c|1*|uOqNKUnu z{Erq$HVBlJEi$T$@vc2f_8A_=E@v03%1mUfjkk#DG1p>Exx7K4d`_h@6|G6LkRd41 z#+e=|7!YhB=xBX;86? zvnqvfe~>|xk?G_gh(lK?$aAXSBryHBFClGqbsR@)vpQmO_#{tME=y%3-Yiu8M`Bh6 zM|zq%Nl9gCARL5fhkXaHS}GRPMq|1P*IwDt;II%QZoCy;U!fJR67|l)1Q@d))XV0n zsFFoyFB0SPI1Oxbv)!ejgCOMI5D*O)hJL1P8_`I6y?6|2s!J{aM2lR&f_G87h&vBH zhl1;};#D3;zF?HPXG8^2$I;7wsy?#(3a)1heOkOShZH_R{{Yz4%$#B*i07b1CSBL& z!Y7WCdjsF4GVr0wld2ny(o@((Xmv?9ASXzKmQ6P|&Rs@@;8!tIy7 zyQ4`PJ}U(Ip=35T{Qc?Zgb@8P!5yGecfGr0faSjvKVf^NozZ7>2H^}F9Crou5Vs)LO`+kY_56UDzp^b3~q@)?~=Eb`9Or1sDbI$#VU)Rm`-jcD&#Kx`UUsG z=BKI*t8t&!>6uA9LGn-78{=Y79M0`G!qXa*kdZ+TQ*Z38%w|B_;%gGUvf=kUeWCuK zzg#lI7u13tJaiXQs8gW+BQUTXfN7qyWN=gmHF}M_8r*3Sh)9EZH zBb~c8bGEX8`T(a#(CwjWbzCBE6u&8NVE9iG1hxx#wIvN44jEYJiintE9Y9mdbF@t; zZZg;@G74dgs84Sj6rEw^v%^?4M6UN@8(gRas(ijR^3n!D@T4m~+SPS~P7XC=C!cuu z)81U86ARSOU*QrtBTG%2zB3hmX*TLz71>=QJ3)xWr?U+AIVfODX|jGFq#TNCv(Qyr zY`A?4jl(kc3}X)gL4>o#vkHjc!bX~%6zyitbSZZ3s}nQ5ivwG1PQ-3 z%9kEX0)8aUPj&5_eer-GKwTHJWxMN%`3xs&Im@pwZIlj90-PZ4OP|vn}N2>=!T|fexLQ2 zcqWVpe-=?z<>jx+EJ;XIm#IX$|Ls0_$$Yh@BidtBR8(QpuX>H8`Vu(S{=#O3@p(Dh zdDJ@UG^V_w(UW`#mbk<_;P4{7XVA*39^K|UcJ|4y@;S);&481cnGC`Ov^XBeV^q@( z)5N^anJ?L3$y^_D(}EH~S%Non!o)qyU`6D-?3IedBBR}(ot`C>Wxx~k;Koel@u=VL zwUNxw1+MQxtm6ULdUJ3+#$tyPhkl9`XV0szLuK1-kE*JxU0yz|9b)9S;MK4J)U(-? zwr!bqG(lQL4VTBcOliyE8ckFJ@*uw^l^8?=i;eueZBp8-TAjGeE1@|0z<@)8@U@8^ zIX8akV=M?=1Rlv_-}2W>DE7~oDSdo!GS(b2mYqq!Lw2K`Dw{C=pqYqDRNrrqZYl!? z1E>)Vd60%%_|Dg&pvv)Z>z%ZSr5IsjFlYYJ1v3x_iel~gjcDL-DUR#iJSxZ8O^S{T zZo4C>$lf7mZcP!9!uw(RTw@@Sr+*96 zUNBqlEqlHk4tfwKB4dWWk-Xu$PNjpOSA59n9kF4mG1OQE`+jX z@V3Z-*;p9-5&^Tx#$quuVaz|GFD@uIo-G{nzS^riDYv{x3O6zZXR1#`Vw3EGg2=2G zc&jOzye^;L>zGp8$%mk1aSjJjje>xH6fhRxZ8UVn)$%i_nH8U(C(!#N`BOY0?|Iwl zGrXV(@1QjpZBbBCQA6!L7fg415M&c1o$$3E>GX<0f#PD8ReM7`GgM9(r}`5|ytB>^ z$u=T{PWbYz!GQ@Nf&gD-G1L0EwAl5r#dHE`GJS$k`yqWr_yB=l%>-AG*3BdM-1yKu zrZ4U&xu(PQukl=)`o)Ulv|deDEp@7Xc;-{Lh#j@^62SElSM#eNN)&F3@H&r4Cw%1@ zd|ON5LJOpzjo_qaQV}Q$kSBQoXwbm2Is8fJPM4^jE{|T0jT3BIs?*VVIQ}rJrJnSo zO}lHL%KWQ<>|D(3rUGJWqfS%|jOU&f zA^c)Rly@!2Esq|#41x?|ffbr61IAYQb;5juY*t&`HT>lx>Vtb~X?^N~&S-$eNgC6e z78ISWOmU=RQC+w04A`Irkti@@U`Y8hKbC#VI)q4L;~VJw4MeBVf_Q?hzO zv_(Kz!tlfv<5syNM>OWOuX!E~)QQJUoNxClP(EkB)43kgQS4>$DxcvQ^zv?`W|Xgn zOqg7Kg+GS6c%gY5!%8LG@ubB_*Vj1(tp7L+5tsr#YR@k4?9MSDqqG`Xn+01yhoVgH z{Q2d=H5uKIk&Eg_w(}x>WGUHnog}!qRHZ7DOt^85mlB~2#vY)ozq2WMwq%&YOoYZG z3bYHDH@}ZDRV39_%v?BoG!V9(o#Wmy%P!D+et^k zYTGi?!4Wt4U(rC`Gwx<}jZ2Ib3X&#OCI-#|;}!3LD*e8t4M6_-(rW_99;w`&E3-5- zZGzK?&pN)n+bOS7bXVbEy@a^j&vV;;LlKn+0GOr7FB zaqkJ%o2l6Kshn)*jbdH>j!SG*)Ul={b4TpFp2=%e9$gc4T}QS@ko|_P&JIKSRRTja zc@sGqZ#2!%+8Xo#7hq9@dbseR@X?e~L8x#dbFthaw%=yOW8)8Xu#Gx%?k_5Hg1aF; z*8U(CFfHm1MXOAC>^~Xp{oi3P&>Qesr6@Zr<5VhaHcxip$fgme0ijM$p~Me_kPvc` zgMCVwtxu_(RqB^>=iO=pQ8Nh-zYef&x0ipW23CE)+&3}LXc~{J!~l@K8y#a)Zm;!* z%|1scn9=TcF6zYX_$@JRHXCw?bts`!bZ;QY zNszuEX5ynGG{GG`-)fJ6FEBz;6m|k~dfzk9T1zlm6+_ml(}B^rcwJ755X}_a`ZT+J z>TE+kd94Vgn~^kN#?vGDsJPopEkVU_aIG_DeUC(SLrL31LW_79?>rD6*W(m2;; zl$KnIJGqrjU0~Zcp~lCcXlfq*2x2mr5vtg))5(%mKbgq={*X3 z9$#4GUBpUyd%!m6H_DE=&7uHutv=^rln_FHz_9n*F5Tek5iX0ZP9t`d9SaJ|9KZM1 z7{5T72+`HJkBOca_NOQ{YtxU2nHA^8{34iMTt~UT@rn)uz4+kN{(v7R-?~8K>=<#n zz#*End_vwN+gxWeKI>uHM*;y)mL$TYGUHCtlsR|8#nk%|_a!EZo({Ycyql?%sr_b| z=n%V%##Ec%E$A#ZTl_OM2$lcYqo4)QfIvsk>#?TpRPaRn1k<8XX@6IFbvJp$QBIJ^ zS)Xrd#;G|g+vm4Ws9s{sJEc407k}HHRKr&`^(7b_Av7aYva3f1tzH{fGdJ@yQ>R}2egk&;p&Xo|e`3x%yCwr{?uyJVV$f`NKos6`rIzl%j+0!-VV6LU?5^12pO0By( z&M&TSxOuUJ!TM)J$tHo-;f;kFvU%@E9C(=Q2G1b=Diu)r7;!m~7q3426)!NQCJFJu zU~4-k2?Gj3%CvjmYHiM7R!3ABAfpQr>v}bbHU+=GCyCnDk%K#4?!jnl23L^gD^MdOt{0rdS{YyBalSe?tPvB|B{ z!LHi5t4DDEgl|o-1DS)!m)2;NG()VV(PIO#h_dTSg)$x3!&X>zn@&#pJDqjRef zfe~$iyj0ep1g&*KHu*)Ujf|EnMbV~cDIrunAuff{r?pi`O`s*Kn?==6HLtW_*VcWw zOhkJwB$tZ?j>Ra)s2%R)g-c8tTG;Bq$t75eJv4tTfFl8?G?Yg6|3>_I(emzlXoK$8 z6BB(dl!9t9T|z>Rw(u= zQ|39FbmoQ0uD`Y$bsZqAq`xXg(>j>`PNOsN9bIB+IhWQ({{)HnwH*v`iU&@|j>|P^ z$Aix&<4T{!*O`fMhVQY*Iq2XNF2InK zqBnDS`q3QJc>4kc8dl#W)DsmD?QK;Ummf$lG!m1oC)+g+{_3q)pf4?S2+bHK*E$L- zjODI%bADfq=|u?A_x70?-0J6Gmt2njd3HbLX7uMtqLLxPMwdw~3||CTdrrC}uPp|2 z6cUz9#3(!H&CguKI=#gBF}R;&Sn@dA^h?Qky6TgDI?+|Uk9qV3U%#AS7DTJF?>aOU zDZP?SOgcxDCnu);f7$sL@|9uxQkH>UeMQ~Z5zF6u`zqbujZ>(Gn=NZSDy}U__~^Nc zw^s0?*a+Q{HAZ8sGXN2@W_Erq08hR{P2Oc>B^N4|&!J~wf;+1Ncn=w{n>%iV~!=F%aA4=)DCcgE9&$bX1*uD(#abY+^ZR#wSDKSws<16^QL(`Eh2 z>1l7o{`VoF4FU<%7U>+{{95^Ko-TZVj0I`*`YK1p?VNhNA-y-&(F40Qoq1aH++rE; z(-xQs6SXz)W0$MNx&`mAAU<~s|qGZDT)}b5N(%8@IIz zFvDgn&_Umu{h3lA)+_JN`@n$v!XN~JJPM)&MVnifLOX2Y;VT z@P;D(h;Ta>Xj=kRB)w>K1b{Go-OF2CmZ%HxA9 z?pJ*Qs>(U_UPW$cJfvrvngNO5ot34PcDE`hPizJA^G&;M0dP4k~G>9 zU_j2sKkq$Fsw1e?b;G~BdTg>v@kd^v@^IJ@kj6}O>$J&y)AZGlxHbz{YYIFd9f_wdmynrYanWjxB`+k%2 z!&F4DIxD9)yN7lvJ6j=h>AlkG6a7TRiMzG(mtTfhS% zE!cagul0PGZhO|_kp1In?)_h%PriTlDE}ScJ?3bX74JFW8(kLw*5t%`B@eMGdKyrf zk_JeB!dyTx3fEGOSF?B5XYtVuQ+OxSa=L`R6a>(z4dLZ<3}`$E_Go*LGk^ra&T6YF zS6_%~YzdJ$!{5v0P7@Jx;LUXY2qYmbBX67)pD^h&+K7l)PuN#6Zto-w{pcI;tK7}Y zdtyTEa=&_>B6lyE?ug^PvO|~ykNw6SuzSj0#eO#dNkpo659IcVoX|SNx|bUJ$+{Po zIWAo~$>Ja@jT}w6XV->qzjoJ#UNG)5LW}{jN?1zz;amRgHKDLK__-*S9t0KnUkVGa zeH0u^*t0>Gib?k`3wk96_>&itw#qy>)=1!@l8heK+l#uNHd z&a2h)S!GpilTmeYHx2&BFBm~{kALFb%}5|Z5~h1%fawg7EnLZPg)fFDmv_pQ%AI%8 zz1o7%yTeMNnN|`=s=?M$X)0tw`lJY7Kn61Wc_&WvubI4qt&d&sXzK%Z%-&+-O)eKF zIsPP0WF+Lrz4#!Fl9j&if8@Z@woC?$Ak2<;CR(8SUjO90?`DZcu$P5akP~-PSPhpY z-Da@Zvjn$CVmW?A+Z*>1iQ5%n(EHSke&84c;AdIjK4PH!Nhc_!O7VoTHT1&robOY| zqnS34!&TiERc4Qjj}_b#ai&Hihy>IJWG#;1TOP4EoCQ%9SnH3-T(E_td0d@5DQ+@< z!D&xnXCcZMo+9|+b<8VL8GQ8T6at89k%WyM8{&R@7EtBE-!>L9KLB0+a{p*v#5A^o zS6*+o{3j{~>YJ?irFCaNG=w?(LaB^&J3}0w^&Wy1ACh=3t?~CilbdHG7dM})nPsV2 zE0!@WXJUafrhwp{NhRC;wZBiG2KFA?t0*it!G-&)83QUlfaB_;Y999u^5zl` zmPNWlkb;?BbAI+m?5Yem=vGs;}0JnAQGZlZ0^x*h3;{*1AXHs9V# z8DXkxWj2&S@zYL2Rn^~i4H2>NN8HC3Cx0T!z zkf8t_=YW8tBWBA_E#a8bUr^Fpa@jUlnp<0~E;KtflUSXfAk(ra_2+tt1 zCCEAT%cPg@VWO)eZ~d0ZwWrN)M4EPRM)@4k7#*asgZyp(J^~D+h!1*>*iGJ6xQKc; z!ZL#@eL!pTp};4_)>01E3Dv9#IF0F9U((B=7Bm+9LMuUmTVe$~0~ZR;-l{YHtsX^? z{=sRYR9MM`!cxPT;j)UEy_O`PhJxLKKp98*TM>-|6{Hbu_|TM_-Vq?h2+`m5VgAqm!36k9Gs(V_b_jtsD_w9G@$>-flk=Xt|P2LF@$R2SONTo4pRt_5*}V zPHC{O5WRkhuaH%g^{|q*yv7XWVGB7V3=s+-|Gk?(4SB@45&K=bg_~&!X2YN15pL_F zW3w>U2WCX3ZYiVvKDcWUOQWgj_JLIX&Tr3Isz+aSi-17@Hq+V01GhKy#AJFRpc~o z&qN~OIBYuFa%{ZYo<`Nk;UZeqP-8ewjDIk+wC!NEF>SY{Oh6n8J4Y}iFnI*TZlg>+ z0VpV5=abZLY;5jDoanyLH}O$6P~0MamGpGVQ7?ok?%qVGs!3z`mg0Ox_5|q0>_bfm zzL(3C;{A38px#3W+~u(L4&*jCI70>1=3fZcft`u%3?dOKP;-FamsSq;{OtE25WBGU zQGttpPFyTr;frFbT+F32yrD&{!Pgz#7|KB=z#F*YCLbV*Js>UNv6Yv+o#Fx-)Szz8 z>yVqp<&R4BM-d$*(#xwW&h~0-{s&fvA5cl`IzDs>w*b1MGYw;#knmMeQHi3EkpwGi zhamz=z7~bH$b;i<&d*7NuWlXV8yMl!`#kAR558&mLGsyH?5%0uu=~D`b~kFOsi`R_ zx-R@Y;A_si8Km|@@y9L)&LMr5gN#|o#^%~uZp{ae<(*x6OxxDGWp~j|Q6WQ@6X&S1 z1%E-c1rI2q9Cbl7zfBoP@E($sIXy8^5I)H{rVq@2B#gP4gH3pg6p%-?MNoV9a$ zFOD>%iXY<|f0mq72ZZxaYD)igSS<;K?O(ec&6kZszJt-fCGmc>#slig^7>HQR2R|YU z5y*Q-p}D)hyL(-JEf!IT6(|hzx*iV{A2ZY4u~dpuG7a8HI&P?}Sw(-2^}MQ-aJzqf z+bo2v?Q8F>-k0^wJoU53Tp5#bnOcaQ2`CDSHuoPq5TBTR8e_Jhxsf{KI=DS&KTj!b z(TF=Nk)hy}f136Wx{G*DM2`<}M|baG{`HuDMY@itZ+MiT=iEjuh`j|qWeBm~eI^r#)AK5VgKh%5-U%k{BM2$MD|BM1GJ@prkKd|)QD(yYSfVXqKkVk zLbW*_K|~nY znHJ42z(nk@;;?b#Z_K6l+l#R?uuz;pnpWb&vp-)bRPfaG=eXA#wA38)F&Mn+aQZzP zFv|PiH930!LVt;<&h{DQDBr4sJk;P*^Pi&xz5MEh^P478!$;vFFWh1D=nOV*Hy@k3 zJ5tS^oj6&~-~0@y3Nrcp!TO*wrbj4#zS9}+u4mu3mssi8R??~Y57F#FqtN6a$7#~- zItx$2I(Ujp^STd&n7cs>4MV#sELJ}1+0%J$ zgPvuI64IolDDNF`qD(;}b*Av&^{zpEnD(?ID2$fH_18PlTQsJIFWYEA_(|8$fl5PD z;}_ZRPPU2ZyRRjaP84g358zwmtPI#xYhfK7L^O_%jhF0*ut|x+(Iyqz!V{dx*Yv^( z4cGBR<8s+qK3%oS1=aop82eLqdy0dIWvYu9^H`lHVzZGv11(lu{<6PKU-a<+$U5Uq z$X~#^@AMdRYeSR|bk0;vzRliyLGk|U``8obiRMZKgz^>c*ji46^aJT#o@JZc%^7Ld z%dGNsw-Cknm;Q1?R$r_5KRf8#fsQ87-%^&u%_BgNZ}^h+a8AV*qV?}t03QS8gLj5! zl(hMnmeFzcaKK*CfexAc4QGg{K*N-}Hp}lWsb-z^wcUUmA)V=!rjCH?!6;N{SY?c&N>o4kGy++Qx_9D#22RoE!PK=-_1?LzpW4+y~MC{ zvEcj+TKeJyURPG~g{rfb2)V4{8l48ahjE5Y)!L@By+zQ|Nsw!(8tftcpk0j`*pFsd z8sZc^u#b4eGyWLlHwcIFOH6kOS*rMNrnWZYW|xu{Gg+8fw0F0j5nRm81z_7r*zdOm zC)j_XLI?db%s5#(*Q@hbv$uczuY-=zS1mZxVt~y zUFz4dUUKu(6>p?$IY5RlbeXC!V&N*iDZ-bGSY^(DwTS!5C??Y$;6wc`$Lw(U9lLmR|7vu*!uSYGC+2^XH2&E3lGzY zV31ATtKkn9engCSE}FfE&adA;c**{;hf`7+HC1Xy4INv$Nyz#`qxBXpuNbg={AubD1g5^nplo%a8_Hf zX&TowV~CRadRJLIH_x>^wHs&1@x z0hbQ6hG-wrp76c?Fes%zv7fY&tRe>69}C6+4TyKcel2?P2(ayg~3OK=81snyJv4E6$<6~b$)!$ zY5kzGXaVs!;=EWboO@-bC?We@PV;RtC#|1K65d}c->68x6vGH^n?+e|xWAof)qp#=?w3QSCsIh zN4Wme6&mJlCPeU9gRN>qw;mzOeICh=jCA|d{s&WHhN`H31SQIo4!@l69X~MssY^tW z6h@r%wBhdag!Fbn+wftOirC+Of`}S9% z*9E=WjVY(#=`N;<6Duk;h6vkPdscgqXx+-%fe>T&`Fu(Dpt_?$ziw?FFZ|aQ=y&K&9xO5F;ljgp0sNA6ah3RzjYlkUFy#m{pU_Sloj9pc8B7UD9pV`Z@e$gIL zbik1fmy#+q64LeuZKeI1TmXmpdtU7tlfR1J^S?+hK~S5&@8SPorw^^Hk%nBqw zhn`Xna!MDZMQES#Foi5d?{ATx;H1#G?-2{29l?e@6+l7(kiW2Sja;0S zTk@L)%)Tv+xxP8G)NR^1J+saaEnPU-({XIl2Fck({C}v)h5)k%WX=8mqH6y}Zd(Y< z%u;3RZ;tXeY6LZVVD+1!DgW{eZX<3>@a496XZYW+;P1cx1E~AIZAh$lP743P0OSyS z+;iTmS2fIE!JZd{y>8ul^rR{{Q@8ffp>+v+gAJ|I8Y| zy}^5q?zr&&=WW&yr88o)!T-z}7U007|8JV)jn{DrR3Lm)gN(KEP5GN%X#c=~dB7-( z-N$ZEfww^2{+AeZCq|tevb^!1U$Q`JM7?_p|NpQ5Ls0+k z%fwp{>gBe}{HOC+;li=8p$_Ld_TcPTVSRVEIj?C&9lZ991?nS{4yU5u&;M_2Vc~)9 zN9|Vuw(zv0b%;QIV{Yv= zpJ1->kPa9mUgsy9nL=TBI`|9mezGpA1xt6lk}v-%DGOAT0*e^yIqnGU_wSVfzkgO+ zDM{+(30%E6W<3iwPf=7LJ1`@Mf71I z4A*609$Mh?HhWlNQ&Kp|B0QP^m+%3;CY3Y0TxVPLVBc7Vv9C`%euefk`DnV z;i0drB=BsX-Y2PxFe9jMbn6){_c-{&D_VHqLMVLa(MeV8i-SGAKBN5K_9h!9yAB>z zDj~BE{)*ds72DhYu+ytafjVc6Q%Fq)qpZ79iEC+O$0}&G?mmj>LQ7+5&Y!UtD4W%; zjr|djjZ~tY$G6uOh%NLx@aH#WC3VSvl+u~=EZC-X3dJtNhia2-+aI4oq#xrAb&P+F zXTPRxE6#T$KBLBVCX;E|v9VnKO&*DLrXDRW?w=_l)9sSnJ(||LHWNNh+?gbN z2yvVud`Kwxk2T7O5Z2bd+k7&A{~_sP(M)Wm^MJEwU>c~fz7X`h@|N(OI*0pb&TgzU z;I(PJg?^Bg)!XG5riTvKx9ksU*gBkiI$c&Neh|~$iJItlN7BqhpGmviRIsyCJZb!q zIw=8!ru6R8SGUKARn&w1!8$cK?3h;Q+ZMKzv9}mrk{bPNV-6XcOq`YBjVLVNIw}BGu9+J(`{&7ZfOZjZpwyoh6 zw%)-s8K!6VUnR~65Z1mza#i2Z%BHi2ZE77Y{HhK0ev#3{lTNr{`a684L@%{4ZWoQF zR1G;X*%+Z3ZBf|kJ3E=&^JmiQTF;&c(tV2x`08N{EAkgM6eWqnTlt=(diW&Gp+v3y zwMk-c;d@vs+h3!u<=-|P)|gk-6fOQaj9Q(0qFx6XYuDNp#wR8)38lc7RFKI!v!mgE zMp|v#O$_nXnVZOGz0Z{Z)e=w-|uVLqVLDZ)IEuX z%Hpyz^N)@%u)#Jm?L54i1J?-U^r^C#`3L#Iz#RCE*tic!%<&-ELL}TXiW+bM9quj(oTf*yzi2i zd5&7r(-mU<)_uLUac8tsyXr1fJ8;c)ha)E^{1cL0?o=kqpobJq>EN>)0oU8bJ*Tkd zVo0yq=7Bqu%(Zx4h3^xA@7Kl1g)T5$TyZr>|C-* z_mnLdOf8vsC&CnqhuNWI+mp;q66wSg;H`kIm-l<;6v& zig!7fwvTq-!cvHQKBM1Pp|me4R6|WQ6WG++b^!2pHU5FS_U+tMKD7CNbIYqc6$q0h z6D=tr9#R#1*{-dG22CBVf18^1CpN0edQK(oPakjNm*sKZwN#032uTQ&K|lULs&cpx z36_zkBGbKTxoSdK$pow?J%Tc6q$T!;VZ=j6IP}Lq3=aMmQ*RknN7Hl-;~XHk`@vlU z!QI{6gS)%Cy95aCaIhc=?(RW@OK^9W?_7EAcdhUDtm&SfuHIF(tGddDbnU6?pTt%` z1xFyo-%h=pgbi_~^zr0@==a0;3I$=NzF~nEtl`FYqOZZmBX6j|hP;OG?+fh`g_RO# zTES15MX8gL_*d?Uw;0%=P=bW#!hI@T4O;zB;^nu(h;1_2sq1VqVb7ue;f05=asPg& zI!vxD{jh5@^sAo?yl?@iG4;ApN$t48OP3=s8~XMSPumE1QTtg^qyH8q1k*5uJ)2<< zPs>b~?)R#Fu!2ey&=1fTFA%0AzjSgf)pO7v&8z@`4=;W8r*p)AGyN3a>IAU|P}k|^8ITbEd(#IXoeKii3NSmQ zafA2S=kiidQkZ;Zo_41dXLn|7uhH6!TxhHE?5FW{)ldBG_ka4tSOVsY!&5e5%$SuK z!HIqi;4{}*&wm3&9b~5vDaGQ-du}CL{W_61Z07sL&;CCNOv;7MJqd_WlBFK=A$W>|s$w+6xOjPY?@NB{B?sfon}| z5ubyO*n73B##u=l=%?ix+rLThAY*~RzSdd3i`rrmm%=cWBOL2FbiztdWl2sWjrd{l z=P;r>ilV<(uT4By-?_)oBm4b-vO>j7xeqohr*W_^_SLw$W{CHcf~(B<)%??U;Bo&! zs>8ZR(kzX<`$)aE@0UOsT!Q1>+n(*y9m$K>f?Zk%hx4YLmDSyQ;Sc-w9e+}-UXPGyaa0~mmtySwWCk5DW^Ohwc+SK`(d?%o1o zuP8fj_pQ;twV(e(FIMmdy2M)DEc%m&YPN}CXvI0Mt|VIBA$ukZ7SHVAl6GGuij1D< zeRySj@xNz|&tr{u2-spPd{UrF+Nw^KvKs&PQR=_Cq-H6__cpDRPl%!@SPMqy>0B!G zsdNWyUM=d{<9svfy5oF`bFzN|K~3gV_#!_$rqHvIP)}FGVJB^iBYJrKPTKoxB``R zH07HY(r;({mr}d)>s&vr^h%BHF7(nQwNL*i@;$=;iH_6EF%EpL&cXB5_m3paMZ$|W zMWLN4V>pD~&doP`dkmOg4IL$<-jn>^nI4YMA&9Q*qpQPOCBG*N(wElBNZ=%C7mh-g z3@JZ)jI%qvIBVXvgwouPE0S1SWEd_MehJPf#Xxj2G4=rGnfp3$vA;{l*0lr+7o~^X z>r1?yT)dVw$aUojw&iz37(Zu}ny<3j|K;_`pu;G(@<3 zRzd__!?QP61`4B1>II}fwznC5CqA=py6@)-gMDV@P@WFOMWYc(U@ae#6c~&Pa2kW4q*s06mv$uN}&jC7k|AR1r z_13F`V=T0#O$cfwB$X6)b>VRS0AHC2nnbF1Uhd(3plfVVBv*MH# z2~+HX`HwNA)%9tl*IzyY40UUU6>=47CgYp#K&aN=fQ_5jIBWAiQq5jyP|nF%97$K2 zuRha_OUn|AStgJcuxz|=X{2HQ<_f%x<7c@UF!8ysbX1l81IXq!CAo8ZnECC`cQ~_A zbNe_qEjP}pDDGR&agwSPM|d5<$+bZ-$}r`MeX^z(ehO*!#ru|9=fNoOl_!^luEd?R zlWf#^<Di~na?bvh=&TnvF*_^sSr5=U-gO!`P@nuF=2`L-z%d|;iJE0 zBR4a{^Fz_ziTn*}7ikkVyXCc3+fZND#C76qH)9>o%lH!ou zKBe&&RA0wh`#Uqh=0u>;TP$ToBEr|Bzb7=HG@Sk4j@^MQozH!BM@xaebr*%ems^## z^8~5G<5iMV+vf)*LE#rqyAt27o6kCd4`tH`y%&M+2TOE7TJb#3H-pV0(v3i5d!q*x zQtva{o*-m#g_g7Ji@9rChjdG>`GuK>b6th}&Kq`t+ugcwPOtd2M5?QcP5*GpaYXO8 zI^g~S_TKHS(FP7Bmq^|cIpCQz;1=+9gR1n~wa%Z6sfXLi?t>W-1-abl?QWfn6v~VO z<<_Wfs#$8l)nJfU3qG66E&Xo?@1HPd&wmCV9%3+(!sP8@yO9jiW)!VGbLer(a$&ty z1t=rZ-~TO=_pHHG+u`E;yl9p5NrcpSP=*abrfEN-y|S7%Uy*cqQ)^nPse8Z7_j$wA zbS!7^+TUIy+z%NgHmQz}%a59ZWGmiC82ENwcQM0+dQY=+Ee`!2itV|)@ar}gACE|O zyAP2NSY14Kz;dxlN?sd(!etou2O_1-_U}QH*Cf|-6ocXrcWwI(df`WP0cPaG{eokg zGRw%NBZq{(uX`=FTRYgs>fv4LYV}A%{@KAhChsite+z?yJk;Ox4muJo5%8Do!ob1J z){FY@C~T*zmA@bjj#=^oQxiznn3^iv%`@J#!GjrdP< zmIbpu+YFuC@rYonkm(M`69FAZLDVM*B{zRYIoRJ4y*xu{KNoRi^2#R+#UI6jQb7v(qt2RX5*83&okz59eq6%RF;G>ZO8jg^%lf-bTNFV2_jJUxF=UgYINIT>K6;pGRDIFUtp#r3S#h#D8waD4JCouNL~W#v#Fvu$pF<)&9-mdA8KB z;zgLs_D|qGnj3ecN2UmYGS;Tx3c??cJN4lnqUAt+?2H+79taezXV{bwB^d88F$~qg zBKd4KqpsSwOZTU$SNM}Nu=Y$#3HsgFW-tHLpGZ<>LYgzK3>6Y$ zen?IAf_vYq>t(zoX{TEf7Sq!0Ora)@S3mt?hAi7}6jQIa+2x!Z=M6fY8)FbHT!?w< zI?GtV+7hZaH()Sbc;^0dU3G`QfAt?0mZ2x{E`8ME{##KouNJs=Db~J)N;r`0Z`p*V z2l5l5?LP75Gg;@=FUv!&&336F4T&ZwBcVWiY6Lb_y=;Ob-|lxgBQrQ|zS6+=u1}t@ zCnrtba+Rg6G8kH3kDE@@AIeF<_Mx<+n{U5g>iCsD9{ZK7ddF-_C4MylwIa$?M<`ba zKRfg0@;7}m#Mth^se6ZWkD4$d6!g{`sN&qW>Cg9_d)q7(dVA#ys{k<6b{fRd#7&V1 z(SBX%QUuh6?;`8cXZ}s;aBXtCl9!eFqF`E0#ZTnVC~rvY57+tuwvo2x0e0)oK`t(_ z=G?U0C){FBKC8O;Y28Xl43zaTC^+LA@4+H(#zBo()j7cWsLdE@C|MarQYlZro5)!o z&}sh(X1BMBwyjZ;;gd93EL7s#OZNlEq@G8*pW{RzsJ2@T!c(@IS(KG!hVI6u#M9vcm4;kawkd-R?f$aCWSpE+BM zHZsmq7}#}PIMZ4NXbrm2O?9LAL<@0`u4g_xsHM$b$a#iN$;IoiYo4vyIcM&{%*f>% zt6dk2`?v8p-*m!PoK*OcaR4My!Qih@GmB#et-X_g2TTceNhmek_BYZpph_VYAEiag zu+d^V+cnQETP*YOdFA`1RuEX%pR4;k^u~$|b~Vbjhnm61BM7~o0V;ly>9!nqgA&zI zsPKizajENa5FfIcTv^9ZR-)DN=-s!rcwS|QXSvc{evL&@wddjgh(naDH5METXYHS2!BoghO_&`ZItkjHF8A{n!3|>K#!Lv zl#3X^<9zVZ&qsISfI-727dpYB8j_lGNsbGO{Rt+es%oBgfB)wo5225MEv7rdhZtq4 z;*n_~<-f&y+iXx}WXegd8TvlrF`M7Dj55L}ns8b(vj2OrYm^e_QK@_UGpf(EI@G z5g+3%wjOWweFtNG(0r>!e$iu#wC<1jdP*Ewg8+9%FhRz<=;!PRR&G5A;H5EQ@-7Kr zeFwVLLcT#?@{tkon@JiA!yrmXSCb?M?RX@=yVrAfLu>BlqC|q}?8xuz=d55|!i@YKV!-?@lO?h_njNk3$JRV?jHOOIa@F# zvak|&?e!m^7NS`lS(WjMsGjKBRCev{fV?)TGK5qJDh!5P-8~s+8)Ov->Fz9xqZyF9-R;fRf1-c}!`sgHlJRV-XY>4%uZ=fq zK(;i$yj%ZT2_TbJM7|hpL>UyQ3ABq*Qvc$FxhW;fc&to{ze=&BhLZ+@$Mrp=J*Z20|xYI&a|Q( z?E@R8oK}Nj$$~a0-X-?7gIW+BEp5*a`2J2Vd@wFiiw$G{_cv6)Qf7BVQKfwkZ(?r+ zx@as5$&7f8_+4?S-j^b@(cap!?B=Csn!;cX5LF06}RXIkKQ*iK3O+-{gF|e~%byDrU+;N+KR3Qo76EYPk3w zhzcjF`3qfh<~Jf--C>=uJLP{OR{8?dA;NW4wZB-Sr;}t&Igq zMX90S>b}_^1Yihxp{V2OQwh#VNFZ93Qg`oH(P{c@!J(<{lGmmCOYTrVHGGQmPh3U| zJw<^5m@RBpJJfeH=;zfsQ*7*|Fuk^6RrezYkNO7*((8p`{0#_-85R;%?Ttnpe>Dl% zPQ%Ds{B?S9ykk>$PT*%y>FWO?SLlk2&a0(pYt8E9SENR!tfsO7a_zjKN3_TVsZG8a zhnipW;<0z+gOC1ch_J6#J7xK75n@qQvC^H(L(1ZOMUWK%CXeE=x&pF{`M2I9gWES$ zE>$_uOsisciD%g6f6OZyxQJ6x-x_1KkR!W!3C(9(Fu@x%%CHFlt88N~m@R2o7eu)# zQO)lw1m=NB%-D+RGvzzj1RAzJFRKJ$>%NJzS&gk})NZvTMAHk4mp6Ar5Wx;)kYnTg z3cpphn=D|#*0S-K-J%3_-Ivg8mH_}a7gSAdjXz9tTdS-C1D)Di$14?&0*YL+IJVis z@VZoE5H?co$+r^^#9Uj9C-@&z8jYBf(4UclcRK8j&+W?j96Lo0^%q#6*=_6fmubQK zMF5MNKLJMg2p|}I7NyipRB9ZmiuUiRb+E;;cR2*ei4=Z6tMSKPo!ZZ!A0gR$*Z8ZUp5oJsJO}zf{C5rux@-@@G+VZE2wGKI^cDj zq9pj7{eO&IFc{~%kpDwlV;HQu!0?iaZf|_?0j+u-11T4_{jBExFk~ zw+AWa3erUSoc1t39TRU!LaLl&;q8gI51y&#F4fY}u5}vFvQK`WvizxHsrO4PVUp%g zJg=T_Me#zwo9BZIF^*7)EV9fk>mkWCvOBKam3X{HZdE3xr;>BNS=7s}JcIks2W5SE zN^q>9i~e~iSD6%wSwUcj`Cl|EE{Ft5AS9XGp=!STOLS?Vi24=Dc4db^M=3To4ECaW zrOMsqf^G2TF~!X9-!fST*Ofm&3CHcWhoV|c%@0i}?XmDr?Qm+DdRR}bHm1={jpd>A z?XneZjq+v;uI1v>pRV1182T9aS2H~cg3bWm?awM9QrT4BIa_MV&f&)Fg=C65%aS9< zw`3Tz+X;{KU)W+E(6V;cBtI<&s7T=~crMJ)PG9X(3|U;vuK5Qe@=jswa7pi67@^}Sdqw%h<5&0MM)YNeEwG3MBiAM5LDR6G)1Rn97yX8amP zsP8Il`*7A^6`1n;mEM}R?B>jTetCgTET9ywOjTZ)DEvhtDfY2EJVe>jl0i{LbI|(Q z>8@b(YL1qZKbMH?{9O1EcL7`3*6=fvf)O*O(I>Zr<_Gr&H?`l<^4AH6g>mR~Vun-A zE%Asp*H)6x^JG-KramdwdQ;T&4x=S|bRdmnYX{i}FJ9Mo%3Pd4y&I6V>>5W5k}RG} zhu;hjy74&3b&4->WfVO~3>#omr_ub-++3_-QI-Pv5iS8fkgsHF>6%gLOy$IXGUjn* zW@DU;9cmr0u<^ux1Uq9f06>XDj0;I=IoM&?1K1C85mEi$1^hLmK@UJj)kr_Dvl2rIdlP zgwUNkA~|KoZ*xqf(%yM`YU7Qcwke9~41)q}QPrWfs%RX5v{IoD?v3CcD%rp#c zdOJ;ECL|#=k-{r>us6VxvL1G-KAVl0TUB2X(!|?-(A{d>ucI&q85Z7Lbk$*p(h^eL z*M1M!*(=Fn40YvwYUYn|BheY;OI)Anb3*0UEdjP%iUvTs1 z!8fULHkd8-@IA5cYEe0ai$E%Q?COA+Pd)4DZ){9#Z`ztn;f?8!p+p76pwJM%kWzDO z9L=pWM$Lkd6@Ff30G1g|ai1zIU3LY}S*^=-#pH2JK=16wQ)ObQI~Gb>fBI+9K);S# zfC10GZm24QX%-307dkhrN$E@h{TmM+luszw7r6PO7_y%fisk&H7xygd@f%$ zvJrMPHk~@cZ?LK_VXbCXH};U+hUxI2a0hyRWtsd1v8K6^U%bj)(QcXL%j6#n`z{;` zus_4JBZmirZH6s3=hP6QiLJTd^ntFb1n7&&tU6Wagh`?%3JJLJGxg3VDyXbc8EOds z1R)d?g;4+tPib06hn-=eeGWF)!WUgAClz!HtcmMFrlT87T{>*$8EL}e(x#l}FfEYf z7KQiAhNcT@mI*6QMJwZk3*wyXwW_J*WAkO;5xX-8wE3$meiYgO>Fi44o4su{YY zKw|Pwxc(M|vh!YeC%OS>#6WOZ#AsmGu|ov`?(lao>4`y?RG$>SHR?RsaT+8+)2=LB zN2t=l2T2enXg->V!@yZG0k45IEiA#qVVR@-J}tJe44irf!Oo^Hu+OKvQcc|Mts-Vv z?#43dQ+nX?0vVS(erd&}kuWlX60vF-sbm=88@l&T-Ab}RlAkGJYF4VJ>Qm>I`IYs{ z8d)7HpKQ94K2ty-K)6)_p*3(hL1(#V?2r)Em9v$y+FC!(;&24@)(O3gX)}Ik#qRpi zx&*as!N|UD>V0f zN2z=;%S$t zk&hwshuG&zZq26{We)U^lKYeefc50p+>E<(3@Tn>XM@7|`o6aK`JYM9Jw;Kpkf(nn zRg0kH7;;H)WzwAF`^lU4K1C_03+5vb5?;w zgUmBkw+uvmY=|ldX+Qe#AnfZ2_DIL{uR+#e_l^^pt9B4@D+ZB)e5h`sGsuu}I8YZu zS&pbP2x`>3e-5@N<&sv4seq`j@7tD*MuCuGpyCZmXFhY9?8AsZgaq2ps47L}F{aXU zuL(2Gn@H8SeXV%xdb-_>l$3$5f4*6+OxzV)jq~cKK-juzBD-N|m2Kj;Z|*-l+x}c% z=ak4n6o*~7HYB0o_6KVar>&E7;Zv%Xf&vZdkS2iEM*~gc5E@-?VPQezn9}4&Oj00) zHk5HOLNE%NrkJ)i8x3tHyaY;t`LvfB5YEs^R`k-75nd@OYxkCCmw52FYT{QK%42vk zn>fd>eWEjlKVn*dGur~Kq}loUx|y_)o<0$q^AXtoyt*<>c7|zbx=(%9y@;_DWcozl ztCw{NLXrGX3=~17mXuHsbbUtY^|j=!Nr=ZGmNyzhEQMLd=2p!_%mf1kcSbMzVyx&T2$zQ%W+EnCcv>Z2sTT%I&%+pg)ds+=u8lH@ zRW?pgy-e~D9oQK3OlMPczZ}@~a7c|avXj@Q(`GCmo-TTm@C}dPkdDA`#i|A_LFo+h zlVVZgxR8e}fK3ypG%_clyZ@*ectE$iAdIl4h_Z=JMT;`?ulBZfz?4QZqzZZHxEldF zn)N*U_nShiT=f1c16XI+;6YUPm8efLy2QQEE4(9ki}ZcLy+3FgcjD^heMi%> zN8&j3Ykm85m!@=V5>9yW#sHP@Nc4iFLrF_`+a*!b7?Bb|AL)Gvz!xooLybXfO@+g>7GZoVJTzo21F#1 z9Re3Wjh6||TX!HR2?nSNU4oa1%B?pDH%xd+XP>_T`ddZ3PVA;Ra2V#<`W-|VA5ElG ziGIXri*M{hq;OKj2U8*=z(UfpWTs!!?)%4{GB?P=BVd*ss_=7}uq5obeJ=0^NYd!2 zK3|H`KIk3Sq&aEC#7w$@jrFqC<3F{TOB2N6=ns=wtxIa^S_d)CrJ96;q(DH<>4m^R zHe!5lxK=2cc5~Q0RfRK>T6}c_lNzUnk>G=tC=At@UK%W-Hf4pEjShJIV5s z_0(=Po2NP=*BXaZIQzfbfL&aiadT@Bk!C>1^muYgsH!3+L*K!qbk5&;v$cZgxfvNS z6~d9l5AIEuSzYy^nDnesNEZPOn<0SxMn6LDW(8mvde-PaK!4!Bt)6KP(GIE)X=zX! zo5AC4Siq*}haQ6;-bfyWpQT`rAgSI^sdfT7_ns!-$T#qc3f2qW$ca8{TbU5I_MktU zgshWS8@8vR*JBnxTc(bJo)4mYtstPZ;rg%f-I5>4LoBd;Sh?|o06#wUK%mw5A?`@f zKWhw_gfBqjJ^vta{$wWm{q=pntCVhA4TGgXpC!Wj=t&==k?G|nU_VT_=78f=W$fU*BY<+wXHeNph5?P$zntz<*gp4is*RbH=GJ4$sLjm=s zd}DvEMRdoUs2r#hhI1;l5lhia>9MHlqPK8zs9+`5@aHh(d5OqB@YcwTBf#4Fz5qHW z$l$#wnByG5pB+`MhAmn?fF?H%Oql!GHhKU<3KVdv%B>c|Xd?=-CxcI5=JXbaERxa$ z8$~NBG1gmjO(UV(CJQ0B`J@Gq9A<75)(!<}av=Km$3n;gcJWaEI1PkA^{Kdmaa<)NQ1#>$lQ$556CsjiCEYp7UbI)HT`u~L3 z+2-dzXIdaymhR4AnS#b|uKYXiCl1)(Pj=GwsGAFiWT}t`n~cOMov_jQENG@uhcGzr z=kQ+!b{Ft14Z=-_Qg8ui93b-@R02&jH8k}kQZ!8~EuPSHUTV~va7q+lbxglTkY|Cm zi(JyU!_Yvk`f$|zDiVJWF^d3sbF{Q)ta@$7nDK_6HCXo)7~706mI_waMLn9x?n0I# zS2x&=Jj?Q{D-``8=lHIP_*LP8VU42ufVCEZEp^$dC}h-p5`;+ItJ4nQsDTDV(+u+y z!C^oHCTVD=O9JR1SQaV{&U;h|h-!=tBqFjCYEhn?lIe58gmptZLKo(AkZ)m7UBO0K zEOyk5s0TX3H?u3&;X!_wrXu$R2{K(u5@B-05s>`vWJ~b`Ds9IrU<{gj8P?kO`q!_m z#4gOH4W3yp0(uJ~+^l8?qVya6RPZIv=8Cg^in^vC1iW2dJ8xnbybXJAnEED|BE&vj zdTBH_(1;X^IRtqIO$kKxdL#e&e$$_C+p=hy+7P}zKD(%xIRt2A5Z1V@+BlHf^}@P8 zpdgtZ1~Tce&T&naW4WDebsO8h(xdyFzZE-X5#1x&+fA>NQ)n_O>Baw|#%)qt)fWSO zuAL2g*T4lFPIw+%6g+3}W@B*Ws`k&hqu%^P26MnRV)eOas-ZHFmtsUzQS`G|4GtRq zp0Gs1#s=ZxRbX5GScA=|zP67Z+*0<_U7;emsY(s7M#}7Tm9^VRM)wC|@1P673XIdS zpE6F#Xo4d<=wLJ6|8o{4o(URBGO*rkqZ})mw6kY%IhTS%R}cyXgmeP*CCM|>xxf9T zTQ%teGlGzAOhy<=W%YO_-uO7019}=I5I_(ktn$Gr#cigjwrw`E1V9=MN0U*;C4%~7 zlCWM##Ul>U=Ekg|@Je(p#!g6|F);|IrWR4Qei1hYehM_CUw+E|j4cBX-_;c9S zhAkY%WnPoY4n;XJ^Hqf3S=(g$I~$jEu!R{$0=rhVw|iQasy)atTfl!9#@O31u)*zXFt+-C_A@ffZev#7H7|$uKZv za0&{Jh)jGb?Pi*Jh1SlWC)*&mvH}gOyUs>8>F(-u`cqanpjeyd zw}#xT9^KH;wzH>pgu&mm93rrP#u=vn`R-?>0~QpLj5ibMOmxTWb8P$EN^S`n*x%g_ zuL>69;RX4T>d>f`+R<|^B3{Gm-hAR%86IR5+O(e$Y`i<}NeJ}}&jg_9I|bdmfK|Y1 zfGO+&L&>|`Cw#g_s@|iRK(_@RZ(#KdA?^asNKrYYbpVIM+iSh@2K9O2ZjMX>awX$$l|{N#bvSw++0?*5|XXIYXpmlh`p zH4D~o-`wu!8=%PpNRv(M@}+I+d3-z5jDE!RRIgf&jL92Dh7Eh8(BtsJIphTiave<~HAmmKHGKHMe|IPN_nS z%c(LU1xK?|f7N6SMrS0NIK=uK4-e05JI+B>uMk$%9MA;&_l_dR`Zk={#~;9YEi?UM z>uEp!oGDDrrRp(he9WJ++_jS=Joyw&hZ`k0D;C!_WI*B{_+cEu=FQ?`kg{eoPr&@G zt|VzY1C57j(W&64(0=TY*ic53Dd7~f#Te8570Y*47M*m(dbO;iXr5vYz z!JBPHO|631>27T=GOJaJw_hdWw_rdw?n+tGkLaj4f%upnxFgIL~kCAQdv>j@34L{n|>ZNt^+7~gwM3)5!lAaH; zg7HGj#1TqOCD3-7&Faualjr;rPO5IDX_5tPZEs_nMwa|loJdAH6m2$!Nz^8%Rn8wu zWbl=@Wqa^DoJQ={O5NzZeyK4gY)&xR<0O&7*B3X6wBqw_# zw#zhy2C*4rhW7D~gU6ihl^9B4vg-PJNhCV>6u6$g7*aImW03;!!#Jfh5~=|=aiACX zR~R4I%JR@P%;=y-T>~~RDlEhJNB<~n0&Q2>hA6Ch2FG?Co%Y+hM0n?s+)-%J`+>s7=&Fh}5NRHq}Y|nMI8+nj_{!Nt%aRWK)sF z*cqnA)_Z?IRavzneC%i@gKNFyyTIghggQXsH_@UPrz>n-k3j%# z&;qt`Ph0eaw>wiajBP#-Y=#z|2$-1TBsknV1$H4`V5;mK~t-fQ{ z;*ogh$OL(K%)*j>MJh}zf_s&8=V@&2w5u7lgxVe&4lEus&0bj{KrUh(Q9 z78tE+4DlNvMxfCODf$A#K1oC*nd#3bLh$b<$1LIRM5BBm93D@Q4uwyF%G67GHM6~! zY_nM7Vyr!pw0*i@=n?C%6KuiKu$)qH0!z{cUsI214#o zKm@UB-_+cM<233< z8PJIQw#i93$zir4F{XHdyi;iK)B2wllu31WuaV4C>c28y0Wr3@M5zOU($h)#=UL^% z9w76UrCEc#;?9l53hIs32zOulT{?UK$76E~afFa-6&uH@*dC!H9_nK}5YS z8KT-EUYCPG;*gCP2eOvLig?D@e`;oFMN_cn_eUcW7ZWrQ?~qa}budLScsQ~&(0=(9 zgF_qatq>`nw%|>{OpkOt>v9x*h#!t`olum~oimF-%-`Hh!2VA;qbIBoIy+8s zma&x8+Olxly$XmdidGFkgsmja4B_Q3_CPU2s5!CWmDvsXdZ!X3sR(fs6EFi^uR!xi zZ$SZZ!tE9e#vxfQHE~p7phv=@$(JS%+pXKDRF|VH&QQt=)1(48kswADG#_->sWPYf z2D9F=`If_IH!zD%6j3C?1$&78GEbjWwc+fv7sYr z#1TOLjVPk(aAMd`-~T9~z&9ccLMUP83OE4hI}Iubsi2OcVnSQ$br>R^)T3)Wn@F<1 z5g8}j+B=r;gj?sLwCK? z>K}kxq@r%Wn+v!^d*3OGLXEqo%)cx9r+3#P(*y}lNrwL3&uZ46gz@N@#nTnAVGtlm zL~#(MnD4(G25RYBc-x;_rwlC2Z8m;ny(*H=%q@#C)l5Pc1)~CUYF(Xt1Iq@8xlv-Q%e0l28J*&kGU^3 zxSjwrue$a}hTp3{t(jSr!%asG6wGz@wYa!vQAHAMsL!1Ks=T|#ek3Wu>DR}12xnvk z0M&)?MX%!Dw|*d6h|GevwY^nob{~Or@ ze9-txh4kA!r?bcLE7+@9p(YXC8foS>$Co0QZKZqu_(q)gpBFo`H^BU1r&=x)Gs28;PWX58<0(-&!&H<->6XE=JC=t5;qD|Quabr;82?I z1#k>B%2qa(bkLXes6A>@$q*6QQHu0QfC7sANP1L6H}B`l>?EI=?+i0^tUb?nE7L_? zT@ITpl_AVXqNd%!CL$f~|fPfP!7t%EM3K^;YTNfnE{<@({`P zy7Vu0htZGuFGAMIP1%2?LjSv7$qJPa0THF-Ysp7e6;Bni`Il>IZe5M~v>whIwoC16 z`CQus%dRI|k)#tD9j-8ue%S0vTBl<5E^{&*yXA1iChKrF=3I5ta7v@~F4U%jk*8h~3N|o3VDVtI8aljsE5Nu_8fk#_nRPsEkZ#*U3s5 zm+`*E%b7xbt&iG|ZxDJTUJT5iF@JqWGnu9|VFEYvIB`#`xe9r}DNIan!AED8_ z`WPGvVixK}xCci-LuB^J{&!uFn-r>x95RxMsZN-6Ef7WnK%-@gBkltkEa3W%MK+51 zZ5&-1K`C`b%h;{p@MK(R=8=TDjv(Kdgyf%G#09yM2c0@+G<84?qQ58O#FO&H*aQc2+27kxukPv^H1I4U{8!uFI^ImGNB-y_?bE|L*Ai z`yZ50_4?$_2~=`XN)=JzXCh!LJj2eT=ci2ciF0SzESKBwXiY92C1U0)uv)D=N+#>m>e1x za9Z1Omi?;-91DHc%Y*#iwcr_YDCw5jg>{J$fb}TGKCTJoIKy*X(+}Wy6wA1|6%6nQ zBACgR_BVepIN9j7vq$QG+lU7lDc+-IWvIev?=L}cHuMt-=NsiGxa*QXmF!IR%i*)V zire6t#Sp;;ISu+ZFd5*KqHAzU(Z5@>L2&58BXn4DeD>i;;~?orv&Pa_KsS0H6rt3x z_KVGo0b&OhRBT#qd4m9}IuzXUwr0K`B7-pt%>!SLnZ14zp1BqDUE!(d0g%I5CqW zCrfpK3WRy9th5#ZCIPo+(_IR5uGJ4>>RRBO(D$b3vNhK56iaH<6=L2h#N9dVC*7KP@>`NXJ_z%gK?jyHg4X z#NZ9MEqf*~-I zlG|i@Qb&+XIIouc8TyqSF0EG@&tna0^Xo z?Sml$I{dINIyG44uvpFv7=D`E0=Rj&&E{*pXXzg69`f0CiA*51vGDC+&bE>Ego+c@~UQF?o@uiw?Ro8|ko`BtFW|aS?YZO8v`DC%K>ZZyQm{e zlSF6W`SKr(M1cb2p}Ks!C9pWlOJAXgElba4X?=d#DH`7K6?AB2#0vQPiuuyMn7Bjp zjIJb8A?S|1Vr9DzGs6y%T_xjyX8F5>>ekk<2c{{pE3ec%k%a=yQi?e|60x$ZAVTxQj5A)(-I@+)VfShMaV_3$XB-n|+{0dm zDSRNJ!lJ_P-SCa%>(}AI*)Rs`nNvQ?IoH_yV${6Xpdll+Jkh^mFVEFR@bCrcB%GdD z0wmAexsS(T`=bnim#gW-xeA_Q$}iSPvxgJ=pPR1X8AfSl%27cLpUqQdrdvHxB+4Z3 z&xYYLlEikzb`PxPZoXHrw{uPYO-@Wy?(KzoYVCasRrP8~N(gk$Fezrf;|3@?Gy^;#o!J^%i>dgw=EP3nEu53T$5J5kVU9$vRahhM$Q%I4)T z^6VJ@)5@sZ);jd8OXuNgEOWmM&t|fby?^)C3js!2k*S1g7gjQnNac+hf@=ZRp#C22R)qib94*_2BtbOUWjX(6LxAF{#?*l(dn2xfj~&S z!O#}!q-`v)zWw^=?}mAeFdnx7_S;{<@sjhu7e%yVQ1H-a99!{(^t-=b@##K9y2F_G zt(9=|K5mHkc=R@2M=Bg`jJnet1s=Sym^)0S38?~|wQ)YC{XfVKca&!|O*M+w9BLiQ zo*R6Q?atCJh@XU#RMUdpYHA&7-A-bE&F)Cys@aM$1aEcwt&XP@a4dZpKSK??HQBt> zBh*6>It3;QDSXhq*<9BhPD*X0X{zLK==6^$^s~jgMW!mAI0A*%CKwC z{7H;S3F8Bsh`Hc{|B{rR$51p{4YQOH;1cnx`|V3%w`{L$PkUk)!$}B{X%|XYV3J|$ zU+mWFkefE%w!_ZhjpE6`W%(K2&=N6J|J^x?+aYL2wCdZ?^s*IdpS@bGUC(I8C>DuQ zsozJ)<>s5hc4;AzJV|HG)77O4fg9pZ)b1iPvdB1k8!X7!$S%yk-s}kxJe_27QfN6x z^WRU^o*!vQ)s&Pj*0$SmAE-YeJNmycc`G$@{gk^!?eGjT47_x&8mUY|B?0!Ie0U1@ z-~LV9l^r)$Gdt|GnRp9_=d-dMpUP0_D}m|jzj?M?n)}%ub9IC*_AD^pa+lrsXO%=g zj28M$PIsP)>?PyS<%MX!X@ddU9sGmk*L0b9mpWJ;@mT8hUjFbHpJe15KcX%_jZH&&}t z`oJT%cvok17F>pxB`J`f=#}}wGt8&rv$B6v_ue2=ir0Wr%(q1xZHA4fe9v96!O#mq zt9lam6@|IOW?#e9MV=-;{pFYZ&iUW7BA9jA+-sF6fkZfTN3G-b1>uBhS3Wb4J#r`)pT&drT9Lq>f1J5>=4r7f1MNqpR z>N_5c5hB7bx|S{|k|fh4Hg&}VJNN6{c8e)y5isw5ewgx=PRNmQw*kihvQv3PJ_o-& ztsc5P$0KpjWVnfU1P!kS5WP{}SFdJKR<5Rr)tzg>tkxaz%<`Y>R~#E!#{&4x0ObH2 zCqUS1ywt}3$JJZMMcoC9!_wU)je;Ul(%s$Nuyi+4ONVr~bayP>jYxNcgmlMB=Wo&H z-p_sCzxR7~=FH5QnKN~AVzTcwxm{`28Jj1b`koGQ-N^H>N-3SSr{;?)PYH4wcY^5S zE7ooYh-ti)K=@b~#7EC=nf3KXr!Mr>*?dogW1ZIQV=bQ1kIz5do_@Zpe4@YBlu|?_ zP*)Z7i}7(yOngvrOO9;OKN&@41(l@W~Y~gDH0p$!mrrsz^GAC?Iuf2rsO17+RfyU^4z?!r6bFc zmZC{B#5R{(Ia9)zQ=ZevJ#hRXX)IDrgjbRb^=%4~K-f9b$V0jA*)N3ytFk2!8 zv%U2Ntv!^H&<;*q`8H=I^S1t&l;PY4YV!G+5~n$E?&NOd+Kz&moLV|BolE}M4L5PB&+9r47>-`1~#x$5ZdE~|dLp;Yc zhW?1MiqkKIQFHmCI}jU_jH03a&uKdKI_(RMXi;IO*wWk?9u?j?tx6|TVNCNTtmX(l zU#%qfmrXJaw8~m9)tn;pQ2t80s(EWQKSq(+x6b>0^Dyd<}fa znZBu!ufN=q(Ef?~Bjk>Sk0R=o0~tzhweeJ_YklCBe0_zn5X?sI^mf(Art4~;awWO2 z32&tQ&Fr+?$aR3XN1=raeFv?dddnb1lt&9t={^*d+w&RM#n}xNGOdLS73$lVc4Dop zv~*+kitKzS9PeTSea-rA@`AnhedS26A)7h}WH9jpxx1eB6~3=0lYhy^0Q&eR-~;$R z3QwIK*CO(~Ke*V@OXnf4b| zf_qY!1I5r-r#CXo6rWn*zM{s%87U{}i~XkTbHT8-BF6B1OIx2&ZS+}8>ssLcV#SwL=k(g3$icG-}iftUYvg$DNkPLIU$B z#RAn1n!2`4!V(<>J#zHRP|Bk`V)sw+Uv=(#!4zrqgB~Ejz1yIqJ9f9|9Y(&a-*e4Ms11a}V90^PV2u4?TzzdqN$}DaNJn=N zi_k3$7l34KrJ<8DYkZVd-WGL6ihK6Po(5r(FW8oo_RuFrP#=^OfZ#%xi69O)tl9B4 zh3}L=SEGfax>D!MO>@&lTMq6MSN9XF7J_=vWkjvQl<^T6$1Z^F@>y6PPb%rOp}I0# zcHl1?yS!yOH7;}JF9$=e=E#yYHombogkM3jYZw|BGQE4aw(rEx9=H1;m25&Lj6BC* zq)_SP0l!m`;vuC}kBB1nR+9cnXfm?r>f&K!tLbbO;Hw17ISlnHYe=L1JhfL#Hxr7& z;koR~uEaW(Ta1iIFG2G>N(F7TFC*}9@3eUJ$1vq9?fN&WhCmG2o*e7o@a{o~LKzPd(`Yi_A{cSzCUh%l``@I0h6?cv{7!<-P8 z*<10$cg)c?bqQC0LRH;(LJP@S&dX(!%6`1Wu{8B|8c4YOkxUH<3arveO&b^*UV?(& zmVcUD?==*^!MM;pyB#2X8VvODUcHe{^ta?dx1*WoBVTdP5mF-a*(OGK#6aT3Ttj`A zQ5}$jD;5~yU<8BLk0m|?DB_gIBd0xe;~uQ6x+-56%H(kO6Iyz>2V{0Yn+s7~ceE&s ze>kO2*x24(!8%3Cxp3h-wv|i&$Mn?~D%MYPSSlGUz z-jnpPi3Ws#mSX3a13Qsve+1p9K|*)G`w=zYxN_^(J=zuNw&1_))74Z29AFyyd63s&2Rbfrd-oh6q;Ll} zOehu>T3tS9@$<>2g1y;R74TOm>d&11zx}bZHFU{KjSV|s9fMr4qP>Te?S9sxmRgH* z7Dz3~$1lLph!sEuslH#MM#OKg)j9GUqN6)T%)uyT>DLbbV{joB%BJafp&0BV6z>Pk z99^wg}pg{3iOWo0ng;$p!tQPwV&(;U!O@Tb~P{V9#gV$DuyK6{qf%YqP!&&^7`!`&U90jdqZ$)Ss)#p zOzs{)vGoo$n_N_PioWPtrwPs4w&_L-P$>a;N4$?s*lv^hQJtKzvGb79X!!xsH&r6^ zZG1VdE$FqX!6z$4#$DEW3ETkD#UV%UL&JRhRi+Y&izLmnvGN&$Duo(L1~~ZT?Y)oa z33afq1XYVyc`IW|-WBs1r5X+0_S1S^B|F?U|H*cf77l(o^SROUF#3Zt4K!JU`BR{V zLUkL5LGI`dM%H7JgdXIPgj?%_hPHj%3*}GeXVN{QLvyPnAuIl)1$M~)hOQm>`|?|? zFa?D}?3*EhyE!Q>wC#6>Rsp)>WF`C=yRmc|4G(|>sE!$e;*n5HhYkPyq)nHqOQ>kh z?3cqoYfS-&fdv^$?&84%!r_aa`fVCq4QCDdnGv2B4$!nJU*+l^dsG9CZq%l`;pvpV zPatWa*|?|$Bi3Sok&lkR11U3XU7f-F5B=u#2MX!iG!&hP#+N&wF4eL@jRN$a8OD}m zhGm$;$R^v31ojCYPMq@ZN_df-cLr%d#L1|oKp`7L`gy%I35Pp2h03^WEiXH)CFmOx zy4+Zt9Ewz^X(-xp6^ed}b&9duKwP-(<8*SLglES-fr5M@x=sh=8AgiL$?+{B@|WAK zwZ2V%_}ooQk0Gt_j#GkyU!6xgZTscxzGB%=k1R+WWU|!(Zvunuu%mi*HygXwKGq$3 z)Ay>Lcv%+nqTTFWb+tX$B3Q4oqjqRSwRG7*THI2;VZpA0+|PUam=Ld$1lqvNwAH0g z^Q3`njXZ`xyr;&Z9Xzc@#Y3GV+q94B1dNdwX&Rzrg458}R$eZk^8S++N`lFs6%7I1 zc1iE*XTu9+R1`b~h_&rP9O^B8Ii$M$TuY$BK|!P@+fR2Ex%u=I&Krr^;Q*HCO~E3!aIXk8TwYwEF681w|B8H0C8?u`b19$Pj`DZ+C+|EE zFM}z5Q3!sTgq~(x~TiGktF2g6`+^2@f#T*ISV+6-!svX3%BtaDxl6FUSGg4x zX!z44hvs{%WW*nDWK06%U<#w3X?=3uv4l#7?ayE~vE7&f0+Lo@p`U0f3fv81<%!h8 znJgsrgSiI?2agoD_KbhN+d$o(;}pHM(RpX^8OW#P1F7@{?+-IAXg8E2x{`0;NwQ#@ z>0EL5vH3KM7I@-sb(sA^Qauq+&bZ~RZ)^X^LPR@uyR#cw%gN!gwG=6sihmH2@O$mcSA*4FP9IvBStwf& z+=@kCq}5G6-C%w{pyX=$aYxl$#mc?)^xYaew*O)`ClNVY96J74J|v5BXKsR{&bNAd zMN;k>Li~4-(!XI)G8B`)rbIi*U}V11O?6Jg4_XIKwlB9)h}k-6B4>UUcqhvFgd2B# z3MpIk^}3`Azl@2$*BE4;RsG&xX35^n_`iw0AWVzKxE;KB|Hg54 zgEOlNKrUM9szuosc9)7iHti1|utUD_DQ5YMG(qFRa6;pC%*Wq6Uq!HAB^?L~0Of=l zWlK=OjxRzSTu3N%JrOzxqdO|`hQ=h0{o2czE0a#Z&Lf|TrYTnkimX#ldD44?BIq*w<%$Zwx9~E6xl5|hiuP@4h zh@;XO)XFt(Q~tf6G3!C-21tqL8&SJ}|4CFT;y_ooe_jIFOfgC#Foe$LS=ieAQUFb zHmb2TLuf(h*TVEYcBwC033N6q9W;}F4J=YmPoB5y948Bq)V1vP-)s@c>h2oM65G-S z8sHU9n#Q*hWfdnrDFin_&!Dj|COGU~q=%vcwDxzgU)VQq?#;BZ za{mGrkqmf;`u08`But*XxG$_*LP^krIIY;}@#md&&q z#m1sVqRJ)Tk0ze8M#QY75{q6}GQBVBgYHTR~(Ex5;sYM*vV`=s*x`G(5o zs!wqAi|^_j~rAxt;ohBeeRn zLBQ}3kA#dleK8H;4C07UBi(^xzAcbTP8S7#L1X}3Sx=6XKDAvJ$C%k1a|f*ucY{67m@YVe>n<+BA$-;we* zWT&mnT3*%R;Wx{tQh=_jTQ`^ah1qjZNUStdJ12Nf|oPEPRoVoY5zG}SzDOfyZONTeznU-%roYsXuL{3LN}1Oh8LRU z=R~|Z*n8kcyVDWDs-~eX3DPExq!oLtDMQ6)>C+DQ&vZf*Mkk`zkJjro`+g%H2f3E~3S+6YvF-@}C-G+5^0wUGz zA}=2Ik5c6WK9B;^ySpuX2r&gI5n&O&O_UD&fIW0jnxhrS7KcVdKCcU3FSYd|z=(pMqT_M^ES95(TnJ=_#q1c-p@nt;{i*jc-4kM{m#L}6yvo4B;n z_!d(q?Tye*AR-cK;YBIVPGHb*n^n?jPrRnKzU)O$n7ZY9thgai#8Roo`Hf*6r1uxn zQ8E+z?t)AF{hN`O`rB01_ow~S!Pu16#6H{!8htgJED*TeLs*FCXgRb5x*`gEm#kxH zUqTvB*G0ST_82Yt)?8+%ivZRMb+RjYmNF&H?V(_4r%cm9?=8sP&g4A+ zuW2$p+Y!JYDIR~8eIZsh-`YAhW5dH?EgUU6rPR2RO}D8y>v?Hr-YR3|->C2iME6!t zWh-H#u{2hVMZ!>5vNd@gblU{wfPJi*Z(F*M>Dh{cUoyu8C`Gd6q_b@v&=)VlsYhZG zQrc3e%v_J*cT}_UA@zLW=j4VHcBnvGDwR)FjPXpCCsNlT#|44ps251NIHm*NzIA5{ zdkNF?6E(JLlPKH8z7~DKLFqHXr|5#@f? zLRn9cHN8$kmJjqxPajl#n=bm5dV-qVUjP1nk!-ovH=n9{3X5s`WM_+2bJ02n@)!fN zE{+MwcLcYl5p1dDa%=Ur)2V3@+W|B98TNF=kRe5uE_H7D!L6O?5yWf_9MS{t zzfEi!0bzN6L8e4($`4m?(x1t3Xp9sZnU9q3>_zCjzQ8~&(cQNyV&*LzT3Jq@CsF)W ze(LPy%KF}5%Tsklacg^D)AyEzdNi59F}MJZo^*`4TwQtb(-DkyHy%kpI6OBJStco3 zhiz;QJ~C2)3C^EMY0I<$V%B9#`7KC4f&zv65dOMTUofCFpLS)_Bm{gm%$-+ji9eN7 zsKhVcEOd#exg_nfjW-~Oy>2PDVD=m-U z_8Au{SZO7pzIJrd40ryu8L2bhHi4Cf>XbD)%!1z*%6;k@ffhpi7m#2FqMv~*F;Ae6 z`?`wrN*YgTLEiT)&nrTYt%!w^iZay>Qn&7`Ah@8J=2@irr%82$;|@c;D#)lANS9*E zqpc>|QT7f^uZxSD+Xy1!d{^3?#_h}XDGYDbq4dUW9Bcksgr|M8v2i5YBJ7tBm>wvZ zu);&ZiK}Bp3v#tXAF0v-7P-Ob@G3UOk)@{{H)|&4Zc(gtXS_g@p|rT7_JfxrjEPa& zo3$Iy1jYfk@iqN7Is?Wpr%P5;#Wk*&w*wS7nRAZ5Qs>W zNv{x}QTCO?i(}I0UK|&iwj&n2q&{1J-o8=bql)RecK8La`%DYx^KnuS=K$i#PNEtFB(;j@D0xs{>MR;jJ#la>ks`T_|v0ey9=-Jvzkq^?EgoZ-k+ zRC44nPCIx^_ze_no}j_6b0l!oN*B$Iu5Z^Y`|L1z_ERP=7`%?{RH{O2KVUxIE;w(m zPwZC5d(7LZce%IN11D0sjoixMc3vjXKOlXO&7oMo?ineD@+lW*5~1lbm!4L<1qmSH zmR&*!1&evEV=y{d18&fH5jkx;SdYvGr0+0JJPAEtZ)Tee_+*7%6VGt|b~a93{8hpG zb=O~TJm%+D-A(MLJ2eSZm(FR6&uM?kJ|vSZPwQR^-HB*cQ~c1UlAC&~&G{<>NU>Oi z2nfX6_JYL}q=1FahtK?|-*R^Z-bwHJeD7Q3>-&2Gytn7WsB>e3BxHS|Q@DIE3d6l; zoj}PB{^Hq6eD4r9epaU7D4{cDp#8Pg*tC_dDVx7pO-Y|2DGKjYu02@LKovBlysBwh zWC^Rn*JJ&~BO~GsG~zc+gIhl=r~+bX1HsDK5(15PXm?-0AXyZHmRPpware7lai3ci2_ zGXO`q)V?3A8>_60`K^-KgO1-9=DqVx7hXwf$JCZ>oQeX=CKYxJPt|~ue-wQuc z`K6KN{fFjmOu(0hmV+_o;e!&4A>%k>98=%GPzWsnoRf!Yh6<#lKUy>DgpdF8{Z=Io zQM+~vZb_4C#Tp=1=@ZvM3vaV2+cC}60j@u#j691uDWbC}OTdDP;$0|&gFoU0v-3Kg z9`;V-?N^`prj(ZU)`9c79Lw<1)IxFi!1Ve`mQl?-lz25x}+Lx{w|9atbqEMo}SM8znSus(A zjsxIS=XH}sKD?i6Y;xVJvF%HCXMbmMB_S?oIM%wUj%UEkRCCVjM#^otKoUU##**?+hBTemtx4Yu2UQ*D3VpwE-l?NUHsrsgdc2KG}u|DSy@Sct_zQeIoIKWL4*r**_?LU@h3WM5$-j#TOQ99U77t2}%(@@N&hF~`V&4r#D} zHPWHrq10i>1{Qj~bY>5t8pDDE@lba5UT$l+-s`MulCKc#8K2cn*BO~j1!xkx%<06& zS5Sg#GWAyld~--Ds9d@SUZmi>zCVlfWMozJcFn$?*deah6C0}MKYA(KvYjz9N^$*I zTDP6RMxIJpWADo{g>#=phSO4I=Ya;$ zG2(S6jpZ05L9=n7`YEvg>xKav1QW#?1>^H9;=jR$?H0}sVZHw+EJ+2ql!zjIBlw!S zNR3Q#^{+;IbZPW+S940!GDmGw<<(7*gxvFQfj;Z^!}Mj2C%JR+*-Dr=EU3=yCMs;+iY2yX7qkQ4H`qiu0rt(D-B3F% zMbesFB`1LeFkWUetZ3(plE8CM?JSJ3nGbG*J<~9qS=fzR(F>$3;`!G@KYSAU=8M$O z4s2lLi)YFX-z&%P%g%B%bpbKnPy54K)pM5r7-hOPzf9U-^Y5h4{R~kySW(1ii zYUIb$2<;HdXFiVeFdRTonAh4~JY#J2GANW^pfB}OqFZk;mYXz9TikRxL5itTw7g_O zX)7o*T8#b{2;#;l?MrP|8A;_}M)f_Y*Fq?}zWasUQ$C?e@~V5NqW$Rm&17lVW)O{h zsTRgtg&L37Y^oNoHrw|KDh}O$366eu%HkT<)zg^sk>d`UuY!42Sf2q1#oD8*lgCb{ zY@?kUr-!;@jpR2`*offyOIAgq;u5Y`9#^bn<$9Od;zNW;e zdsD=8fH7_csykbMe6U8ZO7w>d6wCe*-9Js%{_|Eur0QzNFOK5u z{hQsKm={F~_o&hp8S_^j4kB8hZ)j~5k2IYRO%_)k`hQ*v0jVH62{yN`<>*mZEhBqzposqJSz!w1 z$WaU-p2-W?*50iLlOAoa_W#*_$$E^6BxM3HWcie$9-7$}L5}4siweqdXl71Te6nS?P zwu`cF3kxd8sQ6MmI;_CB-&hU{S_N0*{6&J=|AO*k_;Zuxf;eSpCatggZWFV|{@a%Q zV;NJ)u3WxMvg;=zcLkQ@(9DM^om#6u$b{?$&5?xs^2t>?bSRM&zvv9n0p;msCMz}c z%q|lKe$J=u84$SLU@0R z`X=r zLV;Efsxe+S-EHqb10RF`WIynHvMaf}4!BJ~CH*4H!2eLkY;|twvp`UWQzXZ!<5cUk zGqga?@o~fXP@?vAqgL^W`q89@>z0(LK~fG3emjBDkbP5mOaP8JF;Cv z&spfjKUbk&MVLD0MRC!=-8Kg8yjwj+lmv3XAdXCEcc?$*j(<8?BIp;`<4_V7r(atKw)}-<@4h7(+t20$#= zcG>s)FVDV|ouj6gViR+d^sgn)9zR~HQ#3y;KWr1b`9PBD$lGBHJkJ_bk^z` zbJ!HX1w}RDryl$BnDq{|(xiq&llFe2-u6|RN#toMNsPXaV0{JlULF}(4?>)MYnn>F z_b}i*HzH0M2KL@0$ej#T!*EQeO?}Y|g zqiH@h8GHx2z=u9Ar;6HHM7!+RHc*?w=cCV~-3{xvUwBz(-cL{9-@9rTh~4krT)p33 z-wpn3%>49(_ne6vkiVKge{$FN&Vurn+M#j{cT$44PfNABKQ8PXH6q}_Et+l&-Ihs>FBfF84B;rGc zew!yHtoC4F^e>0iiuhxIQ$-`I?$`$#9&YfVh0FI5s=a;!xTXJ@1yBHdUgzclAw67e z^u8+lee6%ew8_@ys}u{m?j4-kBxZZgxC*|?`pwS%$9&b)t=zKqyvWTGfJ*70^F>Km z_Cpb!R@sWarR&v5K?3YNDQ^pzKSgai@sOzcx7*JSagZ$tULwzO%#W1G(VEPRqt$+W zLAPYv)xEl$oB*@Q^xHk=iZ=NbdlIbjU01sG{~Rob-{!twp3roOz$SvqWM+Vj$Sfl+ z?_d`^KTHZH{Y#SHn3|1-VeKvZSMqW|qK28U&96}ciTn5tG3`Wg0B5(^i8aXhdB`0C zVqp?D<`j5*F(#x&;+BhuK*d^AL1JlX$utd@R#>Q}PfE?t zGy2$szx8MSNoPAV$zCe#A_Thw`hpu=wwcDALu;!fOdAVP*-2uxzb4+R?=S3t~dP@4Ij3@&S?{X|*(4yfj9f`iq13$rn z$P^b}Zzrz;;*Zq4=QcMNzgf9mv?EnUuNFPWWUBt@tg;BrVoGhFH|h4l4A?5&`0lTQ z^ApsB-Q=JP!2iM10m^OMx}CqekU1z-Oc|h$&xvh*{Ru+aDD;2@vRux8 z_(2pgPUuZN)Evz)9Xt8QGV3bHD^YbF$cxIy(}z}9&Dk0McQb;p+L^XcA@XjxphAi| zMQ4-LsfEc^!>8xwE}7y+BOVu+?0+N;b@QJ-DO7F*9Zhszv$o-)ev0iC5_gEV;Z-Kv zto0r9u`IKW$J&cVbd>pBzLv=z8&A^JQx31*asV_t=l2(jB5u&+;Xk`&aKX*;-&7#;|8`u4hOI_5%favuk(6$eBL|FOJ< zm*_BAWU0)_k4A?!TYf_cA{BLijoCoXD+jbO1CcR2C<%d<2M+Y0ev;r~UzI?q6C-J5 zR(44`qfxZVFX!aB<61U0wI5qJ!I1Q1eIs)< ziLgZz$sUtLM2aUmsT-={y)B*3ke3Q(l7Ndok&}SBwDA{)vX#;Syg}B(?_&CC$M9FZHh$y%YtvxIfbSDqrlY; znHW`U=)Ml-8#(M*es9&&P5tFF_AukBdRRx)$)| z9M9L{Tqln+qg%T2DWgwUx-2QKKO(Ul`Jk=gL5{pLf&*t+Md+%miEh7sJbXdKY!8M? z{rLBQ$w+viw=ki+rcs1 zRy!*%LO*6$DYd6D#HDaYqhy!+tCvJmx=a3uOUZ*WuH_GSL3c1<(;r-v+=Zx#Cp0*o z!FnDshpRpn4`bak2x;~st|?L;uC0(O{+M@UXOUTfboR{s;P0^z9=qta8*Xq~#urKu zTy1BIxON7;k<7;eR!j=s-yA!NXgfw7>qQ ziu|v$E&-+ISYA~jPh?HhMp6z0XAt{|)wcwZsoBr>R)x+?k9LUjep1EllzfA1|S&>Y=1>VG#j+ z=5jKBZ4wim!%?<<(#`X2Rz+srdDVFzgr($Et-g4Q+kR;Oy4BH>pxmQ$I7fK%Z3@=Y)G5%17;W(E#FWKV< z{S|#t1*3CEp(&%4W>=u;ttHGsW1|*<(X)|P^BXikx^~((QzP&0+I`FbjR=ZT_`l|j z2{t}He$-$#rF?D)E<70b&dK5zNj@d0G4TfyBkP`;Oko^%tf6`8qJvi)(14a?xZeMt z#tJ~(&nON{WRFdbH-oJtRGt%Dvu^I(ga!SMFCu+ak5I7Kg54R_0{&!+(>i7OAl`bs zPDYL%>)Q%Tq*u^y{F@0TfdCxSIOvCXi;;y~BwVXlrR3=OM8dM6-G2LFugr z3ySN6sgL>hc?RTd2=-JBBZck_HOBV%5Xf9FxsUE>C@fL0jt1m*^a33{Z}#WCiftGh zxQudaHxj0z9xs`ZtH)Xsau4~3@h!n57odSkd)W_AH_oZJs#$1H^9~;Vg+vV@bwkoejd{n+&6bnKTK9%TZ^QKA7Zi~Xa`IPB=gTr$47 z#?H!)#d%rKl*W2{09W`1+grZzf1kYZcl9@;+?&fXkm+k>o~Aeb%k)pICrf*GZ4kq^WIR@L49=gsJGVss3)eU@%$=5~T`*pv(cG8Ql}NXJzEZ z94#8pKeZ}luvskyZYY1z)7Gn;nyvJ^hdjUW0fqlNbvD-?Gz|kCQVF^O^^Q57`!Y$R zZL6e^m9$Y2+9zfV+n~L2@pB?Fb8Vs5%^)fZ`bK9lJ6Xjxh)x_lT%X2WrOyxfu3g#I z6#f`^A+=uiA$hg$y()OhCLi=bT6Zho(PH3vd=AXP(b=`g}tzr{5Aee|C zM$Oz~TTLe8SlBuX+9t%Xonu@0*$;ww26%GDP`jfQv)lf28Xbfw1d0V;gPwG{or373 z0&fW*j$KrIT239>J{qe;rrszg<{=|Xk9A2<>0fR58eQI#&3YGjkD8pa%g4x&f_3%7 z-I5i;F*4wo)Ce1ar9o@Oq)!|q*K{Ov@>gjfy5mX4Q?*>gr|&2p2;Di%y$22S_o8QR zsll8>R?R)6W+wOPg5Ev%Vz&ps{eJk*t=gB*aZ@5sWqZO-eYnt32B^)i<|=h4sKD1# zfd|V0T@YKx!L(4W{jCgQ>ZFrB{;fJ!GMKNADEtJ{1f;#R$C!(*8{StZCyQbLu5#(8 z;e{WbUR`^}f37{nWWJ!ZQ}{3E1NvIybyuxUN6VN2i7lwpxCK#p;>m|RAP5$e`xFd8 zYd&T9KLAn&it9a2r-Uq~<9SQUD*9V9TvN`m{1t_N=2|s`BZ@kM>KI!Uh05#H4oO{p z+py;@zjv1_SsCkp|9aF!>$-PXsuJsGol`r`lPpe)20AU51+}3kE)K<~f{^P~+0l}>YP-$Gk|8Hqk)*j z^`F>cNDLk-E(F!syu6FIjTMCnlQE;fy$I*}k8DmOHnqq;B{J%%HTJ zE5Up)hd*$q5I7MrvQt}cSN?g%O9OJhoBNnXC1tioJ7d}a)O_NA_?mgM|9IvX1Wkmv z>OM5)RC1Z))oc|P#HOR6_Ap3`VbhR*$Xr9GIyr85KpdbU!&QpCCAR#O)?t&QQ{A|(1}E94CZ+ug@AwIZ!kWpo zX@-NtcM9(&LP~u<%!;bbDD$D!Y)J z0^1eaZ;R<-_QUieDi*epVq=`0oa=*HNz&@aWSn+wdFh~kr~jM$MlGR+q_$P-DBxCl zpXj2_OdQY_P*_wzq!LodvS6)|meYjU4lgGO_um2q3P=#B)xBq#zT6ZNTKH|d!?sjDV%$J4u{J|0?-QhfM7{Gc=r!WQ9%k?N|~z8b8fBnYn&x-2Mt({$@zla zHY>yJhOzvu_4`S6k*^*5`>g%kPyrf8bfy1U{%T}dGCr}NsHC)vRvG#Mw_|H#ZMcm%Q9o%67B?gTU4CZbj^sRk((#r7Cm^hY-r%L1Q z9xOM{7Hq<9hyOeP{BK`j|37(0wgx%Fl~&9iS2q&ML>)M5sYOM$YXc_`0hO}_4F4$j zo+5|R>ovzo@|o`wd6hQqJlQN!4{ftcoU`a*Lq@xviq-z zd7I2%%9xxE*h>M`v=&>7F>Fj(z=v}>Gw_-v-LwjVJcs}oqb;g`K=Ks+k8bzUR8P-7C3qRXayI-wGviygmw!e@!!bpqxXd+jHnT(vb zoTtFLexo26@b2s+yV5;BKa3ksHJEQE{BW}tJ@bWqHB0``FUT!rV@_DM>|e%#f(Vu& z+p(~quxR8s2vqO3mvixDp=Q{4)0u0f3GU(NQ*|=`7x4>*92WH0o5R(Cbv{_obGYN* zSS2KT%n)m(kVMRI*KV`kCdYoVY;k>Ejn&?oAumI|J6#V&Hh*|G68kFE%Pa{K5}57x zvi<)582KZ*+cHohU1p&3@f1Z_N1s-W&Gu_la&n5cD;C71DVJ1gB^YEX8hO9h;7RFw zjop5q-(T~e4Al5V*YXS8XlXS7gHq2t{yQC6X@8oVBL%dh)iep(TW8?&zEgc}EyA!p zkTyXtru=-5bCg-#e-Ah5`;T9k7C6-_$F+_R9$GbqsVzc#K4W9!33?B1L|Uakv$v_Z z%zh7AXJ>Ry)0T5Q@*!>$ae=g*Qx|(j^T?vj+!UZA+pnp9`271XBT<9x93}7xSpI)pMe;w6v}xHYV;m*`J;m>P8Z*$=t+#&nOP)ry4p>MlYP4(~1hQ$hJ;_LP zq&nv4yzg>_P^Z7Y=Rd**cQ^mW+2*-A1EsvcXuyenzHG{*af8(Sxt#y zG20OKs3v^bvpUzF8yqahULl!&yFS3RVHyzaiaOtmnu)hyaeK^dDnF|kTw2F$; zKpXtjiS+8US&q|waEBsWL|%&}`iuCcqQP@n3;4LVcyo7{)nSbHVvkzkNiZ!+peOON z-dRdcE{R~xga5mg(Y>xtIONIJ8~Xw(kVecxE(^L1gKhfPrh!1N934nr+77rF_Ig5M zOQ4e)@ZytGKZ^Bj9v?UuHx?Or)|fW2=|r|K!_rlbRXRDZD3w}QKq@q{pjh4K5O$sI zepSr=U_H_rYNA90cz@me^vWQASlsF*+xV4;?4LO2^EVK?hxQz$+onS2Tmub>zyaI% z{4)L*>Z>0F%NUy$|JkyU6$}V0#tqOw!7PRTX9i(0_oZ@dQ#H*<_J>M~pI||JNHzbt zp0JqJf)pH4ga~v7Cfjq!qW=*S;(>TzPr{K}MIkhP>OuMfCdwc;|92ySfZ|6oDK?E5 zkC8!=_uF9O6d3>Ku|O7iVGqiWVWo_GxQgb^j^3k~fQAKS=`H?^zP;k8`8n1!vfZlG z7E*ZjRx@D1;F6Qze{eikqJWf+(G1Zj8;@Q|Bk4l^+JR^Mzd$Gy>>~sp|2+N$yX2U@kR zVBi9`I3UpJgF9MyOu|1`6e3FNaLgyEvPnBUDFc*{CAHPB@ZJE;j_|IG-iUKJ5vJ74G{0R}DI|*SyP&1D2|NDtzBU>;yjycie%YA65W=YDq1|`l6XboKI)7*2nL0|Q~;kg^=0MGX&J6q z3WU{RG6CkBcC*nRI!h;?lCiY3ifi(@NFHr>Gg~~B)0R%7`hT^3Wn7fs^R_f92v|rX z-6-85(kb1DbSvGkAYcFz(kUsqbccvaH_OtXIAaX6CxC zxoK&Mniw7G(cQPM>!U=?2%Mip0F&tu&Ze~yg!aq&)jQ@c8qh)Lv(fIZL&#PC5oY14 zbmQT)<+eFBug#CEmk@F|HJkmDJhRzesUObhW1l%|fH2s|J*Cq;SWzZmw+h})=@m9- ztrsTz(r{SoGPiZQx1YMaRgO&JWikuXC@5^ft8?Zwym@~2NZ0H3OS!wqZCVE(+Bf;* zOMwoQR0*?I=sLh?^ceMZA4*mbhx3y*P#Pj@0@GPPo%4$GTs4;NP>AgGlZ$iQp`s_A zJ*9Khb4y}!bARhrc&{mMa@L|!K9aWK&ud=7{5@u3>WOHU5atYQR7;Roq}nH5r1=;S zue`lD$Z~5OnL8kCYHm>NlU@OmZDg3U*x z+R!$w%0v@=bJ0`effv&Vl=NR;G31D!XE`#tnMw~IMA{0J!4}Q2bUc0!;R3|eS+bh+ zug|{0TwF|}lCHEsT9|N2&iC>v@Z4^dWZRE#Gu3VaUAL~jZw?Kvq1&V*wJK9n`*Zxt zkBUUHriALf4o^4_O?IsE=30j+l^>;wIF`L^-%;tr)87-ld0M9wp%O87#G0}`Y22`; zUonPN#Co(85CP}z8P5_=o^jj^pBEH55r3SA_Z-1=6I9ie4n~lvkaK(xejX`R=F9=b3KthX=x5M=EX$ zK2vx)x$^IV?sXJ?uN^IPlFV^SjE8Ad2+j_wj!~7jSao@-P2`A{qvxz2Esbphd+gdI8cU-_(P?k;q{}HE}rR-yy!Y_ zoAf%8y`%&0!*n>Mw(Udn>OzBwB38v8si z`a_#qovrP7w=_FVc*5${hcCO1bsne1NiCiGq6U9{++34ig^t>amHSY-t$ndyPAiyC zEL)8tup^egwUs^(3yppJwKEAOFxsfM$4yTFUaUrg9r0-(@SRX%mI}m;k*~w={$6WV z<&2|8UvGE%5Mk=Fs(O6HPuV_?UJuoU#Gifmp~|xcZ9qNl1q7 z2EP2rVN`*H-A$|zG#t75)0}O7^yS+Qx#Nw&SAH6b4A$qi_yq@&sLz|WpI<{Ks+j5M zp>r!+VQ5{kFQwM$o01QqaCucFqFIA*V_!L*D88$C8^XA^Ptr4>g*1-igrLh~c~rD> zTu@>*LaKGz?XI=IDYNx6YwW~10Y0sIm@Y2K@4>*Asrg8Jy(!0J#GzxhUbul)3r&k{ zF<5kdyhz0^nPH18-e>8PsXOE#F#VjP9{(*_W@D*M*Z**qs%Njc(yl67d*r;hyM~?EGXhkTBp+T3NvG z>diCC5z@9n&!IF84I}b#Fv`Yzp@FSg%jwS5jn*Bv>%bBxM%_rI!->HBxkKK&-rQOXrt6Y9p z8=D&@?2q*-#KH9J&X+jSk~(5-gT!u6Dl=aWoQ; z8V)o_Y(41Y@FOuL`vib3!po4}nIVyoz!BX7_x|IAFkbI3KZ(Ac33Oh zYa6}b`FrU>!fbnUJ8hB6CkuRicoHss76sdej7bAdV`(;~q$}Tghwg_{ zcHI7&tNtg=4|SZtkG4cVe^LuRU>rIqI{I7_)ka{>SAkd=xB|@%kt~;Cao-;TT5jMR#6n+ngx32;-eeCYTpf%tnu_>48rxp#1y_30SkNZ?Fv0 z5-3}3VuE5M4Sy=VwOuxtm&Lv4bMo1+;I9Mr=X za{W|IoHySip`*3gd7HomQN`l$EwCnq5VRAg_2Nd*>&A*OE-J1kOpkuIzhyh=|2@Bg z_RzK+y5vNv?X|eVj^p_Qx|PqPNj7Tvs-nx^O5E$WdQhm$ScD)grt2i39kT94Zx-{Z zoQMhq%jwA+p4F62GarNJ1hjCgfg*I0VTt2h-cq9?8s1)-VAwujPZcaH7`TTuRLBXHb`h(!747*1__FHi&Qhb8CYgk~*enFMmF}!rqypYwdp13aHl}{< zXc5}E#Xym(Z(l5sBkma&Ul2ZJ#}aw)ClBX+`u9FVJI_YW>~=Pm@NjtH zUUhaE==l_yUOphmqw9;9d-ZT(@)ib}xsl3bnOF|&y;epQ6Q|=e&T_7|6R|U#ZGUER zaO>Qa4T&z!^IASt!_cl8ut<$N*5o%wJy?SZCZxD#yWLE3^9x;?`j$5M27F27H%4K> z%ohuVYYKUI9N2D?Qmb+wy8rY(iw^GmQ1et`C<6^ObvV~Fq@k~m+=4ziyRE+&v@cSN zFPkJ^SsxkBBJ6hp;B8}MUEyj88m}8s3~irD8pJW=F}fScrUNoj%^FW}Ku$?re-X-s z4}N8SWVGp~tflghwevV;)_ATR2m`)O1QeBLr$gnw4@K!n>a(lV_Xsq#T4}IXTzH5G zDq11Wx@8=XWrV=~fe}E5nlN{%%H(Nj2@D?$V7iD7g~J_n++$YPuu-A5*Yz*DFXWb} zwhb#>1ic=q0tfBFo#Dfu5wQUzeTf=p@WxWBl{>cu3My2uYxukyeGM7!ey?XQtL_$c zn_LelLISe_G51;z=DW&}@Wr5Dxi;N%F)zqTBIGov{VD@pyw5pbdcD|9pMQb=xbKw# zKfomL>9KY7Bt&|^Z-f_t(^rHEbJk>`BPLj_>Q*#q7!XZy-LF}3AR};Fm4OZSbA46l zw$dQFy5V5VZRRzzEZqF526X`CkWkjIRVU1XHi)GoM?4WY$g=tVMR<7g5Q2vy`A+ED zPE-5dT)GI9>7d1)!B$n(wl-n;qfX=zn)C2`u=T^M00T#=dx)4ML&AYJz?(bA# zs%=E;jWli}Zu5@X#jYf4LRMiC2qvW|-KkY#s5?iKl^MIxInL%Qa`$GU@Xiy5rF@_I z>dU<`a};2moZ z1GC&OFDp?5P@%_-m|hgEZ|J)jPBv1&!>?%%g=;8dD13zx?H6jZDZ4#i&bYCD5rN6q zP!IM9$zAllD)T4nlqdK9bj8*PWp(hm)q>{Ze2^T!1q!X2>v@0a8uBapm!s2t_5+!f ztsb(IQQxhE-@dit7(joKmi-fv5n7i;l`Z8W+ia;AyTB>1=dXTwmT$;rq5K`OC>6^3 zUhC8jc>cxBEkRY^r*mm-QcF`<2aFK}01F)nQM`2fW@LQ%#;0d&{$%g1>+CpOqpSgC zkVu!$KtxxfQmSaXu^BoIyNQ)>`Lwvu3L6(`2zbpu3ha&w&$`YPf3C-rCz^&7L{?^J zJilB<%a``R%qh z-iB-7&{?0uFxMc%_W>GV-{Z6=Yp+})aIiF$Inqb$g`YtQ?cB1Q1e43?v42>jWMDs? zf`25vw8>c1684fg*&oD3QLuZP+lip^mg-ZNA~LU~1aG^0A?Lveuw%hV3JHA=s$bVG z7g7!K>*%%vsxy!}Y_`|3O2ACaA9b}_jWPFqL;`TE4e3$HUx4Zw@^z+1MLp!~7-s@U zHlN?)pTo&r1>G2dGLyP1nr7Gduq{M?60l^@0=q6Bt}wDuP{^I6fafQ)>u=DD`;<6b z<)vlb1{OZ8!-?mb?)aSN{89V{g?H`Z#W^rvi#q z3wO|72t9sCzUe?zDDwMxD85sM^Q&QaeGD&Tu=bPswy=y3)+$`?bQ_CrfY(wj6`p`2rOYIl8eZrsZ*Kl?$Jh z$E9D6_q6loTtM}yr6Rs&qfHWCOLAvGxpEqLX|EKK)QX!vi9!$u58;sv(eqhYyWK;l z%WLSwRE4X>y}wswMkpmC6VVyWM5&j`Os2+~QJf>{Og2v4jE+^gT?5#f%FhgvHyEug z+I^!1$iQ}QMBt`|cH?3(sS-tO)ssEau9;7mZ=j%cu3FE#-Ew(Px;HaMRp!Rq*yj@4 z*o~Waoi(g>j)i|T<|NMD~&>8PxLm_d4)$Axz#N-B6OvA|hLDlA9UJ`bU zKx?^JUck1v?YeuMZG6&R?G3lZuQh()(;qC%`6ESQ=7SnFBfIR4n>S2On~v4h2vh}Q z6sqNH>!(9gmoR5%M~L*J!>O~|swNwH7QeJwk@qM1UwHFlN~LEy4+-1|KsTN|Rj5{5 zS2!>8Mr;n-60)A0GuLwfah zayleH)KCtS9u-jvOANk@cjUP4kaq|6ZBkty%RX{65*f|H?AR2p$|YP(iPW@Dx;Ic< zHD$*D9ZVwG;osF9-KT@HMt<$2XzUrd^I!JCSylry0l<_8R7sQ?@+u^O#GCK0#QQul zqf$K=%AaE+9ip%yCl^+@2sPkZG1GnLYqLg`kM-{lO^8eAPu#A%vJlMIx*}V8DBv!- z0X^#Ie&5wS`1G6q^ykqDs5S9u+10LXEdVu5N_q;T|hN9S|YFq zonY+|*a)sW9CMi)Y8+p(QhVm0t2OaERRw+f1*U%bd~`qFrI%7e4%*#Ec7Jkx(6eM2Cj3@c*n#66)!cB=Hn2LXpucGA85lE;k(4FL99wO>qa^Y5H4xBl>WP8ACc zdMjJZGCtQo*+N7#)+V(B?sw7Z2dim}W^Iw&f^U;!GnTLvPcbGctONguWMraAkF{yY zY!eOiS)RA9Y!ex;SM{1==B}nqOjlG8gE+nzPx;4HW_Ip-+pIiJ8mc}*>KY7M79$ac z9hfyKs8G>6%1D=%(_1Q{?j8McjV@~bV1Cy>iCJM}do63XXN@})Jt*JXru;B{f3$y) zVtG=RR!dx~-$Qj>;V|u3Ol#sXeK$jKMnZB+26Ill%tgWhJY^hNEbCTy(tDTo6|(MV zfTDTzxry(8rm65&0DhbhBte;LVT?An&Ynpn)bnj?Dme+9sBG^r(!>X zz0NSS(@GJc&&IWLnAToMRG$Zb&_qtE`{nr2aQH*lKMwt-X^^XhB$kXx)|sri{(BEDZA z*O1s)ik`lrL5O@&M#|gLD|bMJMo&^OoP#!Jflmo3=C|YkdcLQo>@v0Mzr+HlJu{K# zc1!@jl%l?*K(8HGW&hYylgnp%4Vghn+e5W~6%mmT4IXt?#i0aBCQ!=5ub?&zr==R+v=y=|6*CCFN<97VE@yNCG?;FuAJN*EtDV~ zWM~O@jqhtuN=xW7(`G)oJc_&+jbp0(TdRva+li(gctN<7*X1ROa9qy-WtqH1h_zM~ zb$kuX$be&Gpjx^4nesbjCL0NVINtYkrh)-Ig8%NgZ)71vrsGs963GNA@QPQ*H)=AVAp$l(;Ga_X zzYmlGg>vzr6R)y^rej$xPaYN3G0g%eRle#KSq7Emk$o>6eJkTf6QXETed*WT)(KPP zPa`6q+%XNz8p1T}W4c;nR%DeH#mFe4nD+yS;2hchaQGyzm)MY*A3Us2`SI1iRejNt z*wBV6(Gs~7;?zYvO3@lnWd@{RK%K8q#qfhano^CwvH?%L)*kw>XJQJGCd)>&PSfVA z;bHT&xkOEQtW%_+S4j#5`OBd{h!T??)11PBcOuIt+-_y_Z+JUI=;mFV_5%NB$HFy1 zo{?1e*jXkAnBiIU9RwAQM$3E|-44y-E>A>y$F0?Z{( zja8y1t5laaTbCgZwEW~q*}q#`?94|QTJ|-XwiC5y<;9pOJ6Dk zbuvq`D6DPVtv=_JOFPErzs$rk7sk}DpTMrTdPdLa9@+juZEJ63QyMm(EkDq+AxyR6`P0?`=ac)hCFok z81Jdgm3U2YSutQ0MT6H1g?Pd@I@g!hDE{l)BIFXNXH0R?V|2;wrh@h#H!RExV@|43 zzUBhYP`4$#@`-%BzxL7SRlw^M&B=M7(P2nA#R>WfKqY&U^kB9R?6b)Bd{+*49U`6H zgWKh!SL&ul6(1v}hLK@sdsIRaATEUf{vuSpE93sg(`ab=zl@E?E)2_Nt7 z-Z)&$>t<7meMW_-*e<%sw)sD2&cD;!2ZtqS$bk1^J4Nj(MWREGH@~KUb5gjj{+ur& zy7aR*rbv24sX5hcU>X_malS=vbWPUin+&YQ9gt#Zh#4J^P>90P^5I z9sMG;80vMgTi34LAdr44s)~WN%(k$6LV1``T-_JJ$PcDyfLD5!WfY1q0#8;A{TC$Y z?!@wW$6JpV>n{;5cP3l(9>5n)&AU;EtTj47ylCosjtWKbD`5Bx=z9N6;K|)lX_A}+ zXN~7uOPtd;OsBHV!3+yh7hBBnab); z<0}E^Ioa4GlzG9aY8??^dMp%S(Zo7qnT7RqLzH>zNpPd}8ux$7rftPs#HM>{1Gc1GiTLw`KL@#FzCtnNy-4_Eg*0Cts{bQWprQA z3X`1tiJAFdbpNh-F~G)ab>!IA-Wj2RUq>Y^E&a>Bno6*BS`D8@N?ImdsQMUr{j<1P z1lMysD%cga7LMEdW&3QZ2*eBhZ3lu)LN7*2&ntJccORQUHofLyq;oAHas2)~ zqQBTVL2#OpU#VR22K#%e=467GBCQf1O06}!nPOxGB3CWu?39c+BrkoOE>ZbNg{5Y#K1%<>*cm{4SC`Q zZk=;Sv%G%S??SLL3HidyXZcGxb$ziRsgSXjXY5FqBSY#SWEF0a;$5}4 zjPEpZpOpwxe`Zd>{(PngxT`PMA=Qn^j9|RL+`RC(wXNH#2lOp^o!NwYblp~mi!uC{ zUW8T{{lI!G%Ya=9%UOd#+((DXLt`Zh%mUzDxVaE}>V)dV!DoCTwq-IFG?wwHL{y@a z*f5#jd^2^oS^v)-e#YMCNj5-PDA`cFX#%v{v+*eglqfW}%!O%&L0;E%!(QlewLn@q zcHf5GL4TrYIk1|DikEr>OX)p2ep|1Fn0V#!R-um!VpsW$M{U~6YdOE zK6LqNMRe*#AB%5T5>Vf}N;mo#Q`2)(PMa4@hTx`gN%?@eX@_%EQMW&3*DlyIPB~zk zU}_5$P^F(DI3m&FH?#5BIB2gl2;cOYORKJ|m)zz0FffAU?eWuYE?kE!++cM-ihy`I zbjPj$KWS%La@m^-g+`P6mnE0w(RYFWpr+R#Joj74!dUUWcfRg2cCJP-^4&<#i(po< zjqh&|1Z{{#pEJ&>4Pl1&!MAMtNtA_hjKUg9z-lZ!S7sa5wfCR zQ|ndR@Jey29E|9Qd+9coSVG+_YYh0){3K>`LEVDoC8n2n!P)-r2Kg~6u|>$X#op!U ztHS!(RkN7aZ+TMxQRKE1ASgRUsMSs$_8!tpT+4?1C> za#fD0xvbV9Jh)Xq;oPm^%#yR*>^_to2AV87!ov&>-h3^(f$9q$B;LClMS3RYEl$nO z1RR&e9SAxMkXJFGkv=|qQ*&oiAxNwD@k--BLy325%dDN*r-oX!%RQevryJ&J|4H{l z&_6|i$QK3(n$wnX%1SukgXZO8Sqbi_?I=5ambrm(FL0JHektBK3@iM73CoVFUT~ee zvEkp*1b+jyFD<0APOk5iLao&;OPD*OU+J7<8`z8=b$^W?i z@00MenE3zeg18!crcp0V0*uE&i=fs5ji$_oYN2VqU&r5ORo7?oq@4;)HR@1a{;T-Q zFb8?^<}|G=${cAK4xdHz%i2dc2=0dbcZ?6%8`7SAIgGXD*p0pf(7~gkDg!&wlPPbO vs~-6qX7HqjYYeu}M(_QtL8Z(grx>gu&R?J9Z@j+-ex$_}o)$iN?)U!y*3y2w From 7c99c2b6eae1159857631c100f5342184307c027 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 May 2014 14:53:38 +0100 Subject: [PATCH 0174/1103] Use Makefile instead --- CONTRIBUTING.rst | 27 +++-- Makefile | 95 +++++++++++++++ README.rst | 4 +- docs/contributing.rst | 25 ++-- extra/release/sphinx-to-rst.py | 13 ++- pavement.py | 205 --------------------------------- requirements/pkgutils.txt | 1 - 7 files changed, 130 insertions(+), 240 deletions(-) create mode 100644 Makefile delete mode 100644 pavement.py diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6a248b428..d76671e02 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -271,7 +271,7 @@ semver: http://semver.org. Stable releases are published at PyPI while development releases are only available in the GitHub git repository as tags. -All version tags starts with "v", so version 0.8.0 is the tag v0.8.0. +All version tags starts with “v”, so version 0.8.0 is the tag v0.8.0. .. _git-branches: @@ -580,13 +580,13 @@ To ensure that your changes conform to PEP8 and to run pyflakes execute: :: - $ paver flake8 + $ make flakecheck -To not return a negative exit code when this command fails use the -``-E`` option, this can be convenient while developing: +To not return a negative exit code when this command fails use +the ``flakes`` target instead: :: - $ paver flake8 -E + $ make flakes§ API reference ~~~~~~~~~~~~~ @@ -595,8 +595,8 @@ To make sure that all modules have a corresponding section in the API reference please execute: :: - $ paver autodoc - $ paver verifyindex + $ make apicheck + $ make indexcheck If files are missing you can add them by copying an existing reference file. @@ -812,7 +812,7 @@ that require 3rd party libraries must be added. :: $ pip install -U requirements/pkgutils.txt - $ paver readme + $ make readme That's all that needs to be done, but remember that if your feature @@ -1013,11 +1013,11 @@ The version number must be updated two places: After you have changed these files you must render the ``README`` files. There is a script to convert sphinx syntax -to generic reStructured Text syntax, and the paver task `readme` +to generic reStructured Text syntax, and the make target `readme` does this for you: :: - $ paver readme + $ make readme Now commit the changes: :: @@ -1035,10 +1035,9 @@ Releasing Commands to make a new public stable release:: - $ paver releaseok # checks pep8, autodoc index, runs tests and more - $ paver removepyc # Remove .pyc files - $ git clean -xdn # Check that there's no left-over files in the repo - $ python setup.py sdist upload # Upload package to PyPI + $ make distcheck # checks pep8, autodoc index, runs tests and more + $ make dist # NOTE: Runs git clean -xdf and removes files not in the repo. + $ python setup.py sdist bdist_wheel upload # Upload package to PyPI If this is a new release series then you also need to do the following: diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..98557216e --- /dev/null +++ b/Makefile @@ -0,0 +1,95 @@ +PYTHON=python +SPHINX_DIR="docs/" +SPHINX_BUILDDIR="${SPHINX_DIR}/.build" +README="README.rst" +CONTRIBUTING="CONTRIBUTING.rst" +CONFIGREF_SRC="docs/configuration.rst" +README_SRC="docs/templates/readme.txt" +CONTRIBUTING_SRC="docs/contributing.rst" +SPHINX2RST="extra/release/sphinx-to-rst.py" +WORKER_GRAPH_FULL="docs/images/worker_graph_full.png" + +SPHINX_HTMLDIR = "${SPHINX_BUILDDIR}/html" + +html: + (cd "$(SPHINX_DIR)"; make html) + mv "$(SPHINX_HTMLDIR)" Documentation + +docsclean: + -rm -rf "$(SPHINX_BUILDDIR)" + +htmlclean: + -rm -rf "$(SPHINX)" + +apicheck: + extra/release/doc4allmods celery + +indexcheck: + extra/release/verify-reference-index.sh + +configcheck: + PYTHONPATH=. $(PYTHON) extra/release/verify_config_reference.py $(CONFIGREF_SRC) + +flakecheck: + flake8 celery + +flakediag: + -$(MAKE) flakecheck + +flakepluscheck: + flakeplus celery --2.6 + +flakeplusdiag: + -$(MAKE) flakepluscheck + +flakes: flakediag flakeplusdiag + +readmeclean: + -rm -f $(README) + +readmecheck: + iconv -f ascii -t ascii $(README) >/dev/null + +$(README): + $(PYTHON) $(SPHINX2RST) $(README_SRC) --ascii > $@ + +readme: readmeclean $(README) readmecheck + +contributingclean: + -rm -f CONTRIBUTING.rst + +$(CONTRIBUTING): + $(PYTHON) $(SPHINX2RST) $(CONTRIBUTING_SRC) > $@ + +contributing: contributingclean $(CONTRIBUTING) + +test: + nosetests -xv celery.tests + +cov: + nosetests -xv celery.tests --with-coverage --cover-html --cover-branch + +removepyc: + -find . -type f -a \( -name "*.pyc" -o -name "*$$py.class" \) | xargs rm + -find . -type d -name "__pycache__" | xargs rm -r + +$(WORKER_GRAPH_FULL): + $(PYTHON) -m celery graph bootsteps | dot -Tpng -o $@ + +graphclean: + -rm -f $(WORKER_GRAPH_FULL) + +graph: graphclean $(WORKER_GRAPH_FULL) + +gitclean: + git clean -xdn + +gitcleanforce: + git clean -xdf + +distcheck: flakecheck apicheck indexcheck configcheck readmecheck test gitclean + +authorcheck: + git shortlog -se | cut -f2 | extra/release/attribution.py + +dist: readme contributing docsclean gitcleanforce removepyc diff --git a/README.rst b/README.rst index 392965271..1e9b64ffb 100644 --- a/README.rst +++ b/README.rst @@ -82,7 +82,7 @@ getting started tutorials: http://docs.celeryproject.org/en/latest/getting-started/next-steps.html Celery is... -============ +========== - **Simple** @@ -120,7 +120,7 @@ Celery is... schedulers, consumers, producers, autoscalers, broker transports and much more. It supports... -============== +============ - **Message Transports** diff --git a/docs/contributing.rst b/docs/contributing.rst index 0bb7693de..ce8c3efb5 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -596,14 +596,14 @@ execute: .. code-block:: bash - $ paver flake8 + $ make flakecheck -To not return a negative exit code when this command fails use the -:option:`-E` option, this can be convenient while developing: +To not return a negative exit code when this command fails use +the ``flakes`` target instead: .. code-block:: bash - $ paver flake8 -E + $ make flakes§ API reference ~~~~~~~~~~~~~ @@ -613,8 +613,8 @@ reference please execute: .. code-block:: bash - $ paver autodoc - $ paver verifyindex + $ make apicheck + $ make indexcheck If files are missing you can add them by copying an existing reference file. @@ -841,7 +841,7 @@ that require 3rd party libraries must be added. .. code-block:: bash $ pip install -U requirements/pkgutils.txt - $ paver readme + $ make readme That's all that needs to be done, but remember that if your feature @@ -1042,12 +1042,12 @@ The version number must be updated two places: After you have changed these files you must render the :file:`README` files. There is a script to convert sphinx syntax -to generic reStructured Text syntax, and the paver task `readme` +to generic reStructured Text syntax, and the make target `readme` does this for you: .. code-block:: bash - $ paver readme + $ make readme Now commit the changes: @@ -1067,10 +1067,9 @@ Releasing Commands to make a new public stable release:: - $ paver releaseok # checks pep8, autodoc index, runs tests and more - $ paver removepyc # Remove .pyc files - $ git clean -xdn # Check that there's no left-over files in the repo - $ python setup.py sdist upload # Upload package to PyPI + $ make distcheck # checks pep8, autodoc index, runs tests and more + $ make dist # NOTE: Runs git clean -xdf and removes files not in the repo. + $ python setup.py sdist bdist_wheel upload # Upload package to PyPI If this is a new release series then you also need to do the following: diff --git a/extra/release/sphinx-to-rst.py b/extra/release/sphinx-to-rst.py index 35417a05f..d9b5c0d9c 100755 --- a/extra/release/sphinx-to-rst.py +++ b/extra/release/sphinx-to-rst.py @@ -138,7 +138,7 @@ def resolve_pending_refs(lines): RE_INCLUDE: include_file} -def _process(lines): +def _process(lines, encoding='utf-8'): lines = list(lines) # non-destructive for i, line in enumerate(lines): for regex, alt in TO_RST_MAP.items(): @@ -150,18 +150,21 @@ def _process(lines): else: lines[i] = regex.sub(alt, line) lines[i] = deref_all(lines[i]) - return resolve_pending_refs(asciify(lines)) + if encoding == 'ascii': + lines = asciify(lines) + return resolve_pending_refs(lines) -def sphinx_to_rst(fh): - return ''.join(_process(fh)) +def sphinx_to_rst(fh, encoding='utf-8'): + return ''.join(_process(fh, encoding)) if __name__ == '__main__': global dirname dirname = os.path.dirname(sys.argv[1]) + encoding = 'ascii' if '--ascii' in sys.argv else 'utf-8' fh = codecs.open(sys.argv[1], encoding='utf-8') try: - print(sphinx_to_rst(fh)) + print(sphinx_to_rst(fh, encoding).encode('utf-8')) finally: fh.close() diff --git a/pavement.py b/pavement.py deleted file mode 100644 index dd7916acd..000000000 --- a/pavement.py +++ /dev/null @@ -1,205 +0,0 @@ -from __future__ import print_function, unicode_literals - -import sys -import traceback - -from paver.easy import task, sh, cmdopts, path, needs, options, Bunch -from paver import doctools # noqa -from paver.setuputils import setup # noqa - -PYCOMPILE_CACHES = ['*.pyc', '*$py.class'] - -options( - sphinx=Bunch(builddir='.build'), -) - - -def sphinx_builddir(options): - return path('docs') / options.sphinx.builddir / 'html' - - -@task -def clean_docs(options): - sphinx_builddir(options).rmtree() - - -@task -@needs('clean_docs', 'paver.doctools.html') -def html(options): - destdir = path('Documentation') - destdir.rmtree() - builtdocs = sphinx_builddir(options) - builtdocs.move(destdir) - - -@task -@needs('paver.doctools.html') -def qhtml(options): - destdir = path('Documentation') - builtdocs = sphinx_builddir(options) - sh('rsync -az {0}/ {1}'.format(builtdocs, destdir)) - - -@task -def autodoc(options): - sh('extra/release/doc4allmods celery') - - -@task -def verifyindex(options): - sh('extra/release/verify-reference-index.sh') - - -@task -def verifyconfigref(options): - sh('PYTHONPATH=. {0} extra/release/verify_config_reference.py \ - docs/configuration.rst'.format(sys.executable)) - - -@task -@cmdopts([ - ('noerror', 'E', 'Ignore errors'), -]) -def flake8(options): - noerror = getattr(options, 'noerror', False) - complexity = getattr(options, 'complexity', 22) - sh("""flake8 celery | perl -mstrict -mwarnings -nle' - my $ignore = m/too complex \((\d+)\)/ && $1 le {0}; - if (! $ignore) {{ print STDERR; our $FOUND_FLAKE = 1 }} - }}{{exit $FOUND_FLAKE; - '""".format(complexity), ignore_error=noerror) - - -@task -@cmdopts([ - ('noerror', 'E', 'Ignore errors'), -]) -def flakeplus(options): - noerror = getattr(options, 'noerror', False) - sh('flakeplus celery --2.6', ignore_error=noerror) - - -@task -@cmdopts([ - ('noerror', 'E', 'Ignore errors') -]) -def flakes(options): - flake8(options) - flakeplus(options) - - -@task -def clean_readme(options): - path('README').unlink_p() - path('README.rst').unlink_p() - - -@task -def clean_contributing(options): - path('CONTRIBUTING.rst').unlink_p() - - -@task -def verify_readme(options): - with open('README.rst') as fp: - try: - fp.read().encode('ascii') - except Exception: - print('README contains non-ascii characters', file=sys.stderr) - print('Original exception below...', file=sys.stderr) - traceback.print_stack(file=sys.stderr) - sh('false') - - -@task -@needs('clean_readme') -def readme(options): - sh('{0} extra/release/sphinx-to-rst.py docs/templates/readme.txt \ - > README.rst'.format(sys.executable)) - verify_readme() - - -@task -@needs('clean_contributing') -def contributing(options): - sh('{0} extra/release/sphinx-to-rst.py docs/contributing.rst \ - > CONTRIBUTING.rst'.format(sys.executable)) - - -@task -def bump(options): - sh("extra/release/bump_version.py \ - celery/__init__.py docs/includes/introduction.txt \ - --before-commit='paver readme'") - - -@task -@cmdopts([ - ('coverage', 'c', 'Enable coverage'), - ('verbose', 'V', 'Make more noise'), -]) -def test(options): - cmd = 'CELERY_LOADER=default nosetests' - if getattr(options, 'coverage', False): - cmd += ' --with-coverage' - if getattr(options, 'verbose', False): - cmd += ' --verbosity=2' - sh(cmd) - - -@task -@cmdopts([ - ('noerror', 'E', 'Ignore errors'), -]) -def pep8(options): - noerror = getattr(options, 'noerror', False) - return sh("""find . -name "*.py" | xargs pep8 | perl -nle'\ - print; $a=1 if $_}{exit($a)'""", ignore_error=noerror) - - -@task -def removepyc(options): - sh('find . -type f -a \\( {0} \\) | xargs rm'.format( - ' -o '.join("-name '{0}'".format(pat) for pat in PYCOMPILE_CACHES))) - sh('find . -type d -name "__pycache__" | xargs rm -r') - - -@task -def update_graphs(options, dest='docs/images/worker_graph_full.png'): - sh('celery graph bootsteps | dot -Tpng -o {dest}'.format( - dest=dest, - )) - - -@task -@needs('removepyc') -def gitclean(options): - sh('git clean -xdn') - - -@task -@needs('removepyc') -def gitcleanforce(options): - sh('git clean -xdf') - - -@task -@needs('flakes', 'autodoc', 'verifyindex', - 'verifyconfigref', 'verify_readme', 'test', 'gitclean') -def releaseok(options): - pass - - -@task -def verify_authors(options): - sh('git shortlog -se | cut -f2 | extra/release/attribution.py') - - -@task -def testloc(options): - sh('sloccount celery/tests') - - -@task -def loc(options): - sh('sloccount celery') diff --git a/requirements/pkgutils.txt b/requirements/pkgutils.txt index de2162e30..35cd96010 100644 --- a/requirements/pkgutils.txt +++ b/requirements/pkgutils.txt @@ -1,6 +1,5 @@ setuptools>=1.3.2 wheel -paver flake8 flakeplus tox From 062c046907d84439e3d6a838e60d7fbc07bf28f3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 May 2014 14:57:38 +0100 Subject: [PATCH 0175/1103] Copyright 2014 --- LICENSE | 2 +- celery/__init__.py | 2 +- docs/conf.py | 4 ++-- docs/copyright.rst | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/LICENSE b/LICENSE index fc1793197..aeb3da0c0 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,5 @@ Copyright (c) 2009, 2010, 2011, 2012 Ask Solem, and individual contributors. All Rights Reserved. -Copyright (c) 2012-2013 GoPivotal, Inc. All rights reserved. +Copyright (c) 2012-2014 GoPivotal, Inc. All rights reserved. Celery is licensed under The BSD License (3 Clause, also known as the new BSD license). The license is an OSI approved Open Source diff --git a/celery/__init__.py b/celery/__init__.py index 6ec3a4e77..67355fbb5 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -2,7 +2,7 @@ """Distributed Task Queue""" # :copyright: (c) 2009 - 2012 Ask Solem and individual contributors, # All rights reserved. -# :copyright: (c) 2012-2013 GoPivotal, Inc., All rights reserved. +# :copyright: (c) 2012-2014 GoPivotal, Inc., All rights reserved. # :license: BSD (3 Clause), see LICENSE for more details. from __future__ import absolute_import, print_function, unicode_literals diff --git a/docs/conf.py b/docs/conf.py index 2cee3992a..4a1c3d6a4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -54,7 +54,7 @@ def linkcode_resolve(domain, info): # General information about the project. project = 'Celery' -copyright = '2009-2013, Ask Solem & Contributors' +copyright = '2009-2014, Ask Solem & Contributors' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -126,7 +126,7 @@ def linkcode_resolve(domain, info): epub_title = 'Celery Manual, Version {0}'.format(version) epub_author = 'Ask Solem' epub_publisher = 'Celery Project' -epub_copyright = '2009-2013' +epub_copyright = '2009-2014' # The language of the text. It defaults to the language option # or en if the language is not set. diff --git a/docs/copyright.rst b/docs/copyright.rst index c7b95e8d0..bfffb3019 100644 --- a/docs/copyright.rst +++ b/docs/copyright.rst @@ -7,7 +7,7 @@ by Ask Solem .. |copy| unicode:: U+000A9 .. COPYRIGHT SIGN -Copyright |copy| 2009-2013, Ask Solem. +Copyright |copy| 2009-2014, Ask Solem. All rights reserved. This material may be copied or distributed only subject to the terms and conditions set forth in the `Creative Commons From 5fdb5b5349a7efd3d44c657f23429953ae5b6357 Mon Sep 17 00:00:00 2001 From: Roger Hu Date: Wed, 21 May 2014 06:02:14 +0000 Subject: [PATCH 0176/1103] Use a different function to demonstrate AssertionError. add() statements may take longer than general other commands. Conflicts: funtests/stress/stress/suite.py --- funtests/stress/stress/app.py | 2 +- funtests/stress/stress/suite.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/funtests/stress/stress/app.py b/funtests/stress/stress/app.py index 077437d89..7c14a15cf 100644 --- a/funtests/stress/stress/app.py +++ b/funtests/stress/stress/app.py @@ -110,7 +110,7 @@ def retries(self): @app.task -def unicode(): +def print_unicode(): print('hiöäüß') diff --git a/funtests/stress/stress/suite.py b/funtests/stress/stress/suite.py index 134850409..2d8a1f67b 100755 --- a/funtests/stress/stress/suite.py +++ b/funtests/stress/stress/suite.py @@ -19,7 +19,7 @@ from .app import ( marker, _marker, add, any_, exiting, kill, sleeping, - sleeping_ignore_limits, any_returning, + sleeping_ignore_limits, any_returning, print_unicode ) from .data import BIG, SMALL from .fbi import FBI @@ -103,11 +103,11 @@ def init_groups(self): except AttributeError: pass else: - for group in groups: - acc[group].append(meth) + for g in groups: + acc[g].append(meth) # sort the tests by the order in which they are defined in the class - for group in values(acc): - group[:] = sorted(group, key=lambda m: m.__func__.__testsort__) + for g in values(acc): + g[:] = sorted(g, key=lambda m: m.__func__.__testsort__) self.groups = dict( (name, testgroup(*tests)) for name, tests in items(acc) ) @@ -236,6 +236,8 @@ def dump_progress(self): _creation_counter = count(0) + + def testcase(*groups): if not groups: raise ValueError('@testcase requires at least one group name') @@ -262,7 +264,7 @@ class Suite(BaseSuite): @testcase('all', 'green') def manyshort(self): - self.join(group(add.s(i, i) for i in range(1000))(), + self.join(group(print_unicode.s(i, i) for i in range(1000))(), timeout=10, propagate=True) @testcase('all') From 8753e2c52e0c4d27abc3c0587a8ec63abc1f6ae8 Mon Sep 17 00:00:00 2001 From: Roger Hu Date: Thu, 22 May 2014 20:50:33 +0000 Subject: [PATCH 0177/1103] Update README.rst file to include CSTRESS_BROKER. --- funtests/stress/README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/funtests/stress/README.rst b/funtests/stress/README.rst index 9a55a9192..7c91b24b6 100644 --- a/funtests/stress/README.rst +++ b/funtests/stress/README.rst @@ -153,6 +153,13 @@ See ``python -m stress --help`` for a list of all available options. Options ======= +Using a different broker +------------------------ +You can set the environment ``CSTRESS_BROKER`` to change the broker used:: + + $ CSTRESS_BROKER='amqp://' celery -A stress worker # … + $ CSTRESS_BROKER='amqp://' python -m stress + Using a different result backend -------------------------------- From 75e5acca9481cc972d48493b39bc0812181037be Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Sun, 25 May 2014 19:05:20 +0100 Subject: [PATCH 0178/1103] eventlet/gevent: Use kombu.async.timer directly --- celery/concurrency/eventlet.py | 28 ++++++++-------------------- celery/concurrency/gevent.py | 19 +++---------------- 2 files changed, 11 insertions(+), 36 deletions(-) diff --git a/celery/concurrency/eventlet.py b/celery/concurrency/eventlet.py index 613b28a86..2371e4829 100644 --- a/celery/concurrency/eventlet.py +++ b/celery/concurrency/eventlet.py @@ -28,9 +28,10 @@ import warnings warnings.warn(RuntimeWarning(W_RACE % side)) +from kombu.async import timer as _timer + from celery import signals -from celery.utils import timer2 from . import base @@ -41,12 +42,12 @@ def apply_target(target, args=(), kwargs={}, callback=None, pid=getpid()) -class Schedule(timer2.Schedule): +class Timer(_timer.Timer): def __init__(self, *args, **kwargs): from eventlet.greenthread import spawn_after from greenlet import GreenletExit - super(Schedule, self).__init__(*args, **kwargs) + super(Timer, self).__init__(*args, **kwargs) self.GreenletExit = GreenletExit self._spawn_after = spawn_after @@ -81,28 +82,15 @@ def clear(self): except (KeyError, self.GreenletExit): pass - @property - def queue(self): - return self._queue - - -class Timer(timer2.Timer): - Schedule = Schedule - - def ensure_started(self): - pass - - def stop(self): - self.schedule.clear() - def cancel(self, tref): try: tref.cancel() - except self.schedule.GreenletExit: + except self.GreenletExit: pass - def start(self): - pass + @property + def queue(self): + return self._queue class TaskPool(base.BasePool): diff --git a/celery/concurrency/gevent.py b/celery/concurrency/gevent.py index f89de92b2..0574178c9 100644 --- a/celery/concurrency/gevent.py +++ b/celery/concurrency/gevent.py @@ -15,7 +15,7 @@ except ImportError: # pragma: no cover Timeout = None # noqa -from celery.utils import timer2 +from kombu.async import timer as _timer from .base import apply_target, BasePool @@ -35,7 +35,7 @@ def apply_timeout(target, args=(), kwargs={}, callback=None, return timeout_callback(False, timeout) -class Schedule(timer2.Schedule): +class Timer(_timer.Timer): def __init__(self, *args, **kwargs): from gevent.greenlet import Greenlet, GreenletExit @@ -45,7 +45,7 @@ class _Greenlet(Greenlet): self._Greenlet = _Greenlet self._GreenletExit = GreenletExit - super(Schedule, self).__init__(*args, **kwargs) + super(Timer, self).__init__(*args, **kwargs) self._queue = set() def _enter(self, eta, priority, entry): @@ -78,19 +78,6 @@ def queue(self): return self._queue -class Timer(timer2.Timer): - Schedule = Schedule - - def ensure_started(self): - pass - - def stop(self): - self.schedule.clear() - - def start(self): - pass - - class TaskPool(BasePool): Timer = Timer From 6f2621e55b0e2df6d9c5ecc937040d4318fbda3e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 27 May 2014 14:13:33 +0100 Subject: [PATCH 0179/1103] Docs requirements now depends on amqp master. Closes #2060 --- requirements/docs.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/docs.txt b/requirements/docs.txt index 3854f9edf..1d142c296 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,2 +1,3 @@ Sphinx SQLAlchemy +https://github.com/celery/py-amqp/zipball/master#egg=amqp From 4e52e8f1f51cc43113c649624ffd86ba5064fa35 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 27 May 2014 14:28:14 +0100 Subject: [PATCH 0180/1103] Fixes flakes after flake8 update --- celery/app/log.py | 4 ++-- celery/concurrency/asynpool.py | 8 +++----- celery/five.py | 2 +- celery/loaders/base.py | 4 ++-- celery/task/base.py | 2 +- celery/tests/backends/test_amqp.py | 14 ++++++++------ celery/tests/bin/test_worker.py | 12 +++++++++--- celery/tests/case.py | 2 +- celery/tests/compat_modules/test_sets.py | 4 ++-- celery/worker/consumer.py | 2 +- celery/worker/loops.py | 2 -- celery/worker/pidbox.py | 5 +++-- celery/worker/request.py | 8 +++++--- docs/_ext/applyxrefs.py | 2 -- docs/conf.py | 19 +++++++------------ examples/app/myapp.py | 7 ++++--- examples/django/proj/__init__.py | 2 ++ examples/eventlet/celeryconfig.py | 2 +- examples/gevent/celeryconfig.py | 2 +- examples/next-steps/setup.py | 2 +- funtests/benchmarks/bench_worker.py | 5 +++-- funtests/stress/stress/fbi.py | 1 - funtests/stress/stress/templates.py | 5 ++--- 23 files changed, 59 insertions(+), 57 deletions(-) diff --git a/celery/app/log.py b/celery/app/log.py index 9dde09b16..2ccf7284b 100644 --- a/celery/app/log.py +++ b/celery/app/log.py @@ -176,8 +176,8 @@ def setup_task_loggers(self, loglevel=None, logfile=None, format=None, formatter=TaskFormatter, **kwargs ) logger.setLevel(loglevel) - logger.propagate = int(propagate) # this is an int for some reason. - # better to not question why. + # this is an int for some reason, better to not question why. + logger.propagate = int(propagate) signals.after_setup_task_logger.send( sender=None, logger=logger, loglevel=loglevel, logfile=logfile, diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 5dd6ac815..509823082 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -595,7 +595,7 @@ def _create_write_handlers(self, hub, active_writers = self._active_writers busy_workers = self._busy_workers diff = all_inqueues.difference - add_reader, add_writer = hub.add_reader, hub.add_writer + add_writer = hub.add_writer hub_add, hub_remove = hub.add, hub.remove mark_write_fd_as_active = active_writes.add mark_write_gen_as_active = active_writers.add @@ -638,8 +638,8 @@ def _put_back(job, _time=time.time): def on_poll_start(): if outbound and len(busy_workers) < len(all_inqueues): - #print('ALL: %r ACTIVE: %r' % (len(all_inqueues), - # len(active_writes))) + # print('ALL: %r ACTIVE: %r' % (len(all_inqueues), + # len(active_writes))) inactive = diff(active_writes) [hub_add(fd, None, WRITE | ERR, consolidate=True) for fd in inactive] @@ -1134,8 +1134,6 @@ def on_partial_read(self, job, proc): self._queues[self.create_process_queues()] = None except ValueError: pass - # Not in queue map, make sure sockets are closed. - #self.destroy_queues((proc.inq, proc.outq, proc.synq)) assert len(self._queues) == before def destroy_queues(self, queues, proc): diff --git a/celery/five.py b/celery/five.py index 94a4ab8ca..4418a4ac3 100644 --- a/celery/five.py +++ b/celery/five.py @@ -20,7 +20,7 @@ __all__ += _all_five -############## Module Generation ########################## +# ############# Module Generation ########################## # Utilities to dynamically # recreate modules, either for lazy loading or diff --git a/celery/loaders/base.py b/celery/loaders/base.py index 533530b1d..cf89ba2b5 100644 --- a/celery/loaders/base.py +++ b/celery/loaders/base.py @@ -190,12 +190,12 @@ def getarg(arg): """Parse a single configuration definition from the command-line.""" - ## find key/value + # ## find key/value # ns.key=value|ns_key=value (case insensitive) key, value = arg.split('=', 1) key = key.upper().replace('.', '_') - ## find namespace. + # ## find namespace. # .key=value|_key=value expands to default namespace. if key[0] == '_': ns, key = namespace, key[1:] diff --git a/celery/task/base.py b/celery/task/base.py index a47885aeb..d8ff5e880 100644 --- a/celery/task/base.py +++ b/celery/task/base.py @@ -39,7 +39,7 @@ class Task(BaseTask): __bound__ = False __v2_compat__ = True - #- Deprecated compat. attributes -: + # - Deprecated compat. attributes -: queue = None routing_key = None diff --git a/celery/tests/backends/test_amqp.py b/celery/tests/backends/test_amqp.py index 23a6c46d7..37847a8f9 100644 --- a/celery/tests/backends/test_amqp.py +++ b/celery/tests/backends/test_amqp.py @@ -271,12 +271,14 @@ def test_get_many(self): tids.append(tid) res = list(b.get_many(tids, timeout=1)) - expected_results = [(tid, {'status': states.SUCCESS, - 'result': i, - 'traceback': None, - 'task_id': tid, - 'children': None}) - for i, tid in enumerate(tids)] + expected_results = [ + (_tid, {'status': states.SUCCESS, + 'result': i, + 'traceback': None, + 'task_id': _tid, + 'children': None}) + for i, _tid in enumerate(tids) + ] self.assertEqual(sorted(res), sorted(expected_results)) self.assertDictEqual(b._cache[res[0][0]], res[0][1]) cached_res = list(b.get_many(tids, timeout=1)) diff --git a/celery/tests/bin/test_worker.py b/celery/tests/bin/test_worker.py index 864271172..c6d6033af 100644 --- a/celery/tests/bin/test_worker.py +++ b/celery/tests/bin/test_worker.py @@ -207,7 +207,10 @@ def test_startup_info(self): # test when there are too few output lines # to draft the ascii art onto prev, cd.ARTLINES = cd.ARTLINES, ['the quick brown fox'] - self.assertTrue(worker.startup_info()) + try: + self.assertTrue(worker.startup_info()) + finally: + cd.ARTLINES = prev @disable_stdouts def test_run(self): @@ -323,8 +326,11 @@ def test_setup_logging_no_color(self): app=self.app, redirect_stdouts=False, no_color=True, ) prev, self.app.log.setup = self.app.log.setup, Mock() - worker.setup_logging() - self.assertFalse(self.app.log.setup.call_args[1]['colorize']) + try: + worker.setup_logging() + self.assertFalse(self.app.log.setup.call_args[1]['colorize']) + finally: + self.app.log.setup = prev @disable_stdouts def test_startup_info_pool_is_str(self): diff --git a/celery/tests/case.py b/celery/tests/case.py index a05c8c5a1..09108670d 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -235,7 +235,7 @@ def _is_magic_module(m): # pyflakes refuses to accept 'noqa' for this isinstance. cls, modtype = m.__class__, types.ModuleType - return (not cls is modtype and ( + return (cls is not modtype and ( '__getattr__' in vars(m.__class__) or '__getattribute__' in vars(m.__class__))) diff --git a/celery/tests/compat_modules/test_sets.py b/celery/tests/compat_modules/test_sets.py index dc38d19a5..4176143d5 100644 --- a/celery/tests/compat_modules/test_sets.py +++ b/celery/tests/compat_modules/test_sets.py @@ -135,8 +135,8 @@ def test_is_JSON_serializable(self): s = self.MockTask.subtask( (2, ), {'cache': True}, {'routing_key': 'CPU-bound'}, ) - s.args = list(s.args) # tuples are not preserved - # but this doesn't matter. + # tuples are not preserved, but this doesn't matter. + s.args = list(s.args) self.assertEqual(s, self.subtask(json.loads(json.dumps(s)))) def test_repr(self): diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 6a3a56379..7abbf74e0 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -263,7 +263,7 @@ def _limit_task(self, request, bucket, tokens): self.on_task_request(request) def start(self): - blueprint, loop = self.blueprint, self.loop + blueprint = self.blueprint while blueprint.state != CLOSE: self.restart_count += 1 maybe_shutdown() diff --git a/celery/worker/loops.py b/celery/worker/loops.py index 5faa99e24..adfd99d04 100644 --- a/celery/worker/loops.py +++ b/celery/worker/loops.py @@ -26,11 +26,9 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos, """Non-blocking event loop consuming messages until connection is lost, or shutdown is requested.""" update_qos = qos.update - readers, writers = hub.readers, hub.writers hbtick = connection.heartbeat_check errors = connection.connection_errors heartbeat = connection.get_heartbeat_interval() # negotiated - hub_add, hub_remove = hub.add, hub.remove on_task_received = obj.create_task_handler() diff --git a/celery/worker/pidbox.py b/celery/worker/pidbox.py index 99c7a1a39..4a5ae1704 100644 --- a/celery/worker/pidbox.py +++ b/celery/worker/pidbox.py @@ -31,8 +31,9 @@ def __init__(self, c): self._forward_clock = self.c.app.clock.forward def on_message(self, body, message): - self._forward_clock() # just increase clock as clients usually don't - # have a valid clock to adjust with. + # just increase clock as clients usually don't + # have a valid clock to adjust with. + self._forward_clock() try: self.node.handle_message(body, message) except KeyError as exc: diff --git a/celery/worker/request.py b/celery/worker/request.py index c03b42d54..5058b79b6 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -388,9 +388,11 @@ def info(self, safe=False): 'worker_pid': self.worker_pid} def __str__(self): - return '{0.name}[{0.id}]{1}{2}'.format(self, - ' eta:[{0}]'.format(self.eta) if self.eta else '', - ' expires:[{0}]'.format(self.expires) if self.expires else '') + return '{0.name}[{0.id}]{1}{2}'.format( + self, + ' eta:[{0}]'.format(self.eta) if self.eta else '', + ' expires:[{0}]'.format(self.expires) if self.expires else '', + ) shortinfo = __str__ def __repr__(self): diff --git a/docs/_ext/applyxrefs.py b/docs/_ext/applyxrefs.py index deed5d90b..a9a9d8c2a 100644 --- a/docs/_ext/applyxrefs.py +++ b/docs/_ext/applyxrefs.py @@ -49,7 +49,6 @@ def has_target(fn): if not readok: return (True, None) - #print fn, len(lines) if len(lines) < 1: print("Not touching empty file %s." % fn) return (True, None) @@ -71,7 +70,6 @@ def main(argv=None): files.extend([(dirpath, f) for f in filenames]) files.sort() files = [os.path.join(p, fn) for p, fn in files if fn.endswith('.txt')] - #print files for fn in files: if fn in DONT_TOUCH: diff --git a/docs/conf.py b/docs/conf.py index 4a1c3d6a4..e46cc67fc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -67,11 +67,6 @@ def linkcode_resolve(domain, info): exclude_trees = ['.build'] -#unused_docs = [ -# 'xreftest.rst', -# 'tutorials/otherqueues', -#] - # If true, '()' will be appended to :func: etc. cross-reference text. add_function_parentheses = True @@ -116,7 +111,7 @@ def linkcode_resolve(domain, info): 'sourcelink.html', 'searchbox.html'], } -### Issuetracker +# ## Issuetracker github_project = 'celery/celery' @@ -142,13 +137,13 @@ def linkcode_resolve(domain, info): # A unique identification for the text. epub_uid = 'Celery Manual, Version {0}'.format(version) -# HTML files that should be inserted before the pages created by sphinx. -# The format is a list of tuples containing the path and title. -#epub_pre_files = [] +# ## HTML files that should be inserted before the pages created by sphinx. +# ## The format is a list of tuples containing the path and title. +# epub_pre_files = [] -# HTML files shat should be inserted after the pages created by sphinx. -# The format is a list of tuples containing the path and title. -#epub_post_files = [] +# ## HTML files shat should be inserted after the pages created by sphinx. +# ## The format is a list of tuples containing the path and title. +# epub_post_files = [] # A list of files that should not be packed into the epub file. epub_exclude_files = ['search.html'] diff --git a/examples/app/myapp.py b/examples/app/myapp.py index 51a624b2a..b72e9baab 100644 --- a/examples/app/myapp.py +++ b/examples/app/myapp.py @@ -27,11 +27,12 @@ app = Celery( 'myapp', broker='amqp://guest@localhost//', - # add result backend here if needed. - #backend='rpc' + # ## add result backend here if needed. + # backend='rpc' ) -@app.task() + +@app.task def add(x, y): return x + y diff --git a/examples/django/proj/__init__.py b/examples/django/proj/__init__.py index b64e43e83..ff99efb2c 100644 --- a/examples/django/proj/__init__.py +++ b/examples/django/proj/__init__.py @@ -3,3 +3,5 @@ # This will make sure the app is always imported when # Django starts so that shared_task will use this app. from .celery import app as celery_app + +__all__ = ['celery_app'] diff --git a/examples/eventlet/celeryconfig.py b/examples/eventlet/celeryconfig.py index a816c004f..2dc32edc2 100644 --- a/examples/eventlet/celeryconfig.py +++ b/examples/eventlet/celeryconfig.py @@ -2,7 +2,7 @@ import sys sys.path.insert(0, os.getcwd()) -## Start worker with -P eventlet +# ## Start worker with -P eventlet # Never use the CELERYD_POOL setting as that will patch # the worker too late. diff --git a/examples/gevent/celeryconfig.py b/examples/gevent/celeryconfig.py index 36d6a6c46..c7d94783f 100644 --- a/examples/gevent/celeryconfig.py +++ b/examples/gevent/celeryconfig.py @@ -2,7 +2,7 @@ import sys sys.path.insert(0, os.getcwd()) -### Note: Start worker with -P gevent, +# ## Note: Start worker with -P gevent, # do not use the CELERYD_POOL option. BROKER_URL = 'amqp://guest:guest@localhost:5672//' diff --git a/examples/next-steps/setup.py b/examples/next-steps/setup.py index 7eaccf9da..0132b3509 100644 --- a/examples/next-steps/setup.py +++ b/examples/next-steps/setup.py @@ -15,6 +15,6 @@ zip_safe=False, install_requires=[ 'celery>=3.0', - #'requests', + # 'requests', ], ) diff --git a/funtests/benchmarks/bench_worker.py b/funtests/benchmarks/bench_worker.py index 87f3615f0..53ba923bd 100644 --- a/funtests/benchmarks/bench_worker.py +++ b/funtests/benchmarks/bench_worker.py @@ -48,8 +48,9 @@ def tdiff(then): @app.task(cur=0, time_start=None, queue='bench.worker', bare=True) def it(_, n): - i = it.cur # use internal counter, as ordering can be skewed - # by previous runs, or the broker. + # use internal counter, as ordering can be skewed + # by previous runs, or the broker. + i = it.cur if i and not i % 5000: print('({0} so far: {1}s)'.format(i, tdiff(it.subt)), file=sys.stderr) it.subt = time.time() diff --git a/funtests/stress/stress/fbi.py b/funtests/stress/stress/fbi.py index f0b40fdcf..5f6625166 100644 --- a/funtests/stress/stress/fbi.py +++ b/funtests/stress/stress/fbi.py @@ -64,4 +64,3 @@ def diag(self, ids, file=sys.stderr): self.ffwd() for tid in ids: print(self.state_of(tid), file=file) - #print(self.query(ids), file=file) diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index 977cfec4d..c6c2a0f45 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -2,8 +2,6 @@ import os -from functools import partial - from celery.five import items from kombu import Exchange, Queue from kombu.utils import symbol_by_name @@ -121,11 +119,12 @@ class execv(default): @template() class sqs(default): - BROKER_URL='sqs://' + BROKER_URL = 'sqs://' BROKER_TRANSPORT_OPTIONS = { 'region': os.environ.get('AWS_REGION', 'us-east-1'), } + @template() class proto1(default): CELERY_TASK_PROTOCOL = 1 From 2cb227bd1e98c8b7b2e719dc33323367dbbc8f3e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 27 May 2014 16:38:53 +0100 Subject: [PATCH 0181/1103] Programs: celery inspect/control now supports --json argument --- celery/bin/celery.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/celery/bin/celery.py b/celery/bin/celery.py index 57c243040..191d28ade 100644 --- a/celery/bin/celery.py +++ b/celery/bin/celery.py @@ -269,7 +269,10 @@ class _RemoteControl(Command): Option('--timeout', '-t', type='float', help='Timeout in seconds (float) waiting for reply'), Option('--destination', '-d', - help='Comma separated list of destination node names.')) + help='Comma separated list of destination node names.'), + Option('--json', '-j', action='store_true', + help='Use json as output format.'), + ) def __init__(self, *args, **kwargs): self.show_body = kwargs.pop('show_body', True) @@ -335,6 +338,7 @@ def do_call_method(self, args, **kwargs): if self.app.connection().transport.driver_type == 'sql': raise self.Error('Broadcast not supported by SQL broker transport') + output_json = kwargs.get('json') destination = kwargs.get('destination') timeout = kwargs.get('timeout') or self.choices[method][0] if destination and isinstance(destination, string_t): @@ -342,12 +346,16 @@ def do_call_method(self, args, **kwargs): handler = getattr(self, method, self.call) + callback = None if output_json else self.say_remote_command_reply + replies = handler(method, *args[1:], timeout=timeout, destination=destination, - callback=self.say_remote_command_reply) + callback=callback) if not replies: raise self.Error('No nodes replied within time constraint.', status=EX_UNAVAILABLE) + if output_json: + self.out(json.dumps(replies)) return replies From e8be916196ec5c0d86dd0ab0e39f44f59dbaf949 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 28 May 2014 18:37:28 +0100 Subject: [PATCH 0182/1103] Removes debugging print statement --- celery/app/trace.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/celery/app/trace.py b/celery/app/trace.py index 3e04628a2..c26961cde 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -491,11 +491,9 @@ def _fast_trace_task(task, uuid, request, body, content_type, embed = None tasks, accept, hostname = _loc if content_type: - X = loads( + args, kwargs, embed = loads( body, content_type, content_encoding, accept=accept, ) - print(X) - args, kwargs, embed = X else: args, kwargs = body request.update({ From 6be5fc64ef6e245689c0961bce30b5d554aa96d6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 28 May 2014 18:52:30 +0100 Subject: [PATCH 0183/1103] Tests passing --- celery/tests/concurrency/test_eventlet.py | 22 ++-------------------- celery/tests/concurrency/test_gevent.py | 17 ++--------------- 2 files changed, 4 insertions(+), 35 deletions(-) diff --git a/celery/tests/concurrency/test_eventlet.py b/celery/tests/concurrency/test_eventlet.py index 162e4f2cf..d9447f46c 100644 --- a/celery/tests/concurrency/test_eventlet.py +++ b/celery/tests/concurrency/test_eventlet.py @@ -5,7 +5,6 @@ from celery.app.defaults import is_pypy from celery.concurrency.eventlet import ( apply_target, - Schedule, Timer, TaskPool, ) @@ -54,14 +53,14 @@ def test_aaa_is_patched(self): ) -class test_Schedule(EventletCase): +class test_Timer(EventletCase): def test_sched(self): with mock_module(*eventlet_modules): with patch_many('eventlet.greenthread.spawn_after', 'greenlet.GreenletExit') as (spawn_after, GreenletExit): - x = Schedule() + x = Timer() x.GreenletExit = KeyError entry = Mock() g = x._enter(1, 0, entry) @@ -99,20 +98,3 @@ def test_pool(self): def test_apply_target(self, base): apply_target(Mock(), getpid=Mock()) self.assertTrue(base.apply_target.called) - - -class test_Timer(EventletCase): - - def test_timer(self): - x = Timer() - x.ensure_started() - x.schedule = Mock() - x.start() - x.stop() - x.schedule.clear.assert_called_with() - - tref = Mock() - x.cancel(tref) - x.schedule.GreenletExit = KeyError - tref.cancel.side_effect = KeyError() - x.cancel(tref) diff --git a/celery/tests/concurrency/test_gevent.py b/celery/tests/concurrency/test_gevent.py index baa105ba4..c4a61db6d 100644 --- a/celery/tests/concurrency/test_gevent.py +++ b/celery/tests/concurrency/test_gevent.py @@ -1,7 +1,6 @@ from __future__ import absolute_import from celery.concurrency.gevent import ( - Schedule, Timer, TaskPool, apply_timeout, @@ -43,7 +42,7 @@ def test_is_patched(self): self.assertTrue(patch_all.called) -class test_Schedule(AppCase): +class test_Timer(AppCase): def test_sched(self): with mock_module(*gevent_modules): @@ -51,7 +50,7 @@ def test_sched(self): 'gevent.greenlet.GreenletExit') as (greenlet, GreenletExit): greenlet.Greenlet = object - x = Schedule() + x = Timer() greenlet.Greenlet = Mock() x._Greenlet.spawn_later = Mock() x._GreenletExit = KeyError @@ -100,18 +99,6 @@ def test_pool(self): self.assertEqual(x.num_processes, 3) -class test_Timer(AppCase): - - def test_timer(self): - with mock_module(*gevent_modules): - x = Timer() - x.ensure_started() - x.schedule = Mock() - x.start() - x.stop() - x.schedule.clear.assert_called_with() - - class test_apply_timeout(AppCase): def test_apply_timeout(self): From 335b9b6269cb3706cb4269d341a794bf62c3f4ec Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 May 2014 11:58:13 +0100 Subject: [PATCH 0184/1103] App after forkers were not running due to errors being silenced. Closes #2055 --- celery/app/base.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index c0174df93..696c3ea83 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -75,7 +75,7 @@ def _unpickle_appattr(reverse_name, args): return get_current_app()._rgetattr(reverse_name)(*args) -def _global_after_fork(): +def _global_after_fork(obj): # Previously every app would call: # `register_after_fork(app, app._after_fork)` # but this created a leak as `register_after_fork` stores concrete object @@ -84,12 +84,14 @@ def _global_after_fork(): # # See Issue #1949 from celery import _state - from multiprocessing.util import info - for app in _state.apps: + from multiprocessing import util as mputil + for app in _state._apps: try: - app._after_fork() + app._after_fork(obj) except Exception as exc: - info('after forker raised exception: %r' % (exc, ), exc_info=1) + if mputil._logger: + mputil._logger.info( + 'after forker raised exception: %r', exc, exc_info=1) def _ensure_after_fork(): From a3ea7d34ff817a0ea795248c8ae6b6fd009ccda1 Mon Sep 17 00:00:00 2001 From: Alexey Kotlyarov Date: Wed, 21 May 2014 08:29:16 +1000 Subject: [PATCH 0185/1103] Add myself as a contributor --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index ac27a0c74..a28946434 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -162,3 +162,4 @@ Matthew Duggan, 2014/04/10 Brian Bouterse, 2014/04/10 Dmitry Malinovsky, 2014/04/28 Luke Pomfrey, 2014/05/06 +Alexey Kotlyarov, 2014/05/16 From 807be90dec1e3b708e760663e9500c9dec286b6a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 May 2014 12:05:01 +0100 Subject: [PATCH 0186/1103] Yaml safe_load does not handle exceptions. Closes #2051 --- celery/backends/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 996171abf..b69e9e238 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -43,7 +43,7 @@ __all__ = ['BaseBackend', 'KeyValueStoreBackend', 'DisabledBackend'] -EXCEPTION_ABLE_CODECS = frozenset(['pickle', 'yaml']) +EXCEPTION_ABLE_CODECS = frozenset(['pickle']) PY3 = sys.version_info >= (3, 0) logger = get_logger(__name__) From 8d4354a0c9cf7d9daf7b36b156b6ec3daa2925da Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 May 2014 12:39:19 +0100 Subject: [PATCH 0187/1103] Default serializer is now json --- celery/app/defaults.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index b09cc0256..1fa5cb337 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -139,7 +139,7 @@ def __repr__(self): 'RESULT_ENGINE_OPTIONS': Option(type='dict'), 'RESULT_EXCHANGE': Option('celeryresults'), 'RESULT_EXCHANGE_TYPE': Option('direct'), - 'RESULT_SERIALIZER': Option('pickle'), + 'RESULT_SERIALIZER': Option('json'), 'RESULT_PERSISTENT': Option(None, type='bool'), 'ROUTES': Option(type='any'), 'SEND_EVENTS': Option(False, type='bool'), @@ -154,7 +154,7 @@ def __repr__(self): 'interval_max': 1, 'interval_step': 0.2}, type='dict'), 'TASK_RESULT_EXPIRES': Option(timedelta(days=1), type='float'), - 'TASK_SERIALIZER': Option('pickle'), + 'TASK_SERIALIZER': Option('json'), 'TIMEZONE': Option(type='string'), 'TRACK_STARTED': Option(False, type='bool'), 'REDIRECT_STDOUTS': Option(True, type='bool'), From 1c6065383ef366ea9209d058513ad23453d94c2c Mon Sep 17 00:00:00 2001 From: Mher Movsisyan Date: Thu, 29 May 2014 16:35:36 +0400 Subject: [PATCH 0188/1103] Adds grow/shrink methods to eventlet TaskPool --- celery/concurrency/eventlet.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/celery/concurrency/eventlet.py b/celery/concurrency/eventlet.py index 2371e4829..c501985fa 100644 --- a/celery/concurrency/eventlet.py +++ b/celery/concurrency/eventlet.py @@ -131,6 +131,16 @@ def on_apply(self, target, args=None, kwargs=None, callback=None, callback, accept_callback, self.getpid) + def grow(self, n=1): + limit = self.limit + n + self._pool.resize(limit) + self.limit = limit + + def shrink(self, n=1): + limit = self.limit - n + self._pool.resize(limit) + self.limit = limit + def _get_info(self): return { 'max-concurrency': self.limit, From 4b89e6ec1027bef29c7ecf36fcfe0797bfefb763 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 May 2014 13:44:15 +0100 Subject: [PATCH 0189/1103] Tests passing --- celery/tests/backends/test_base.py | 3 +++ celery/tests/backends/test_cache.py | 1 + celery/tests/backends/test_database.py | 1 + celery/tests/case.py | 1 + celery/tests/tasks/test_result.py | 2 ++ celery/tests/tasks/test_tasks.py | 2 +- celery/tests/worker/test_request.py | 2 ++ 7 files changed, 11 insertions(+), 1 deletion(-) diff --git a/celery/tests/backends/test_base.py b/celery/tests/backends/test_base.py index 58e3e8d52..6f626b76b 100644 --- a/celery/tests/backends/test_base.py +++ b/celery/tests/backends/test_base.py @@ -98,12 +98,14 @@ def setup(self): self.b = BaseBackend(self.app) def test_unpickleable(self): + self.b.serializer = 'pickle' x = self.b.prepare_exception(Unpickleable(1, 2, 'foo')) self.assertIsInstance(x, KeyError) y = self.b.exception_to_python(x) self.assertIsInstance(y, KeyError) def test_impossible(self): + self.b.serializer = 'pickle' x = self.b.prepare_exception(Impossible()) self.assertIsInstance(x, UnpickleableExceptionWrapper) self.assertTrue(str(x)) @@ -115,6 +117,7 @@ def test_impossible(self): self.assertEqual(y.__class__.__module__, 'foo.module') def test_regular(self): + self.b.serializer = 'pickle' x = self.b.prepare_exception(KeyError('baz')) self.assertIsInstance(x, KeyError) y = self.b.exception_to_python(x) diff --git a/celery/tests/backends/test_cache.py b/celery/tests/backends/test_cache.py index 051760a8d..f741b852e 100644 --- a/celery/tests/backends/test_cache.py +++ b/celery/tests/backends/test_cache.py @@ -31,6 +31,7 @@ def __init__(self, data): class test_CacheBackend(AppCase): def setup(self): + self.app.conf.CELERY_RESULT_SERIALIZER = 'pickle' self.tb = CacheBackend(backend='memory://', app=self.app) self.tid = uuid() diff --git a/celery/tests/backends/test_database.py b/celery/tests/backends/test_database.py index 6b5bf9420..70d8339bf 100644 --- a/celery/tests/backends/test_database.py +++ b/celery/tests/backends/test_database.py @@ -40,6 +40,7 @@ def setup(self): if DatabaseBackend is None: raise SkipTest('sqlalchemy not installed') self.uri = 'sqlite:///test.db' + self.app.conf.CELERY_RESULT_SERIALIZER = 'pickle' def test_retry_helper(self): from celery.backends.database import DatabaseError diff --git a/celery/tests/case.py b/celery/tests/case.py index 09108670d..8878c351c 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -93,6 +93,7 @@ 'CELERY_QUEUES': ( Queue('testcelery', routing_key='testcelery'), ), + 'CELERY_ACCEPT_CONTENT': ('json', 'pickle'), 'CELERY_ENABLE_UTC': True, 'CELERY_TIMEZONE': 'UTC', 'CELERYD_LOG_COLOR': False, diff --git a/celery/tests/tasks/test_result.py b/celery/tests/tasks/test_result.py index 61bf09dfe..559ce78fd 100644 --- a/celery/tests/tasks/test_result.py +++ b/celery/tests/tasks/test_result.py @@ -44,6 +44,7 @@ def make_mock_group(app, size=10): class test_AsyncResult(AppCase): def setup(self): + self.app.conf.CELERY_RESULT_SERIALIZER = 'pickle' self.task1 = mock_task('task1', states.SUCCESS, 'the') self.task2 = mock_task('task2', states.SUCCESS, 'quick') self.task3 = mock_task('task3', states.FAILURE, KeyError('brown')) @@ -614,6 +615,7 @@ def test_result(self): class test_failed_AsyncResult(test_GroupResult): def setup(self): + self.app.conf.CELERY_RESULT_SERIALIZER = 'pickle' self.size = 11 subtasks = make_mock_group(self.app, 10) failed = mock_task('ts11', states.FAILURE, KeyError('Baz')) diff --git a/celery/tests/tasks/test_tasks.py b/celery/tests/tasks/test_tasks.py index 6f11ba1be..35d71a132 100644 --- a/celery/tests/tasks/test_tasks.py +++ b/celery/tests/tasks/test_tasks.py @@ -235,7 +235,7 @@ def test_AsyncResult(self): def assertNextTaskDataEqual(self, consumer, presult, task_name, test_eta=False, test_expires=False, **kwargs): - next_task = consumer.queues[0].get(accept=['pickle']) + next_task = consumer.queues[0].get(accept=['pickle', 'json']) task_data = next_task.decode() self.assertEqual(task_data['id'], presult.id) self.assertEqual(task_data['task'], task_name) diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index 7a202fa72..a0ca1f156 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -210,6 +210,7 @@ def send(self, event, **fields): class test_Request(AppCase): def setup(self): + self.app.conf.CELERY_RESULT_SERIALIZER = 'pickle' @self.app.task(shared=False) def add(x, y, **kw_): @@ -807,6 +808,7 @@ def test_execute_fail(self): kwargs={}, ) self.assertIsInstance(job.execute(), ExceptionInfo) + assert self.mytask_raising.backend.serializer == 'pickle' meta = self.mytask_raising.backend.get_task_meta(tid) self.assertEqual(meta['status'], states.FAILURE) self.assertIsInstance(meta['result'], KeyError) From 0673da5c09ac22bdd49ba811c470b73a036ee776 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 May 2014 15:29:57 +0100 Subject: [PATCH 0190/1103] Canvas: Unroll groups within groups to a single group. Closes #1509 --- celery/canvas.py | 79 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index e664d5394..eb0eb25ea 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -341,6 +341,7 @@ def _apply_async(self): @Signature.register_type class chain(Signature): + tasks = _getitem_property('kwargs.tasks') def __init__(self, *tasks, **options): tasks = (regen(tasks[0]) if len(tasks) == 1 and is_list(tasks[0]) @@ -348,7 +349,6 @@ def __init__(self, *tasks, **options): Signature.__init__( self, 'celery.chain', (), {'tasks': tasks}, **options ) - self.tasks = tasks self.subtask_type = 'chain' def __call__(self, *args, **kwargs): @@ -557,6 +557,7 @@ def _maybe_group(tasks): @Signature.register_type class group(Signature): + tasks = _getitem_property('kwargs.tasks') def __init__(self, *tasks, **options): if len(tasks) == 1: @@ -564,7 +565,7 @@ def __init__(self, *tasks, **options): Signature.__init__( self, 'celery.group', (), {'tasks': tasks}, **options ) - self.tasks, self.subtask_type = tasks, 'group' + self.subtask_type = 'group' @classmethod def from_dict(self, d, app=None): @@ -586,9 +587,17 @@ def _prepared(self, tasks, partial_args, group_id, root_id, dict=dict, else: # serialized sigs must be converted to Signature. task = from_dict(task) - if partial_args and not task.immutable: - task.args = tuple(partial_args) + tuple(task.args) - yield task, task.freeze(group_id=group_id, root_id=root_id) + if isinstance(task, group): + # needs yield_from :( + unroll = task._prepared( + task.tasks, partial_args, group_id, root_id, + ) + for taskN, resN in unroll: + yield taskN, resN + else: + if partial_args and not task.immutable: + task.args = tuple(partial_args) + tuple(task.args) + yield task, task.freeze(group_id=group_id, root_id=root_id) def _apply_tasks(self, tasks, producer=None, app=None, **options): app = app or self.app @@ -650,6 +659,17 @@ def link_error(self, sig): def __call__(self, *partial_args, **options): return self.apply_async(partial_args, **options) + def _freeze_unroll(self, new_tasks, group_id, chord, root_id): + stack = deque(self.tasks) + while stack: + task = maybe_signature(stack.popleft(), app=self._app).clone() + if isinstance(task, group): + stack.extendleft(task.tasks) + else: + new_tasks.append(task) + yield task.freeze(group_id=group_id, + chord=chord, root_id=root_id) + def freeze(self, _id=None, group_id=None, chord=None, root_id=None): opts = self.options try: @@ -659,16 +679,18 @@ def freeze(self, _id=None, group_id=None, chord=None, root_id=None): if group_id: opts['group_id'] = group_id if chord: - opts['chord'] = group_id + opts['chord'] = chord root_id = opts.setdefault('root_id', root_id) - new_tasks, results = [], [] - for task in self.tasks: - task = maybe_signature(task, app=self._app).clone() - results.append(task.freeze( - group_id=group_id, chord=chord, root_id=root_id, - )) - new_tasks.append(task) - self.tasks = self.kwargs['tasks'] = new_tasks + new_tasks = [] + # Need to unroll subgroups early so that chord gets the + # right result instance for chord_unlock etc. + results = list(self._freeze_unroll( + new_tasks, group_id, chord, root_id, + )) + if isinstance(self.tasks, MutableSequence): + self.tasks[:] = new_tasks + else: + self.tasks = new_tasks return self.app.GroupResult(gid, results) _freeze = freeze @@ -689,7 +711,7 @@ def app(self): app = self._app if app is None: try: - app = self.tasks[0]._app + app = self.tasks[0].app except (KeyError, IndexError): pass return app if app is not None else current_app @@ -723,11 +745,14 @@ def _unpack_args(header=None, body=None, **kwargs): @cached_property def app(self): + return self._get_app(self.body) + + def _get_app(self, body=None): app = self._app if app is None: app = self.tasks[0]._app - if app is None: - app = self.body._app + if app is None and body is not None: + app = body._app return app if app is not None else current_app def apply_async(self, args=(), kwargs={}, task_id=None, @@ -736,7 +761,7 @@ def apply_async(self, args=(), kwargs={}, task_id=None, body = kwargs.get('body') or self.kwargs['body'] kwargs = dict(self.kwargs, **kwargs) body = body.clone(**options) - app = self.app + app = self._get_app(body) tasks = (self.tasks.clone() if isinstance(self.tasks, group) else group(self.tasks)) if app.conf.CELERY_ALWAYS_EAGER: @@ -752,15 +777,29 @@ def apply(self, args=(), kwargs={}, propagate=True, body=None, **options): args=(tasks.apply().get(propagate=propagate), ), ) + def _traverse_tasks(self, tasks, value=None): + stack = deque(tasks) + while stack: + task = stack.popleft() + if isinstance(task, group): + stack.extend(task.tasks) + else: + yield task if value is None else value + + def __length_hint__(self): + return sum(self._traverse_tasks(self.tasks, 1)) + def run(self, header, body, partial_args, app=None, interval=None, countdown=1, max_retries=None, propagate=None, eager=False, task_id=None, **options): - app = app or self.app + app = app or self._get_app(body) propagate = (app.conf.CELERY_CHORD_PROPAGATES if propagate is None else propagate) group_id = uuid() root_id = body.options.get('root_id') - body.setdefault('chord_size', len(header.tasks)) + if 'chord_size' not in body: + body['chord_size'] = self.__length_hint__() + results = header.freeze( group_id=group_id, chord=body, root_id=root_id).results bodyres = body.freeze(task_id, root_id=root_id) From b9ee0f7ba949c8cbf568cf2722e82eb65a83e0ae Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 May 2014 16:02:06 +0100 Subject: [PATCH 0191/1103] Cosmetics for #1575 --- celery/backends/riak.py | 45 ++++------ celery/tests/backends/test_riak.py | 139 +++++++++++++---------------- celery/tests/case.py | 9 +- 3 files changed, 84 insertions(+), 109 deletions(-) diff --git a/celery/backends/riak.py b/celery/backends/riak.py index 725e396a5..944ef5b8a 100644 --- a/celery/backends/riak.py +++ b/celery/backends/riak.py @@ -1,44 +1,37 @@ # -*- coding: utf-8 -*- """ celery.backends.riak - ~~~~~~~~~~~~~~~~~~~~~~~ + ~~~~~~~~~~~~~~~~~~~~ Riak result store backend. """ -from __future__ import absolute_import, print_function - -from datetime import datetime +from __future__ import absolute_import try: import riak - from riak import RiakClient, RiakNode + from riak import RiakClient from riak.resolver import last_written_resolver except ImportError: # pragma: no cover - riak = None # noqa + riak = RiakClient = last_written_resolver = None # noqa from kombu.utils.url import _parse_url -from celery import states from celery.exceptions import ImproperlyConfigured from celery.utils.timeutils import maybe_timedelta from .base import KeyValueStoreBackend +E_BUCKET_NAME = """\ +Riak bucket names must be composed of ASCII characters only, not: {0!r}\ +""" -class NonAsciiBucket(Exception): - """ Bucket must ne ascii charchters only. """ - - -class Validators(object): - - @classmethod - def validate_riak_bucket_name(cls, bucket_name): - try: - bucket_name.decode('ascii') - except UnicodeDecodeError as ude: - return False - return True +def is_ascii(s): + try: + s.decode('ascii') + except UnicodeDecodeError: + return False + return True class RiakBackend(KeyValueStoreBackend): @@ -47,7 +40,7 @@ class RiakBackend(KeyValueStoreBackend): protocol = 'pbc' #: default Riak bucket name (`default`) - bucket_name = "celery" + bucket_name = 'celery' #: default Riak server hostname (`localhost`) host = 'localhost' @@ -94,13 +87,13 @@ def __init__(self, host=None, port=None, bucket_name=None, protocol=None, self.protocol = protocol or config.get('protocol', self.protocol) # riak bucket must be ascii letters or numbers only - if not Validators.validate_riak_bucket_name(self.bucket_name): - raise NonAsciiBucket("Riak bucket names must be ASCII characters") + if not is_ascii(self.bucket_name): + raise ValueError(E_BUCKET_NAME.format(self.bucket_name)) self._client = None def _get_client(self): - """Get client connection""" + """Get client connection.""" if self._client is None or not self._client.is_alive(): self._client = RiakClient(protocol=self.protocol, host=self.host, @@ -109,7 +102,7 @@ def _get_client(self): return self._client def _get_bucket(self): - """Connect to our bucket""" + """Connect to our bucket.""" if ( self._client is None or not self._client.is_alive() or not self._bucket @@ -129,8 +122,6 @@ def get(self, key): return self.bucket.get(key).data def set(self, key, value): - # RiakBucket.new(key=None, data=None, content_type='application/json', - # encoded_data=None) _key = self.bucket.new(key, data=value) _key.store() diff --git a/celery/tests/backends/test_riak.py b/celery/tests/backends/test_riak.py index fd2a3728f..07d8bf802 100644 --- a/celery/tests/backends/test_riak.py +++ b/celery/tests/backends/test_riak.py @@ -5,11 +5,10 @@ from mock import MagicMock, Mock, patch, sentinel from nose import SkipTest -from celery import Celery from celery.backends import riak as module -from celery.backends.riak import RiakBackend, riak, NonAsciiBucket +from celery.backends.riak import RiakBackend, riak from celery.exceptions import ImproperlyConfigured -from celery.tests.utils import AppCase +from celery.tests.case import AppCase RIAK_BUCKET = 'riak_bucket' @@ -17,12 +16,14 @@ class test_RiakBackend(AppCase): - def setUp(self): + def setup(self): if riak is None: raise SkipTest('riak is not installed.') - from celery.app import current_app - self.app = self._current_app = current_app() - self.backend = RiakBackend(app=self.app) + self.app.conf.CELERY_RESULT_BACKEND = 'riak://' + + @property + def backend(self): + return self.app.backend def test_init_no_riak(self): """ @@ -36,26 +37,20 @@ def test_init_no_riak(self): module.riak = prev def test_init_no_settings(self): - """ - test init no settings - """ - celery = Celery(set_as_current=False) - celery.conf.CELERY_RIAK_BACKEND_SETTINGS = [] + """Test init no settings.""" + self.app.conf.CELERY_RIAK_BACKEND_SETTINGS = [] with self.assertRaises(ImproperlyConfigured): - RiakBackend(app=celery) + RiakBackend(app=self.app) def test_init_settings_is_None(self): """ Test init settings is None """ - celery = Celery(set_as_current=False) - celery.conf.CELERY_RIAK_BACKEND_SETTINGS = None - RiakBackend(app=celery) + self.app.conf.CELERY_RIAK_BACKEND_SETTINGS = None + self.assertTrue(self.app.backend) def test_get_client_client_exists(self): - """ - Test get existing client - """ + """Test get existing client.""" with patch('riak.client.RiakClient') as mock_connection: self.backend._client = sentinel._client @@ -66,62 +61,55 @@ def test_get_client_client_exists(self): self.assertFalse(mock_connection.called) def test_get(self): - """ - Test get + """Test get + RiakBackend.get should return and take two params db conn to riak is mocked TODO Should test on key not exists """ - celery = Celery(set_as_current=False) - - celery.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = {} - - backend = RiakBackend(app=celery) - backend._client = Mock() - backend._bucket = Mock() - mocked_get = backend._bucket.get = Mock() + self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = {} + self.backend._client = Mock(name='_client') + self.backend._bucket = Mock(name='_bucket') + mocked_get = self.backend._bucket.get = Mock(name='bucket.get') mocked_get.return_value.data = sentinel.retval # should return None - self.assertEqual(backend.get('1f3fab'), sentinel.retval) - backend._bucket.get.assert_called_once_with('1f3fab') + self.assertEqual(self.backend.get('1f3fab'), sentinel.retval) + self.backend._bucket.get.assert_called_once_with('1f3fab') def test_set(self): - """ - Test set + """Test set + RiakBackend.set should return None and take two params - db conn to couchbase is mocked + db conn to couchbase is mocked. + """ - celery = Celery(set_as_current=False) - celery.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = None - backend = RiakBackend(app=celery) - backend._client = MagicMock() - backend._bucket = MagicMock() - backend._bucket.set = MagicMock() + self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = None + self.backend._client = MagicMock() + self.backend._bucket = MagicMock() + self.backend._bucket.set = MagicMock() # should return None - self.assertIsNone(backend.set(sentinel.key, sentinel.value)) + self.assertIsNone(self.backend.set(sentinel.key, sentinel.value)) def test_delete(self): - """ - Test get + """Test get + RiakBackend.get should return and take two params db conn to couchbase is mocked TODO Should test on key not exists - """ - celery = Celery(set_as_current=False) - celery.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = {} + """ + self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = {} - backend = RiakBackend(app=celery) - backend._client = Mock() - backend._bucket = Mock() - mocked_delete = backend._client.delete = Mock() + self.backend._client = Mock(name='_client') + self.backend._bucket = Mock(name='_bucket') + mocked_delete = self.backend._client.delete = Mock('client.delete') mocked_delete.return_value = None # should return None - self.assertIsNone(backend.delete('1f3fab')) - backend._bucket.delete.assert_called_once_with('1f3fab') + self.assertIsNone(self.backend.delete('1f3fab')) + self.backend._bucket.delete.assert_called_once_with('1f3fab') def test_config_params(self): """ @@ -129,21 +117,22 @@ def test_config_params(self): celery.conf.CELERY_RIAK_BACKEND_SETTINGS is properly set """ - celery = Celery(set_as_current=False) - celery.conf.CELERY_RIAK_BACKEND_SETTINGS = {'bucket': 'mycoolbucket', - 'host': 'there.host.com', - 'port': '1234'} - backend = RiakBackend(app=celery) - self.assertEqual(backend.bucket_name, "mycoolbucket") - self.assertEqual(backend.host, 'there.host.com') - self.assertEqual(backend.port, 1234) + self.app.conf.CELERY_RIAK_BACKEND_SETTINGS = { + 'bucket': 'mycoolbucket', + 'host': 'there.host.com', + 'port': '1234', + } + self.assertEqual(self.backend.bucket_name, 'mycoolbucket') + self.assertEqual(self.backend.host, 'there.host.com') + self.assertEqual(self.backend.port, 1234) def test_backend_by_url(self, url='riak://myhost/mycoolbucket'): """ test get backend by url """ + from celery import backends from celery.backends.riak import RiakBackend - backend, url_ = backends.get_backend_by_url(url) + backend, url_ = backends.get_backend_by_url(url, self.app.loader) self.assertIs(backend, RiakBackend) self.assertEqual(url_, url) @@ -151,24 +140,20 @@ def test_backend_params_by_url(self): """ test get backend params by url """ - celery = Celery(set_as_current=False, - backend='riak://myhost:123/mycoolbucket') - backend = celery.backend - self.assertEqual(backend.bucket_name, "mycoolbucket") - self.assertEqual(backend.host, "myhost") - self.assertEqual(backend.port, 123) + self.app.conf.CELERY_RESULT_BACKEND = 'riak://myhost:123/mycoolbucket' + self.assertEqual(self.backend.bucket_name, 'mycoolbucket') + self.assertEqual(self.backend.host, 'myhost') + self.assertEqual(self.backend.port, 123) def test_non_ASCII_bucket_raises(self): - """ - test celery.conf.CELERY_RIAK_BACKEND_SETTINGS + """test celery.conf.CELERY_RIAK_BACKEND_SETTINGS and celery.conf.CELERY_RIAK_BACKEND_SETTINGS is properly set """ - with self.assertRaises(NonAsciiBucket): - celery = Celery(set_as_current=False) - celery.conf.CELERY_RIAK_BACKEND_SETTINGS = { - 'bucket': 'héhé', - 'host': 'there.host.com', - 'port': '1234', - } - RiakBackend(app=celery) + self.app.conf.CELERY_RIAK_BACKEND_SETTINGS = { + 'bucket': 'héhé', + 'host': 'there.host.com', + 'port': '1234', + } + with self.assertRaises(ValueError): + RiakBackend(app=self.app) diff --git a/celery/tests/case.py b/celery/tests/case.py index 8878c351c..50bb87a36 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -84,6 +84,9 @@ """ CELERY_TEST_CONFIG = { + 'BROKER_URL': 'memory://', + 'CELERY_RESULT_BACKEND': 'cache+memory://', + #: Don't want log output when running suite. 'CELERYD_HIJACK_ROOT_LOGGER': False, 'CELERY_SEND_TASK_ERROR_EMAILS': False, @@ -124,12 +127,8 @@ def __init__(self, *args, **kwargs): self.already_setup = True -def UnitApp(name=None, broker=None, backend=None, - set_as_current=False, log=UnitLogging, **kwargs): - +def UnitApp(name=None, set_as_current=False, log=UnitLogging, **kwargs): app = Celery(name or 'celery.tests', - broker=broker or 'memory://', - backend=backend or 'cache+memory://', set_as_current=set_as_current, log=log, **kwargs) From 2d7904a836f56e2bb03661ea6e986ffd041e6bda Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 May 2014 17:30:52 +0100 Subject: [PATCH 0192/1103] Task: Store state for exceptions occurring outside of task body. Closes #1582 --- celery/app/trace.py | 34 ++++++++++++++--------------- celery/tests/app/test_app.py | 2 +- celery/tests/tasks/test_trace.py | 4 ++-- celery/tests/worker/test_request.py | 16 ++++++++++---- 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/celery/app/trace.py b/celery/app/trace.py index c26961cde..b34ca7e2b 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -140,7 +140,7 @@ def __init__(self, state, retval=None): self.state = state self.retval = retval - def handle_error_state(self, task, eager=False): + def handle_error_state(self, task, req, eager=False): store_errors = not eager if task.ignore_result: store_errors = task.store_errors_even_if_ignored @@ -148,19 +148,18 @@ def handle_error_state(self, task, eager=False): return { RETRY: self.handle_retry, FAILURE: self.handle_failure, - }[self.state](task, store_errors=store_errors) + }[self.state](task, req, store_errors=store_errors) - def handle_reject(self, task, **kwargs): - self._log_error(task, ExceptionInfo()) + def handle_reject(self, task, req, **kwargs): + self._log_error(task, req, ExceptionInfo()) - def handle_ignore(self, task, **kwargs): - self._log_error(task, ExceptionInfo()) + def handle_ignore(self, task, req, **kwargs): + self._log_error(task, req, ExceptionInfo()) - def handle_retry(self, task, store_errors=True): + def handle_retry(self, task, req, store_errors=True): """Handle retry exception.""" # the exception raised is the Retry semi-predicate, # and it's exc' attribute is the original exception raised (if any). - req = task.request type_, _, tb = sys.exc_info() try: reason = self.retval @@ -180,9 +179,8 @@ def handle_retry(self, task, store_errors=True): finally: del(tb) - def handle_failure(self, task, store_errors=True): + def handle_failure(self, task, req, store_errors=True): """Handle exception.""" - req = task.request type_, _, tb = sys.exc_info() try: exc = self.retval @@ -199,13 +197,12 @@ def handle_failure(self, task, store_errors=True): kwargs=req.kwargs, traceback=tb, einfo=einfo) - self._log_error(task, einfo) + self._log_error(task, req, einfo) return einfo finally: del(tb) - def _log_error(self, task, einfo): - req = task.request + def _log_error(self, task, req, einfo): eobj = einfo.exception = get_pickled_exception(einfo.exception) exception, traceback, exc_info, sargs, skwargs = ( safe_repr(eobj), @@ -308,7 +305,7 @@ def on_error(request, exc, uuid, state=FAILURE, call_errbacks=True): if propagate: raise I = Info(state, exc) - R = I.handle_error_state(task, eager=eager) + R = I.handle_error_state(task, request, eager=eager) if call_errbacks: group( [signature(errback, app=app) @@ -329,6 +326,7 @@ def trace_task(uuid, args, kwargs, request=None): # we want the main variables (I, and R) to stand out visually from the # the rest of the variables, so breaking PEP8 is worth it ;) R = I = T = Rstr = retval = state = None + task_request = None time_start = monotonic() try: try: @@ -359,11 +357,11 @@ def trace_task(uuid, args, kwargs, request=None): except Reject as exc: I, R = Info(REJECTED, exc), ExceptionInfo(internal=True) state, retval = I.state, I.retval - I.handle_reject(task) + I.handle_reject(task, task_request) except Ignore as exc: I, R = Info(IGNORED, exc), ExceptionInfo(internal=True) state, retval = I.state, I.retval - I.handle_ignore(task) + I.handle_ignore(task, task_request) except Retry as exc: I, R, state, retval = on_error( task_request, exc, uuid, RETRY, call_errbacks=False, @@ -448,6 +446,8 @@ def trace_task(uuid, args, kwargs, request=None): if eager: raise R = report_internal_error(task, exc) + if task_request is not None: + I, _, _, _ = on_error(task_request, exc, uuid) return trace_ok_t(R, I, T, Rstr) return trace_task @@ -459,7 +459,7 @@ def trace_task(task, uuid, args, kwargs, request={}, **opts): task.__trace__ = build_tracer(task.name, task, **opts) return task.__trace__(uuid, args, kwargs, request) except Exception as exc: - return report_internal_error(task, exc) + return trace_ok_t(report_internal_error(task, exc), None, 0.0, None) def _trace_task_ret(name, uuid, request, body, content_type, diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index 69187d0a0..1fcdf3ffc 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -432,7 +432,7 @@ def test_config_from_cmdline(self): {'foo': 'bar'}) def test_compat_setting_CELERY_BACKEND(self): - + self.app.conf.defaults[0]['CELERY_RESULT_BACKEND'] = None self.app.config_from_object(Object(CELERY_BACKEND='set_by_us')) self.assertEqual(self.app.conf.CELERY_RESULT_BACKEND, 'set_by_us') diff --git a/celery/tests/tasks/test_trace.py b/celery/tests/tasks/test_trace.py index 3d55d9041..3149206fe 100644 --- a/celery/tests/tasks/test_trace.py +++ b/celery/tests/tasks/test_trace.py @@ -172,9 +172,9 @@ class TI(TraceInfo): def test_handle_error_state(self): x = self.TI(states.FAILURE) x.handle_failure = Mock() - x.handle_error_state(self.add_cast) + x.handle_error_state(self.add_cast, self.add_cast.request) x.handle_failure.assert_called_with( - self.add_cast, + self.add_cast, self.add_cast.request, store_errors=self.add_cast.store_errors_even_if_ignored, ) diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index a0ca1f156..392c6d509 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -698,11 +698,15 @@ def test_worker_task_trace_handle_retry(self): raise Retry(str(exc), exc=exc) except Retry as exc: w = TraceInfo(states.RETRY, exc) - w.handle_retry(self.mytask, store_errors=False) + w.handle_retry( + self.mytask, self.mytask.request, store_errors=False, + ) self.assertEqual( self.mytask.backend.get_status(tid), states.PENDING, ) - w.handle_retry(self.mytask, store_errors=True) + w.handle_retry( + self.mytask, self.mytask.request, store_errors=True, + ) self.assertEqual( self.mytask.backend.get_status(tid), states.RETRY, ) @@ -718,11 +722,15 @@ def test_worker_task_trace_handle_failure(self): raise ValueError('foo') except Exception as exc: w = TraceInfo(states.FAILURE, exc) - w.handle_failure(self.mytask, store_errors=False) + w.handle_failure( + self.mytask, self.mytask.request, store_errors=False, + ) self.assertEqual( self.mytask.backend.get_status(tid), states.PENDING, ) - w.handle_failure(self.mytask, store_errors=True) + w.handle_failure( + self.mytask, self.mytask.request, store_errors=True, + ) self.assertEqual( self.mytask.backend.get_status(tid), states.FAILURE, ) From 3a148b3f2b95fba09b81d47db4fb9455a2c60b24 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 May 2014 17:50:30 +0100 Subject: [PATCH 0193/1103] Makes sure chain.link_* works like .apply_async(link*=) --- celery/canvas.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index eb0eb25ea..0384a4d8b 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -355,12 +355,18 @@ def __call__(self, *args, **kwargs): if self.tasks: return self.apply_async(args, kwargs) - def apply_async(self, args=(), kwargs={}, group_id=None, chord=None, - task_id=None, link=None, link_error=None, - publisher=None, root_id=None, **options): + def apply_async(self, args=(), kwargs={}, **options): + # python is best at unpacking kwargs, so .run is here to do that. app = self.app if app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) + return self.run(args, kwargs, app=app, **( + dict(self.options, **options) if options else self.options)) + + def run(self, args=(), kwargs={}, group_id=None, chord=None, + task_id=None, link=None, link_error=None, + publisher=None, producer=None, root_id=None, app=None, **options): + app = app or self.app tasks, results = self.prepare_steps( args, self.tasks, root_id, link_error, ) @@ -443,8 +449,7 @@ def apply(self, args=(), kwargs={}, **options): last, fargs = None, args for task in self.tasks: res = task.clone(fargs).apply( - last and (last.get(), ), **options - ) + last and (last.get(), ), **dict(self.options, **options)) res.parent, last, fargs = last, res, None return last From c016714be89746068017d1cfdc991f48836c1c12 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 May 2014 20:10:58 +0100 Subject: [PATCH 0194/1103] Removes the metaclass for the new task base class (tasks are no longer automatically registered, the @task decorator binds and registers the task) --- celery/app/base.py | 19 ++++-- celery/app/builtins.py | 93 ++++++++++++--------------- celery/app/task.py | 99 +---------------------------- celery/task/base.py | 102 +++++++++++++++++++++++++++++- celery/tests/app/test_builtins.py | 6 +- celery/tests/tasks/test_tasks.py | 5 -- celery/utils/__init__.py | 1 + 7 files changed, 158 insertions(+), 167 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 696c3ea83..0e320ec7f 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -33,6 +33,7 @@ from celery.five import items, values from celery.loaders import get_loader_cls from celery.local import PromiseProxy, maybe_evaluate +from celery.utils import gen_task_name from celery.utils.dispatch import Signal from celery.utils.functional import first, maybe_list from celery.utils.imports import instantiate, symbol_by_name @@ -224,16 +225,16 @@ def worker_main(self, argv=None): def task(self, *args, **opts): """Creates new task class from any callable.""" - if _EXECV and not opts.get('_force_evaluate'): + if _EXECV and opts.get('lazy', True): # When using execv the task in the original module will point to a # different app, so doing things like 'add.request' will point to - # a differnt task instance. This makes sure it will always use + # a different task instance. This makes sure it will always use # the task instance from the current app. # Really need a better solution for this :( from . import shared_task - return shared_task(*args, _force_evaluate=True, **opts) + return shared_task(*args, lazy=False, **opts) - def inner_create_task_cls(shared=True, filter=None, **opts): + def inner_create_task_cls(shared=True, filter=None, lazy=True, **opts): _filt = filter # stupid 2to3 def _create_task_cls(fun): @@ -241,7 +242,7 @@ def _create_task_cls(fun): cons = lambda app: app._task_from_fun(fun, **opts) cons.__name__ = fun.__name__ connect_on_app_finalize(cons) - if self.finalized or opts.get('_force_evaluate'): + if not lazy or self.finalized: ret = self._task_from_fun(fun, **opts) else: # return a proxy object that evaluates on first use @@ -264,19 +265,25 @@ def _create_task_cls(fun): sum([len(args), len(opts)]))) return inner_create_task_cls(**opts) - def _task_from_fun(self, fun, **options): + def _task_from_fun(self, fun, name=None, **options): if not self.finalized and not self.autofinalize: raise RuntimeError('Contract breach: app not finalized') base = options.pop('base', None) or self.Task bind = options.pop('bind', False) + name = name or gen_task_name(self, fun.__name__, fun.__module__) + T = type(fun.__name__, (base, ), dict({ 'app': self, + 'name': name, 'run': fun if bind else staticmethod(fun), '_decorated': True, '__doc__': fun.__doc__, '__module__': fun.__module__, '__wrapped__': fun}, **options))() + if T.name not in self._tasks: + self._tasks.register(T) + T.bind(self) # connects task to this app task = self._tasks[T.name] # return global instance. return task diff --git a/celery/app/builtins.py b/celery/app/builtins.py index 0100cb86c..4144607b9 100644 --- a/celery/app/builtins.py +++ b/celery/app/builtins.py @@ -27,8 +27,7 @@ def add_backend_cleanup_task(app): :program:`celery beat` to be running). """ - @app.task(name='celery.backend_cleanup', - shared=False, _force_evaluate=True) + @app.task(name='celery.backend_cleanup', shared=False, lazy=False) def backend_cleanup(): app.backend.cleanup() return backend_cleanup @@ -48,7 +47,7 @@ def add_unlock_chord_task(app): default_propagate = app.conf.CELERY_CHORD_PROPAGATES @app.task(name='celery.chord_unlock', max_retries=None, shared=False, - default_retry_delay=1, ignore_result=True, _force_evaluate=True) + default_retry_delay=1, ignore_result=True, lazy=False) def unlock_chord(group_id, callback, interval=None, propagate=None, max_retries=None, result=None, Result=app.AsyncResult, GroupResult=app.GroupResult, @@ -106,7 +105,7 @@ def unlock_chord(group_id, callback, interval=None, propagate=None, def add_map_task(app): from celery.canvas import signature - @app.task(name='celery.map', shared=False, _force_evaluate=True) + @app.task(name='celery.map', shared=False, lazy=False) def xmap(task, it): task = signature(task, app=app).type return [task(item) for item in it] @@ -117,7 +116,7 @@ def xmap(task, it): def add_starmap_task(app): from celery.canvas import signature - @app.task(name='celery.starmap', shared=False, _force_evaluate=True) + @app.task(name='celery.starmap', shared=False, lazy=False) def xstarmap(task, it): task = signature(task, app=app).type return [task(*item) for item in it] @@ -128,7 +127,7 @@ def xstarmap(task, it): def add_chunk_task(app): from celery.canvas import chunks as _chunks - @app.task(name='celery.chunks', shared=False, _force_evaluate=True) + @app.task(name='celery.chunks', shared=False, lazy=False) def chunks(task, it, n): return _chunks.apply_chunks(task, it, n) return chunks @@ -137,43 +136,35 @@ def chunks(task, it, n): @connect_on_app_finalize def add_group_task(app): """No longer used, but here for backwards compatibility.""" - _app = app from celery.canvas import maybe_signature from celery.result import result_from_tuple - class Group(app.Task): - app = _app - name = 'celery.group' - _decorated = True - - def run(self, tasks, result, group_id, partial_args, - add_to_parent=True): - app = self.app - result = result_from_tuple(result, app) - # any partial args are added to all tasks in the group - taskit = (maybe_signature(task, app=app).clone(partial_args) - for i, task in enumerate(tasks)) - with app.producer_or_acquire() as pub: - [stask.apply_async(group_id=group_id, producer=pub, - add_to_parent=False) for stask in taskit] - parent = get_current_worker_task() - if add_to_parent and parent: - parent.add_trail(result) - return result - return Group + @app.task(name='celery.group', bind=True, shared=False, lazy=False) + def group(self, tasks, result, group_id, partial_args, add_to_parent=True): + app = self.app + result = result_from_tuple(result, app) + # any partial args are added to all tasks in the group + taskit = (maybe_signature(task, app=app).clone(partial_args) + for i, task in enumerate(tasks)) + with app.producer_or_acquire() as producer: + [stask.apply_async(group_id=group_id, producer=producer, + add_to_parent=False) for stask in taskit] + parent = get_current_worker_task() + if add_to_parent and parent: + parent.add_trail(result) + return result + return group @connect_on_app_finalize def add_chain_task(app): """No longer used, but here for backwards compatibility.""" - _app = app - class Chain(app.Task): - app = _app - name = 'celery.chain' - _decorated = True + @app.task(name='celery.chain', shared=False, lazy=False) + def chain(*args, **kwargs): + raise NotImplementedError('chain is not a real task') + return chain - return Chain @connect_on_app_finalize @@ -183,23 +174,19 @@ def add_chord_task(app): from celery.canvas import maybe_signature _app = app - class Chord(app.Task): - app = _app - name = 'celery.chord' - ignore_result = False - _decorated = True - - def run(self, header, body, partial_args=(), interval=None, - countdown=1, max_retries=None, propagate=None, - eager=False, **kwargs): - app = self.app - # - convert back to group if serialized - tasks = header.tasks if isinstance(header, group) else header - header = group([ - maybe_signature(s, app=app) for s in tasks - ], app=self.app) - body = maybe_signature(body, app=app) - ch = _chord(header, body) - return ch.run(header, body, partial_args, app, interval, - countdown, max_retries, propagate, **kwargs) - return Chord + @app.task(name='celery.chord', bind=True, ignore_result=False, + shared=False, lazy=False) + def chord(self, header, body, partial_args=(), interval=None, + countdown=1, max_retries=None, propagate=None, + eager=False, **kwargs): + app = self.app + # - convert back to group if serialized + tasks = header.tasks if isinstance(header, group) else header + header = group([ + maybe_signature(s, app=app) for s in tasks + ], app=self.app) + body = maybe_signature(body, app=app) + ch = _chord(header, body) + return ch.run(header, body, partial_args, app, interval, + countdown, max_retries, propagate, **kwargs) + return chord diff --git a/celery/app/task.py b/celery/app/task.py index 1fa6b3381..c8b04b9ce 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -17,10 +17,9 @@ from celery._state import _task_stack from celery.canvas import signature from celery.exceptions import MaxRetriesExceededError, Reject, Retry -from celery.five import class_property, items, with_metaclass -from celery.local import Proxy +from celery.five import class_property, items from celery.result import EagerResult -from celery.utils import gen_task_name, uuid, maybe_reraise +from celery.utils import uuid, maybe_reraise from celery.utils.functional import mattrgetter, maybe_list from celery.utils.imports import instantiate from celery.utils.mail import ErrorMail @@ -45,22 +44,6 @@ R_INSTANCE = '<@task: {0.name} of {app}{flags}>' -class _CompatShared(object): - - def __init__(self, name, cons): - self.name = name - self.cons = cons - - def __hash__(self): - return hash(self.name) - - def __repr__(self): - return '' % (self.name, ) - - def __call__(self, app): - return self.cons(app) - - def _strflags(flags, default=''): if flags: return ' ({0})'.format(', '.join(flags)) @@ -130,84 +113,6 @@ def children(self): return self._children -class TaskType(type): - """Meta class for tasks. - - Automatically registers the task in the task registry (except - if the :attr:`Task.abstract`` attribute is set). - - If no :attr:`Task.name` attribute is provided, then the name is generated - from the module and class name. - - """ - _creation_count = {} # used by old non-abstract task classes - - def __new__(cls, name, bases, attrs): - new = super(TaskType, cls).__new__ - task_module = attrs.get('__module__') or '__main__' - - # - Abstract class: abstract attribute should not be inherited. - abstract = attrs.pop('abstract', None) - if abstract or not attrs.get('autoregister', True): - return new(cls, name, bases, attrs) - - # The 'app' attribute is now a property, with the real app located - # in the '_app' attribute. Previously this was a regular attribute, - # so we should support classes defining it. - app = attrs.pop('_app', None) or attrs.pop('app', None) - - # Attempt to inherit app from one the bases - if not isinstance(app, Proxy) and app is None: - for base in bases: - if getattr(base, '_app', None): - app = base._app - break - else: - app = current_app._get_current_object() - attrs['_app'] = app - - # - Automatically generate missing/empty name. - task_name = attrs.get('name') - if not task_name: - attrs['name'] = task_name = gen_task_name(app, name, task_module) - - if not attrs.get('_decorated'): - # non decorated tasks must also be shared in case - # an app is created multiple times due to modules - # imported under multiple names. - # Hairy stuff, here to be compatible with 2.x. - # People should not use non-abstract task classes anymore, - # use the task decorator. - from celery._state import connect_on_app_finalize - unique_name = '.'.join([task_module, name]) - if unique_name not in cls._creation_count: - # the creation count is used as a safety - # so that the same task is not added recursively - # to the set of constructors. - cls._creation_count[unique_name] = 1 - connect_on_app_finalize(_CompatShared( - unique_name, - lambda app: TaskType.__new__(cls, name, bases, - dict(attrs, _app=app)), - )) - - # - Create and register class. - # Because of the way import happens (recursively) - # we may or may not be the first time the task tries to register - # with the framework. There should only be one class for each task - # name, so we always return the registered version. - tasks = app._tasks - if task_name not in tasks: - tasks.register(new(cls, name, bases, attrs)) - instance = tasks[task_name] - instance.bind(app) - return instance.__class__ - - def __repr__(cls): - return _reprtask(cls) - - -@with_metaclass(TaskType) class Task(object): """Task base class. diff --git a/celery/task/base.py b/celery/task/base.py index d8ff5e880..8412b9517 100644 --- a/celery/task/base.py +++ b/celery/task/base.py @@ -14,12 +14,14 @@ from kombu import Exchange from celery import current_app -from celery.app.task import Context, TaskType, Task as BaseTask # noqa -from celery.five import class_property, reclassmethod +from celery.app.task import Context, Task as BaseTask, _reprtask +from celery.five import class_property, reclassmethod, with_metaclass +from celery.local import Proxy from celery.schedules import maybe_schedule +from celery.utils import gen_task_name from celery.utils.log import get_task_logger -__all__ = ['Task', 'PeriodicTask', 'task'] +__all__ = ['Context', 'Task', 'TaskType', 'PeriodicTask', 'task'] #: list of methods that must be classmethods in the old API. _COMPAT_CLASSMETHODS = ( @@ -29,6 +31,100 @@ ) +class _CompatShared(object): + + def __init__(self, name, cons): + self.name = name + self.cons = cons + + def __hash__(self): + return hash(self.name) + + def __repr__(self): + return '' % (self.name, ) + + def __call__(self, app): + return self.cons(app) + + +class TaskType(type): + """Meta class for tasks. + + Automatically registers the task in the task registry (except + if the :attr:`Task.abstract`` attribute is set). + + If no :attr:`Task.name` attribute is provided, then the name is generated + from the module and class name. + + """ + _creation_count = {} # used by old non-abstract task classes + + def __new__(cls, name, bases, attrs): + new = super(TaskType, cls).__new__ + task_module = attrs.get('__module__') or '__main__' + + # - Abstract class: abstract attribute should not be inherited. + abstract = attrs.pop('abstract', None) + if abstract or not attrs.get('autoregister', True): + return new(cls, name, bases, attrs) + + # The 'app' attribute is now a property, with the real app located + # in the '_app' attribute. Previously this was a regular attribute, + # so we should support classes defining it. + app = attrs.pop('_app', None) or attrs.pop('app', None) + + # Attempt to inherit app from one the bases + if not isinstance(app, Proxy) and app is None: + for base in bases: + if getattr(base, '_app', None): + app = base._app + break + else: + app = current_app._get_current_object() + attrs['_app'] = app + + # - Automatically generate missing/empty name. + task_name = attrs.get('name') + if not task_name: + attrs['name'] = task_name = gen_task_name(app, name, task_module) + + if not attrs.get('_decorated'): + # non decorated tasks must also be shared in case + # an app is created multiple times due to modules + # imported under multiple names. + # Hairy stuff, here to be compatible with 2.x. + # People should not use non-abstract task classes anymore, + # use the task decorator. + from celery._state import connect_on_app_finalize + unique_name = '.'.join([task_module, name]) + if unique_name not in cls._creation_count: + # the creation count is used as a safety + # so that the same task is not added recursively + # to the set of constructors. + cls._creation_count[unique_name] = 1 + connect_on_app_finalize(_CompatShared( + unique_name, + lambda app: TaskType.__new__(cls, name, bases, + dict(attrs, _app=app)), + )) + + # - Create and register class. + # Because of the way import happens (recursively) + # we may or may not be the first time the task tries to register + # with the framework. There should only be one class for each task + # name, so we always return the registered version. + tasks = app._tasks + if task_name not in tasks: + tasks.register(new(cls, name, bases, attrs)) + instance = tasks[task_name] + instance.bind(app) + return instance.__class__ + + def __repr__(cls): + return _reprtask(cls) + + +@with_metaclass(TaskType) class Task(BaseTask): """Deprecated Task base class. diff --git a/celery/tests/app/test_builtins.py b/celery/tests/app/test_builtins.py index 305877f47..89e23c6e9 100644 --- a/celery/tests/app/test_builtins.py +++ b/celery/tests/app/test_builtins.py @@ -78,7 +78,7 @@ def chunks_mul(l): class test_group(BuiltinsCase): def setup(self): - self.task = builtins.add_group_task(self.app)() + self.task = builtins.add_group_task(self.app) super(test_group, self).setup() def test_apply_async_eager(self): @@ -125,7 +125,7 @@ class test_chain(BuiltinsCase): def setup(self): BuiltinsCase.setup(self) - self.task = builtins.add_chain_task(self.app)() + self.task = builtins.add_chain_task(self.app) def test_apply_async(self): c = self.add.s(2, 2) | self.add.s(4) | self.add.s(8) @@ -181,7 +181,7 @@ def s(*args, **kwargs): class test_chord(BuiltinsCase): def setup(self): - self.task = builtins.add_chord_task(self.app)() + self.task = builtins.add_chord_task(self.app) super(test_chord, self).setup() def test_apply_async(self): diff --git a/celery/tests/tasks/test_tasks.py b/celery/tests/tasks/test_tasks.py index 35d71a132..8d9da1f46 100644 --- a/celery/tests/tasks/test_tasks.py +++ b/celery/tests/tasks/test_tasks.py @@ -358,11 +358,6 @@ def test_context_get(self): finally: self.mytask.pop_request() - def test_task_class_repr(self): - self.assertIn('class Task of', repr(self.mytask.app.Task)) - self.mytask.app.Task._app = None - self.assertIn('unbound', repr(self.mytask.app.Task, )) - def test_annotate(self): with patch('celery.app.task.resolve_all_annotations') as anno: anno.return_value = [{'FOO': 'BAR'}] diff --git a/celery/utils/__init__.py b/celery/utils/__init__.py index 2e31c9f68..5661f6dfd 100644 --- a/celery/utils/__init__.py +++ b/celery/utils/__init__.py @@ -298,6 +298,7 @@ def jsonify(obj, def gen_task_name(app, name, module_name): """Generate task name from name/module pair.""" + module_name = module_name or '__main__' try: module = sys.modules[module_name] except KeyError: From 0624888cfb7cc74f7fd790ff07b6d04aec27686e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 May 2014 20:27:21 +0100 Subject: [PATCH 0195/1103] Cosmetics --- celery/app/base.py | 33 ++++++++++++++++----------------- celery/app/task.py | 3 +++ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 0e320ec7f..e99affd36 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -265,26 +265,25 @@ def _create_task_cls(fun): sum([len(args), len(opts)]))) return inner_create_task_cls(**opts) - def _task_from_fun(self, fun, name=None, **options): + def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): if not self.finalized and not self.autofinalize: raise RuntimeError('Contract breach: app not finalized') - base = options.pop('base', None) or self.Task - bind = options.pop('bind', False) - name = name or gen_task_name(self, fun.__name__, fun.__module__) - - T = type(fun.__name__, (base, ), dict({ - 'app': self, - 'name': name, - 'run': fun if bind else staticmethod(fun), - '_decorated': True, - '__doc__': fun.__doc__, - '__module__': fun.__module__, - '__wrapped__': fun}, **options))() - if T.name not in self._tasks: - self._tasks.register(T) - T.bind(self) # connects task to this app - task = self._tasks[T.name] # return global instance. + base = base or self.Task + + if name not in self._tasks: + task = type(fun.__name__, (base, ), dict({ + 'app': self, + 'name': name, + 'run': fun if bind else staticmethod(fun), + '_decorated': True, + '__doc__': fun.__doc__, + '__module__': fun.__module__, + '__wrapped__': fun}, **options))() + self._tasks[task.name] = task + task.bind(self) # connects task to this app + else: + task = self._tasks[name] return task def finalize(self, auto=False): diff --git a/celery/app/task.py b/celery/app/task.py index c8b04b9ce..6bc1f2b54 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -43,6 +43,9 @@ R_SELF_TASK = '<@task {0.name} bound to other {0.__self__}>' R_INSTANCE = '<@task: {0.name} of {app}{flags}>' +#: Here for backwards compatibility as tasks no longer use a custom metaclass. +TaskType = type + def _strflags(flags, default=''): if flags: From 1a140eb03bed1181c04fb0b24dce2607ce375e34 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 30 May 2014 13:56:30 +0100 Subject: [PATCH 0196/1103] Events: Fixes Django timezone problems. Closes #1802 --- celery/utils/timeutils.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/celery/utils/timeutils.py b/celery/utils/timeutils.py index 453be35de..494aa6f45 100644 --- a/celery/utils/timeutils.py +++ b/celery/utils/timeutils.py @@ -49,9 +49,6 @@ _local_timezone = None -__timezone__ = -_time.timezone -__altzone__ = -_time.altzone - class LocalTimezone(tzinfo): """Local time implementation taken from Python's docs. @@ -333,10 +330,10 @@ def _fields(self, **extra): }, **extra) -def utcoffset(): - if _time.daylight: - return __altzone__ // 3600 - return __timezone__ // 3600 +def utcoffset(time=_time): + if time.daylight: + return time.altzone // 3600 + return time.timezone // 3600 def adjust_timestamp(ts, offset, here=utcoffset): From 740d4d17bca099afdc6e6251b4f4a78c99f8005c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 30 May 2014 13:57:47 +0100 Subject: [PATCH 0197/1103] Django: Call django.setup always if available --- celery/fixups/django.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index ab20325f5..f9ad33119 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -134,13 +134,16 @@ def __init__(self, app): ) def validate_models(self): + import django + try: + django.setup() + except AttributeError: + pass s = io.StringIO() try: from django.core.management.validation import get_validation_errors except ImportError: - import django from django.core.management.base import BaseCommand - django.setup() cmd = BaseCommand() cmd.stdout, cmd.stderr = sys.stdout, sys.stderr cmd.check() From 00ed1e2feea348686ca73fbe8f543161b2937b45 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 30 May 2014 14:06:06 +0100 Subject: [PATCH 0198/1103] Do not document the CHORD_PROPAGATES setting anymore --- docs/configuration.rst | 23 ----------------------- docs/userguide/canvas.rst | 11 +---------- extra/release/verify_config_reference.py | 1 + 3 files changed, 2 insertions(+), 33 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 06bf08446..3d8f2f2d8 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1218,29 +1218,6 @@ This is the total number of results to cache before older results are evicted. The default is 5000. 0 or None means no limit, and a value of :const:`-1` will disable the cache. -.. setting:: CELERY_CHORD_PROPAGATES - -CELERY_CHORD_PROPAGATES -~~~~~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 3.0.14 - -This setting defines what happens when a task part of a chord raises an -exception: - -- If propagate is True the chord callback will change state to FAILURE - with the exception value set to a :exc:`~@ChordError` - instance containing information about the error and the task that failed. - - This is the default behavior in Celery 3.1+ - -- If propagate is False the exception value will instead be forwarded - to the chord callback. - - This was the default behavior before version 3.1. - -.. setting:: CELERY_TRACK_STARTED - CELERY_TRACK_STARTED ~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/userguide/canvas.rst b/docs/userguide/canvas.rst index f74e8e4be..5f86032f0 100644 --- a/docs/userguide/canvas.rst +++ b/docs/userguide/canvas.rst @@ -721,11 +721,7 @@ Error handling So what happens if one of the tasks raises an exception? -This was not documented for some time and before version 3.1 -the exception value will be forwarded to the chord callback. - - -From 3.1 errors will propagate to the callback, so the callback will not be executed +Errors will propagate to the callback, so the callback will not be executed instead the callback changes to failure state, and the error is set to the :exc:`~@ChordError` exception: @@ -743,11 +739,6 @@ to the :exc:`~@ChordError` exception: celery.exceptions.ChordError: Dependency 97de6f3f-ea67-4517-a21c-d867c61fcb47 raised ValueError('something something',) -If you're running 3.0.14 or later you can enable the new behavior via -the :setting:`CELERY_CHORD_PROPAGATES` setting:: - - CELERY_CHORD_PROPAGATES = True - While the traceback may be different depending on which result backend is being used, you can see the error description includes the id of the task that failed and a string representation of the original exception. You can also diff --git a/extra/release/verify_config_reference.py b/extra/release/verify_config_reference.py index 8f4038cc4..36183d8bb 100644 --- a/extra/release/verify_config_reference.py +++ b/extra/release/verify_config_reference.py @@ -13,6 +13,7 @@ 'BROKER_PASSWORD', 'BROKER_VHOST', 'BROKER_PORT', + 'CELERY_CHORD_PROPAGATES', 'CELERY_REDIS_HOST', 'CELERY_REDIS_PORT', 'CELERY_REDIS_DB', From c965f80ef30296f280fc5cc8cd1e55dc79094970 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 30 May 2014 14:08:49 +0100 Subject: [PATCH 0199/1103] 3.1 release notes did not announce the chord error behavior being defined. Closes #1805 --- docs/whatsnew-3.1.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/whatsnew-3.1.rst b/docs/whatsnew-3.1.rst index 99b6c3ad3..93f04978d 100644 --- a/docs/whatsnew-3.1.rst +++ b/docs/whatsnew-3.1.rst @@ -706,6 +706,13 @@ In Other News >>> g() +- Chord exception behavior defined (Issue #1172). + + From this version the chord callback will change state to FAILURE + when a task part of a chord raises an exception. + + See more at :ref:`chord-errors`. + - New ability to specify additional command line options to the worker and beat programs. From 1607a42fe425ed0708725949967b95b4b048d4c2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Sun, 1 Jun 2014 16:10:33 +0100 Subject: [PATCH 0200/1103] Docs now requires kombu master --- requirements/docs.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/docs.txt b/requirements/docs.txt index 1d142c296..f191f06f1 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,3 +1,4 @@ Sphinx SQLAlchemy https://github.com/celery/py-amqp/zipball/master#egg=amqp +https://github.com/celery/kombu/zipball/master#egg=kombu From 4e960c51a24cd17845e6fc9ad05980892b7962fe Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Sun, 1 Jun 2014 16:16:51 +0100 Subject: [PATCH 0201/1103] Docs: force update of amqp+kombu --- requirements/docs.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/docs.txt b/requirements/docs.txt index f191f06f1..70028e681 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,4 +1,4 @@ Sphinx SQLAlchemy -https://github.com/celery/py-amqp/zipball/master#egg=amqp -https://github.com/celery/kombu/zipball/master#egg=kombu +https://github.com/celery/py-amqp/zipball/master +https://github.com/celery/kombu/zipball/master From 694a859fd79459230e07544c4126ab242e0d2ee5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 2 Jun 2014 15:07:06 +0100 Subject: [PATCH 0202/1103] Documents the event messsage protocol --- docs/internals/protocol.rst | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/docs/internals/protocol.rst b/docs/internals/protocol.rst index c51aa396f..6b3167c78 100644 --- a/docs/internals/protocol.rst +++ b/docs/internals/protocol.rst @@ -295,3 +295,82 @@ The MIME-types supported by default are shown in the following table. pickle application/x-python-serialize msgpack application/x-msgpack =============== ================================= + +.. _message-protocol-event: + +Event Messages +============== + +Event messages are always JSON serialized and can contain arbitrary message +body fields. + +Since version 3.2. the body can consist of either a single mapping (one event), +or a list of mappings (multiple events). + +There are also standard fields that must always be present in an event +message: + +Standard body fields +-------------------- + +- *string* ``type`` + + The type of event. This is a string containing the *category* and + *action* separated by a dash delimeter (e.g. ``task-succeeded``). + +- *string* ``hostname`` + + The fully qualified hostname of where the event occurred at. + +- *unsigned long long* ``clock`` + + The logical clock value for this event (Lamport timestamp). + +- *float* ``timestamp`` + + The UNIX timestamp corresponding to the time of when the event occurred. + +- *signed short* ``utcoffset`` + + This field describes the timezone of the originating host, and is + specified as the number of hours ahead of/behind UTC. E.g. ``-2`` or + ``+1``. + +- *unsigned long long* ``pid`` + + The process id of the process the event originated in. + +Standard event types +-------------------- + +For a list of standard event types and their fields see the +:ref:`event-reference`. + +Example message +--------------- + +This is the message fields for a ``task-succeeded`` event: + +.. code-block:: python + + properties = { + 'routing_key': 'task.succeeded', + 'exchange': 'celeryev', + 'content_type': 'application/json', + 'content_encoding': 'utf-8', + 'delivery_mode': 1, + } + headers = { + 'hostname': 'worker1@george.vandelay.com', + } + body = { + 'type': 'task-succeeded', + 'hostname': 'worker1@george.vandelay.com', + 'pid': 6335, + 'clock': 393912923921, + 'timestamp': 1401717709.101747, + 'utcoffset': -1, + 'uuid': '9011d855-fdd1-4f8f-adb3-a413b499eafb', + 'retval': '4', + 'runtime': 0.0003212, + ) From 2bd839dbc47eaa89f81f9aab16fc305c6703a3df Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 2 Jun 2014 15:58:21 +0100 Subject: [PATCH 0203/1103] Remote control doc improvements --- .../first-steps-with-celery.rst | 2 +- docs/history/changelog-2.2.rst | 2 +- docs/userguide/workers.rst | 20 +++++++++++-------- docs/whatsnew-3.0.rst | 10 +++++----- docs/whatsnew-3.1.rst | 7 ++++--- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/docs/getting-started/first-steps-with-celery.rst b/docs/getting-started/first-steps-with-celery.rst index c79f5dcbd..b3ea28706 100644 --- a/docs/getting-started/first-steps-with-celery.rst +++ b/docs/getting-started/first-steps-with-celery.rst @@ -379,7 +379,7 @@ for the task at runtime: .. code-block:: bash - $ celery control rate_limit tasks.add 10/m + $ celery -A tasks control rate_limit tasks.add 10/m worker@example.com: OK new rate limit set successfully diff --git a/docs/history/changelog-2.2.rst b/docs/history/changelog-2.2.rst index 2f8ba7894..5db27d0a7 100644 --- a/docs/history/changelog-2.2.rst +++ b/docs/history/changelog-2.2.rst @@ -371,7 +371,7 @@ Fixes objects with a broken ``__repr__`` does not crash the worker, or otherwise make errors hard to understand (Issue #298). -* Remote control command ``active_queues``: did not account for queues added +* Remote control command :control:`active_queues`: did not account for queues added at runtime. In addition the dictionary replied by this command now has a different diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index ef98f7db4..4bfa93b7a 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -290,12 +290,16 @@ Of course, using the higher-level interface to set rate limits is much more convenient, but there are commands that can only be requested using :meth:`~@control.broadcast`. +Commands +======== + .. control:: revoke -Revoking tasks -============== -pool support: all -broker support: *amqp, redis* +``revoke``: Revoking tasks +-------------------------- +:pool support: all +:broker support: *amqp, redis* +:command: :program:`celery -A proj control revoke ` All worker nodes keeps a memory of revoked task ids, either in-memory or persistent on disk (see :ref:`worker-persistent-revokes`). @@ -580,7 +584,7 @@ named "``foo``" you can use the :program:`celery control` program: .. code-block:: bash - $ celery control add_consumer foo + $ celery -A proj control add_consumer foo -> worker1.local: OK started consuming from u'foo' @@ -589,7 +593,7 @@ If you want to specify a specific worker you can use the .. code-block:: bash - $ celery control add_consumer foo -d worker1.local + $ celery -A proj control add_consumer foo -d worker1.local The same can be accomplished dynamically using the :meth:`@control.add_consumer` method:: @@ -631,14 +635,14 @@ you can use the :program:`celery control` program: .. code-block:: bash - $ celery control cancel_consumer foo + $ celery -A proj control cancel_consumer foo The :option:`--destination` argument can be used to specify a worker, or a list of workers, to act on the command: .. code-block:: bash - $ celery control cancel_consumer foo -d worker1.local + $ celery -A proj control cancel_consumer foo -d worker1.local You can also cancel consumers programmatically using the diff --git a/docs/whatsnew-3.0.rst b/docs/whatsnew-3.0.rst index bd0136eb0..8c2f83102 100644 --- a/docs/whatsnew-3.0.rst +++ b/docs/whatsnew-3.0.rst @@ -457,7 +457,7 @@ New remote control commands These commands were previously experimental, but they have proven stable and is now documented as part of the offical API. -- ``add_consumer``/``cancel_consumer`` +- :control:`add_consumer`/:control:`cancel_consumer` Tells workers to consume from a new queue, or cancel consuming from a queue. This command has also been changed so that the worker remembers @@ -486,7 +486,7 @@ stable and is now documented as part of the offical API. Remember that a control command without *destination* will be sent to **all workers**. -- ``autoscale`` +- :control:`autoscale` Tells workers with `--autoscale` enabled to change autoscale max/min concurrency settings. @@ -504,7 +504,7 @@ stable and is now documented as part of the offical API. $ celery control -d w1.example.com autoscale 10 5 -- ``pool_grow``/``pool_shrink`` +- :control:`pool_grow`/:control:`pool_shrink` Tells workers to add or remove pool processes. @@ -523,8 +523,8 @@ stable and is now documented as part of the offical API. $ celery control -d w1.example.com pool_grow 2 $ celery control -d w1.example.com pool_shrink 2 -- :program:`celery control` now supports ``rate_limit`` & ``time_limit`` - commands. +- :program:`celery control` now supports :control:`rate_limit` and + :control:`time_limit` commands. See ``celery control --help`` for details. diff --git a/docs/whatsnew-3.1.rst b/docs/whatsnew-3.1.rst index 93f04978d..49a75b511 100644 --- a/docs/whatsnew-3.1.rst +++ b/docs/whatsnew-3.1.rst @@ -76,7 +76,7 @@ so I cannot recommend them for production use. The next version of Celery 3.2 will focus on performance and removing rarely used parts of the library. Work has also started on a new message protocol, supporting multiple languages and more. The initial draft can -be found :ref:`here `. +be found :ref:`here Date: Mon, 2 Jun 2014 16:23:37 +0100 Subject: [PATCH 0204/1103] inspect registers now ignores built-in tasks --- celery/worker/control.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/celery/worker/control.py b/celery/worker/control.py index d0b119d85..da4a52dc3 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -274,10 +274,13 @@ def hello(state, from_node, revoked=None, **kwargs): @Panel.register -def dump_tasks(state, taskinfoitems=None, **kwargs): - tasks = state.app.tasks +def dump_tasks(state, taskinfoitems=None, builtins=False, **kwargs): + reg = state.app.tasks taskinfoitems = taskinfoitems or DEFAULT_TASK_INFO_ITEMS + tasks = reg if builtins else ( + task for task in reg if not task.startswith('celery.')) + def _extract_info(task): fields = { field: str(getattr(task, field, None)) for field in taskinfoitems @@ -288,7 +291,7 @@ def _extract_info(task): return '{0} [{1}]'.format(task.name, ' '.join(info)) return task.name - return [_extract_info(tasks[task]) for task in sorted(tasks)] + return [_extract_info(reg[task]) for task in sorted(tasks)] @Panel.register From 03399b4d7c26fb593e61acf34f111b66b340ba4e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 3 Jun 2014 15:27:11 +0100 Subject: [PATCH 0205/1103] Events are now buffered up in the worker and sent as a list. Not using timers, so will only buffer up to $concurrency events --- celery/events/__init__.py | 88 ++++++++++++++++++++++++++------------ celery/utils/functional.py | 1 - celery/worker/consumer.py | 10 +++++ 3 files changed, 70 insertions(+), 29 deletions(-) diff --git a/celery/events/__init__.py b/celery/events/__init__.py index 93747cf56..c5cc03df1 100644 --- a/celery/events/__init__.py +++ b/celery/events/__init__.py @@ -14,7 +14,7 @@ import time import threading -from collections import deque +from collections import defaultdict, deque from contextlib import contextmanager from copy import copy from operator import itemgetter @@ -25,6 +25,7 @@ from kombu.utils import cached_property from celery.app import app_or_default +from celery.five import items from celery.utils import anon_nodename, uuid from celery.utils.functional import dictfilter from celery.utils.timeutils import adjust_timestamp, utcoffset, maybe_s_to_ms @@ -115,12 +116,17 @@ class EventDispatcher(object): def __init__(self, connection=None, hostname=None, enabled=True, channel=None, buffer_while_offline=True, app=None, - serializer=None, groups=None, delivery_mode=1): + serializer=None, groups=None, delivery_mode=1, + buffer_group=None, buffer_limit=24, on_send_buffered=None): self.app = app_or_default(app or self.app) self.connection = connection self.channel = channel self.hostname = hostname or anon_nodename() self.buffer_while_offline = buffer_while_offline + self.buffer_group = buffer_group or frozenset() + self.buffer_limit = buffer_limit + self.on_send_buffered = on_send_buffered + self._group_buffer = defaultdict(list) self.mutex = threading.Lock() self.producer = None self._outbound_buffer = deque() @@ -164,9 +170,8 @@ def disable(self): for callback in self.on_disabled: callback() - def publish(self, type, fields, producer, retry=False, - retry_policy=None, blind=False, utcoffset=utcoffset, - Event=Event): + def publish(self, type, fields, producer, + blind=False, Event=Event, **kwargs): """Publish event using a custom :class:`~kombu.Producer` instance. @@ -184,15 +189,20 @@ def publish(self, type, fields, producer, retry=False, :keyword utcoffset: Function returning the current utcoffset in hours. """ - + clock = None if blind else self.clock.forward() + event = Event(type, hostname=self.hostname, utcoffset=utcoffset(), + pid=self.pid, clock=clock, **fields) with self.mutex: - clock = None if blind else self.clock.forward() - event = Event(type, hostname=self.hostname, utcoffset=utcoffset(), - pid=self.pid, clock=clock, **fields) - exchange = self.exchange + return self._publish(event, producer, + routing_key=type.replace('-', '.'), **kwargs) + + def _publish(self, event, producer, routing_key, retry=False, + retry_policy=None, utcoffset=utcoffset): + exchange = self.exchange + try: producer.publish( event, - routing_key=type.replace('-', '.'), + routing_key=routing_key, exchange=exchange.name, retry=retry, retry_policy=retry_policy, @@ -201,8 +211,12 @@ def publish(self, type, fields, producer, retry=False, headers=self.headers, delivery_mode=self.delivery_mode, ) + except Exception as exc: + if not self.buffer_while_offline: + raise + self._outbound_buffer.append((event, routing_key, exc)) - def send(self, type, blind=False, **fields): + def send(self, type, blind=False, utcoffset=utcoffset, **fields): """Send event. :param type: Event type name, with group separated by dash (`-`). @@ -218,24 +232,38 @@ def send(self, type, blind=False, **fields): """ if self.enabled: - groups = self.groups - if groups and group_from(type) not in groups: + groups, group = self.groups, group_from(type) + if groups and group not in groups: return - try: - self.publish(type, fields, self.producer, blind) - except Exception as exc: - if not self.buffer_while_offline: - raise - self._outbound_buffer.append((type, fields, exc)) + if group in self.buffer_group: + clock = self.clock.forward() + event = Event(type, hostname=self.hostname, + utcoffset=utcoffset(), + pid=self.pid, clock=clock, **fields) + buf = self._group_buffer[group] + buf.append(event) + if len(buf) >= self.buffer_limit: + self.flush() + elif self.on_send_buffered: + self.on_send_buffered() + else: + return self.publish(type, fields, self.producer, blind) - def flush(self): + def flush(self, errors=True, groups=True): """Flushes the outbound buffer.""" - while self._outbound_buffer: + if errors: + buf = list(self._outbound_buffer) try: - type, fields, _ = self._outbound_buffer.popleft() - except IndexError: - return - self.send(type, **fields) + with self.mutex: + for event, routing_key, _ in buf: + self._publish(event, self.producer, routing_key) + finally: + self._outbound_buffer.clear() + if groups: + with self.mutex: + for group, events in items(self._group_buffer): + self._publish(events, self.producer, '%s.multi' % group) + events[:] = [] # list.clear def extend_buffer(self, other): """Copies the outbound buffer of another instance.""" @@ -357,8 +385,12 @@ def event_from_message(self, body, localize=True, body['local_received'] = now() return type, body - def _receive(self, body, message): - self.process(*self.event_from_message(body)) + def _receive(self, body, message, list=list, isinstance=isinstance): + if isinstance(body, list): # 3.2: List of events + process, from_message = self.process, self.event_from_message + [process(*from_message(event)) for event in body] + else: + self.process(*self.event_from_message(body)) @property def connection(self): diff --git a/celery/utils/functional.py b/celery/utils/functional.py index be7a2289c..c696a17b1 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -239,7 +239,6 @@ def chunks(it, n): [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10]] """ - # XXX This function is not used anymore, at least not by Celery itself. for first in it: yield [first] + list(islice(it, n - 1)) diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 7abbf74e0..2dfc5b399 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -371,6 +371,14 @@ def _error_handler(exc, interval, next_step=CONNECTION_RETRY_STEP): conn.transport.register_with_event_loop(conn.connection, self.hub) return conn + def _flush_events(self): + if self.event_dispatcher: + self.event_dispatcher.flush() + + def on_send_event_buffered(self): + if self.hub: + self.hub._ready.add(self._flush_events) + def add_task_queue(self, queue, exchange=None, exchange_type=None, routing_key=None, **options): cset = self.task_consumer @@ -516,6 +524,8 @@ def start(self, c): dis = c.event_dispatcher = c.app.events.Dispatcher( c.connect(), hostname=c.hostname, enabled=self.send_events, groups=self.groups, + buffer_group=['task'] if c.hub else None, + on_send_buffered=c.on_send_event_buffered if c.hub else None, ) if prev: dis.extend_buffer(prev) From e2f05fe02c64d56a6d2ba6df16a0aaab9c067ddb Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 4 Jun 2014 22:46:36 +0100 Subject: [PATCH 0206/1103] Prefork: Use i % total to cycle between ready_fds instead of shuffle --- celery/concurrency/asynpool.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 509823082..e209f6e1f 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -668,14 +668,25 @@ def on_inqueue_close(fd, proc): pass self.on_inqueue_close = on_inqueue_close - def schedule_writes(ready_fds, shuffle=random.shuffle): + def schedule_writes(ready_fds, shuffle=random.shuffle, curindex=[0]): # Schedule write operation to ready file descriptor. # The file descriptor is writeable, but that does not # mean the process is currently reading from the socket. # The socket is buffered so writeable simply means that # the buffer can accept at least 1 byte of data. - shuffle(ready_fds) - for ready_fd in ready_fds: + + # This means we have to cycle between the ready fds. + # the first version used shuffle, but using i % total + # is about 30% faster with many processes. The latter + # also shows more fairness in write stats when used with + # many processes [XXX On OS X, this may vary depending + # on event loop implementation (i.e select vs epoll), so + # have to test further] + total = len(ready_fds) + + for i in range(total): + ready_fd = ready_fds[curindex[0] % total] + curindex[0] += 1 if ready_fd in active_writes: # already writing to this fd continue From 2707314760923f9f5996ed9f36aa3bf64ecc2bbf Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 4 Jun 2014 22:47:25 +0100 Subject: [PATCH 0207/1103] Asynpool: remove outdated localization to random.shuffle --- celery/concurrency/asynpool.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index e209f6e1f..bfb5f3eca 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -20,7 +20,6 @@ import errno import os -import random import select import socket import struct @@ -668,7 +667,7 @@ def on_inqueue_close(fd, proc): pass self.on_inqueue_close = on_inqueue_close - def schedule_writes(ready_fds, shuffle=random.shuffle, curindex=[0]): + def schedule_writes(ready_fds, curindex=[0]): # Schedule write operation to ready file descriptor. # The file descriptor is writeable, but that does not # mean the process is currently reading from the socket. From f06fe0adad33615995640a6d7e0e6a8c1a18f939 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 4 Jun 2014 22:59:13 +0100 Subject: [PATCH 0208/1103] Cosmetics --- celery/concurrency/asynpool.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index bfb5f3eca..d86cfabba 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -667,7 +667,7 @@ def on_inqueue_close(fd, proc): pass self.on_inqueue_close = on_inqueue_close - def schedule_writes(ready_fds, curindex=[0]): + def schedule_writes(ready_fds, total_write_count=[0]): # Schedule write operation to ready file descriptor. # The file descriptor is writeable, but that does not # mean the process is currently reading from the socket. @@ -675,17 +675,18 @@ def schedule_writes(ready_fds, curindex=[0]): # the buffer can accept at least 1 byte of data. # This means we have to cycle between the ready fds. - # the first version used shuffle, but using i % total - # is about 30% faster with many processes. The latter - # also shows more fairness in write stats when used with - # many processes [XXX On OS X, this may vary depending + # the first version used shuffle, but this version + # using `total_writes % ready_fds` is about 30% faster + # with many processes, and also leans more towards fairness + # in write stats when used with many processes + # [XXX On OS X, this may vary depending # on event loop implementation (i.e select vs epoll), so # have to test further] - total = len(ready_fds) + num_ready = len(ready_fds) - for i in range(total): - ready_fd = ready_fds[curindex[0] % total] - curindex[0] += 1 + for i in range(num_ready): + ready_fd = ready_fds[total_write_count[0] % num_ready] + total_write_count[0] += 1 if ready_fd in active_writes: # already writing to this fd continue From 7ced601d88c2d90e02ef14ed2e9b39564ba69e0a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 5 Jun 2014 16:18:28 +0100 Subject: [PATCH 0209/1103] Stresstests: CSTRESS_TRANS enables transient messages --- funtests/benchmarks/bench_worker.py | 23 +++++++++++++---------- funtests/stress/stress/templates.py | 10 ++++++++-- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/funtests/benchmarks/bench_worker.py b/funtests/benchmarks/bench_worker.py index 53ba923bd..a85a7f1c2 100644 --- a/funtests/benchmarks/bench_worker.py +++ b/funtests/benchmarks/bench_worker.py @@ -2,7 +2,6 @@ import os import sys -import time os.environ.update( NOSETPS='yes', @@ -11,6 +10,7 @@ from celery import Celery, group from celery.five import range +from kombu.five import monotonic DEFAULT_ITS = 40000 @@ -24,7 +24,6 @@ BROKER_POOL_LIMIT=10, CELERYD_POOL='solo', CELERYD_PREFETCH_MULTIPLIER=0, - CELERY_DISABLE_RATE_LIMITS=True, CELERY_DEFAULT_DELIVERY_MODE=1, CELERY_QUEUES={ 'bench.worker': { @@ -43,7 +42,7 @@ def tdiff(then): - return time.time() - then + return monotonic() - then @app.task(cur=0, time_start=None, queue='bench.worker', bare=True) @@ -53,23 +52,27 @@ def it(_, n): i = it.cur if i and not i % 5000: print('({0} so far: {1}s)'.format(i, tdiff(it.subt)), file=sys.stderr) - it.subt = time.time() + it.subt = monotonic() if not i: - it.subt = it.time_start = time.time() - elif i == n - 1: + it.subt = it.time_start = monotonic() + elif i > n - 2: total = tdiff(it.time_start) print('({0} so far: {1}s)'.format(i, tdiff(it.subt)), file=sys.stderr) print('-- process {0} tasks: {1}s total, {2} tasks/s} '.format( n, total, n / (total + .0), )) - sys.exit() + import os + os._exit() it.cur += 1 def bench_apply(n=DEFAULT_ITS): - time_start = time.time() - group(it.s(i, n) for i in range(n))() - print('-- apply {0} tasks: {1}s'.format(n, time.time() - time_start)) + time_start = monotonic() + task = it._get_current_object() + with app.producer_or_acquire() as producer: + [task.apply_async((i, n), producer=producer) for i in range(n)] + #group(s(i, n) for i in range(n))() + print('-- apply {0} tasks: {1}s'.format(n, monotonic() - time_start)) def bench_work(n=DEFAULT_ITS, loglevel='CRITICAL'): diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index c6c2a0f45..bbcfd1469 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -6,7 +6,9 @@ from kombu import Exchange, Queue from kombu.utils import symbol_by_name -CSTRESS_QUEUE = os.environ.get('CSTRESS_QUEUE_NAME', 'c.stress') +CSTRESS_TRANS = os.environ.get('CSTRESS_TRANS', False) +default_queue = 'c.stress.trans' if CSTRESS_TRANS else 'c.stress' +CSTRESS_QUEUE = os.environ.get('CSTRESS_QUEUE_NAME', default_queue) templates = {} @@ -57,7 +59,9 @@ class default(object): CELERY_QUEUES = [ Queue(CSTRESS_QUEUE, exchange=Exchange(CSTRESS_QUEUE), - routing_key=CSTRESS_QUEUE), + routing_key=CSTRESS_QUEUE, + durable=not CSTRESS_TRANS, + no_ack=CSTRESS_TRANS), ] CELERY_MAX_CACHED_RESULTS = -1 BROKER_URL = os.environ.get('CSTRESS_BROKER', 'amqp://') @@ -69,6 +73,8 @@ class default(object): 'interval_step': 0.1, } CELERY_TASK_PROTOCOL = 2 + if CSTRESS_TRANS: + CELERY_DEFAULT_DELIVERY_MODE = 1 @template() From 6449e7ee2864b29515ce6bb8f7ee19c6b706abf6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 5 Jun 2014 20:15:40 +0100 Subject: [PATCH 0210/1103] Docs: Use celery -A proj consistently --- docs/faq.rst | 16 ++++---- docs/getting-started/next-steps.rst | 6 +-- docs/tutorials/daemonizing.rst | 6 ++- docs/userguide/calling.rst | 2 +- docs/userguide/concurrency/eventlet.rst | 2 +- docs/userguide/monitoring.rst | 49 +++++++++++++------------ docs/userguide/periodic-tasks.rst | 6 +-- docs/userguide/routing.rst | 16 ++++---- docs/userguide/tasks.rst | 4 +- docs/userguide/workers.rst | 14 +++---- 10 files changed, 63 insertions(+), 58 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 875798b6c..54e9df8d1 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -361,14 +361,14 @@ all configured task queues: .. code-block:: bash - $ celery purge + $ celery -A proj purge or programatically: .. code-block:: python - >>> from celery import current_app as celery - >>> celery.control.purge() + >>> from proj.celery import app + >>> app.control.purge() 1753 If you only want to purge messages from a specific queue @@ -376,7 +376,7 @@ you have to use the AMQP API or the :program:`celery amqp` utility: .. code-block:: bash - $ celery amqp queue.purge + $ celery -A proj amqp queue.purge The number 1753 is the number of messages deleted. @@ -680,8 +680,8 @@ Can I cancel the execution of a task? or if you only have the task id:: - >>> from celery import current_app as celery - >>> celery.control.revoke(task_id) + >>> from proj.celery import app + >>> app.control.revoke(task_id) .. _faq-node-not-receiving-broadcast-commands: @@ -698,8 +698,8 @@ using the :option:`-n` argument to :mod:`~celery.bin.worker`: .. code-block:: bash - $ celery worker -n worker1@%h - $ celery worker -n worker2@%h + $ celery -A proj worker -n worker1@%h + $ celery -A proj worker -n worker2@%h where ``%h`` is automatically expanded into the current hostname. diff --git a/docs/getting-started/next-steps.rst b/docs/getting-started/next-steps.rst index bc10f801a..e90e7688f 100644 --- a/docs/getting-started/next-steps.rst +++ b/docs/getting-started/next-steps.rst @@ -74,7 +74,7 @@ The :program:`celery` program can be used to start the worker: .. code-block:: bash - $ celery worker --app=proj -l info + $ celery -A proj worker -l info When the worker starts you should see a banner and some messages:: @@ -160,7 +160,7 @@ You can restart it too: .. code-block:: bash - $ celery multi restart w1 -A proj -l info + $ celery multi restart w1 -A proj -l info celery multi v3.1.1 (Cipater) > Stopping nodes... > w1.halcyon.local: TERM -> 64024 @@ -201,7 +201,7 @@ you are encouraged to put these in a dedicated directory: $ mkdir -p /var/run/celery $ mkdir -p /var/log/celery $ celery multi start w1 -A proj -l info --pidfile=/var/run/celery/%n.pid \ - --logfile=/var/log/celery/%n.pid + --logfile=/var/log/celery/%n%I.log With the multi command you can start multiple workers, and there is a powerful command-line syntax to specify arguments for different workers too, diff --git a/docs/tutorials/daemonizing.rst b/docs/tutorials/daemonizing.rst index 311ceae85..a6ad47ea3 100644 --- a/docs/tutorials/daemonizing.rst +++ b/docs/tutorials/daemonizing.rst @@ -55,10 +55,14 @@ must also export them (e.g. ``export DISPLAY=":0"``) .. code-block:: bash $ celery multi start worker1 \ + -A proj \ --pidfile="$HOME/run/celery/%n.pid" \ --logfile="$HOME/log/celery/%n%I.log" - $ celery multi restart worker1 --pidfile="$HOME/run/celery/%n.pid" + $ celery multi restart worker1 \ + -A proj \ + --logfile="$HOME/log/celery/%n%I.log" \ + --pidfile="$HOME/run/celery/%n.pid $ celery multi stopwait worker1 --pidfile="$HOME/run/celery/%n.pid" diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index 9701c4a1a..7fc2d8d72 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -468,7 +468,7 @@ the workers :option:`-Q` argument: .. code-block:: bash - $ celery worker -l info -Q celery,priority.high + $ celery -A proj worker -l info -Q celery,priority.high .. seealso:: diff --git a/docs/userguide/concurrency/eventlet.rst b/docs/userguide/concurrency/eventlet.rst index d1545b211..aec95fd33 100644 --- a/docs/userguide/concurrency/eventlet.rst +++ b/docs/userguide/concurrency/eventlet.rst @@ -44,7 +44,7 @@ You can enable the Eventlet pool by using the ``-P`` option to .. code-block:: bash - $ celery worker -P eventlet -c 1000 + $ celery -A proj worker -P eventlet -c 1000 .. _eventlet-examples: diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index e5cffe837..6b85ae328 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -58,32 +58,33 @@ Commands .. code-block:: bash - $ celery status + $ celery -A proj status * **result**: Show the result of a task .. code-block:: bash - $ celery result -t tasks.add 4e196aa4-0141-4601-8138-7aa33db0f577 + $ celery -A proj result -t tasks.add 4e196aa4-0141-4601-8138-7aa33db0f577 Note that you can omit the name of the task as long as the task doesn't use a custom result backend. * **purge**: Purge messages from all configured task queues. - .. code-block:: bash - - $ celery purge - .. warning:: There is no undo for this operation, and messages will be permanently deleted! + .. code-block:: bash + + $ celery -A proj purge + + * **inspect active**: List active tasks .. code-block:: bash - $ celery inspect active + $ celery -A proj inspect active These are all the tasks that are currently being executed. @@ -91,7 +92,7 @@ Commands .. code-block:: bash - $ celery inspect scheduled + $ celery -A proj inspect scheduled These are tasks reserved by the worker because they have the `eta` or `countdown` argument set. @@ -100,7 +101,7 @@ Commands .. code-block:: bash - $ celery inspect reserved + $ celery -A proj inspect reserved This will list all tasks that have been prefetched by the worker, and is currently waiting to be executed (does not include tasks @@ -110,37 +111,37 @@ Commands .. code-block:: bash - $ celery inspect revoked + $ celery -A proj inspect revoked * **inspect registered**: List registered tasks .. code-block:: bash - $ celery inspect registered + $ celery -A proj inspect registered * **inspect stats**: Show worker statistics (see :ref:`worker-statistics`) .. code-block:: bash - $ celery inspect stats + $ celery -A proj inspect stats * **control enable_events**: Enable events .. code-block:: bash - $ celery control enable_events + $ celery -A proj control enable_events * **control disable_events**: Disable events .. code-block:: bash - $ celery control disable_events + $ celery -A proj control disable_events * **migrate**: Migrate tasks from one broker to another (**EXPERIMENTAL**). .. code-block:: bash - $ celery migrate redis://localhost amqp://localhost + $ celery -A proj migrate redis://localhost amqp://localhost This command will migrate all the tasks on one broker to another. As this command is new and experimental you should be sure to have @@ -164,9 +165,9 @@ You can specify a single, or a list of workers by using the .. code-block:: bash - $ celery inspect -d w1,w2 reserved + $ celery -A proj inspect -d w1,w2 reserved - $ celery control -d w1,w2 enable_events + $ celery -A proj control -d w1,w2 enable_events .. _monitoring-flower: @@ -232,13 +233,13 @@ Running the flower command will start a web-server that you can visit: .. code-block:: bash - $ celery flower + $ celery -A proj flower The default port is http://localhost:5555, but you can change this using the `--port` argument: .. code-block:: bash - $ celery flower --port=5555 + $ celery -A proj flower --port=5555 Broker URL can also be passed through the `--broker` argument : @@ -273,7 +274,7 @@ Starting: .. code-block:: bash - $ celery events + $ celery -A proj events You should see a screen like: @@ -285,13 +286,13 @@ You should see a screen like: .. code-block:: bash - $ celery events --camera= --frequency=1.0 + $ celery -A proj events --camera= --frequency=1.0 and it includes a tool to dump events to :file:`stdout`: .. code-block:: bash - $ celery events --dump + $ celery -A proj events --dump For a complete list of options use ``--help``: @@ -457,7 +458,7 @@ arguments: .. code-block:: bash - $ celery events -c myapp.Camera --frequency=2.0 + $ celery -A proj events -c myapp.Camera --frequency=2.0 .. _monitoring-camera: @@ -497,7 +498,7 @@ it with the :option:`-c` option: .. code-block:: bash - $ celery events -c myapp.DumpCam --frequency=2.0 + $ celery -A proj events -c myapp.DumpCam --frequency=2.0 Or you can use it programmatically like this: diff --git a/docs/userguide/periodic-tasks.rst b/docs/userguide/periodic-tasks.rst index 92d065b6d..a81c45f1a 100644 --- a/docs/userguide/periodic-tasks.rst +++ b/docs/userguide/periodic-tasks.rst @@ -247,7 +247,7 @@ To start the :program:`celery beat` service: .. code-block:: bash - $ celery beat + $ celery -A proj beat You can also start embed `beat` inside the worker by enabling workers `-B` option, this is convenient if you will never run @@ -256,7 +256,7 @@ reason is not recommended for production use: .. code-block:: bash - $ celery worker -B + $ celery -A proj worker -B Beat needs to store the last run times of the tasks in a local database file (named `celerybeat-schedule` by default), so it needs access to @@ -265,7 +265,7 @@ location for this file: .. code-block:: bash - $ celery beat -s /home/celery/var/run/celerybeat-schedule + $ celery -A proj beat -s /home/celery/var/run/celerybeat-schedule .. note:: diff --git a/docs/userguide/routing.rst b/docs/userguide/routing.rst index f0a27a5d4..9ea1e503c 100644 --- a/docs/userguide/routing.rst +++ b/docs/userguide/routing.rst @@ -45,14 +45,14 @@ Now you can start server `z` to only process the feeds queue like this: .. code-block:: bash - user@z:/$ celery worker -Q feeds + user@z:/$ celery -A proj worker -Q feeds You can specify as many queues as you want, so you can make this server process the default queue as well: .. code-block:: bash - user@z:/$ celery worker -Q feeds,celery + user@z:/$ celery -A proj worker -Q feeds,celery .. _routing-changing-default-queue: @@ -147,21 +147,21 @@ start it with the ``-Q`` option: .. code-block:: bash - user@z:/$ celery worker -Q feed_tasks --hostname=z@%h + user@z:/$ celery -A proj worker -Q feed_tasks --hostname=z@%h Servers `x` and `y` must be configured to consume from the default queue: .. code-block:: bash - user@x:/$ celery worker -Q default --hostname=x@%h - user@y:/$ celery worker -Q default --hostname=y@%h + user@x:/$ celery -A proj worker -Q default --hostname=x@%h + user@y:/$ celery -A proj worker -Q default --hostname=y@%h If you want, you can even have your feed processing worker handle regular tasks as well, maybe in times when there's a lot of work to do: .. code-block:: python - user@z:/$ celery worker -Q feed_tasks,default --hostname=z@%h + user@z:/$ celery -A proj worker -Q feed_tasks,default --hostname=z@%h If you have another queue but on another exchange you want to add, just specify a custom exchange and exchange type: @@ -367,7 +367,7 @@ or just start with no arguments to start it in shell-mode: .. code-block:: bash - $ celery amqp + $ celery -A proj amqp -> connecting to amqp://guest@localhost:5672/. -> connected. 1> @@ -381,7 +381,7 @@ Let's create a queue you can send messages to: .. code-block:: bash - $ celery amqp + $ celery -A proj amqp 1> exchange.declare testexchange direct ok. 2> queue.declare testqueue diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 24df1cdb9..28362ff16 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -1166,8 +1166,8 @@ yourself: .. code-block:: python - >>> from celery import current_app - >>> current_app.tasks + >>> from proj.celery import app + >>> app.tasks {'celery.chord_unlock': <@task: celery.chord_unlock>, 'celery.backend_cleanup': diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index 4bfa93b7a..cbfe81fb9 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -23,7 +23,7 @@ You can start the worker in the foreground by executing the command: .. code-block:: bash - $ celery --app=app worker -l info + $ celery -A proj worker -l info For a full list of available command-line options see :mod:`~celery.bin.worker`, or simply do: @@ -38,9 +38,9 @@ host name with the :option:`--hostname|-n` argument: .. code-block:: bash - $ celery worker --loglevel=INFO --concurrency=10 -n worker1.%h - $ celery worker --loglevel=INFO --concurrency=10 -n worker2.%h - $ celery worker --loglevel=INFO --concurrency=10 -n worker3.%h + $ celery -A proj worker --loglevel=INFO --concurrency=10 -n worker1.%h + $ celery -A proj worker --loglevel=INFO --concurrency=10 -n worker2.%h + $ celery -A proj worker --loglevel=INFO --concurrency=10 -n worker3.%h The hostname argument can expand the following variables: @@ -560,7 +560,7 @@ by giving a comma separated list of queues to the :option:`-Q` option: .. code-block:: bash - $ celery worker -l info -Q foo,bar,baz + $ celery -A proj worker -l info -Q foo,bar,baz If the queue name is defined in :setting:`CELERY_QUEUES` it will use that configuration, but if it's not defined in the list of queues Celery will @@ -663,7 +663,7 @@ the :control:`active_queues` control command: .. code-block:: bash - $ celery inspect active_queues + $ celery -A proj inspect active_queues [...] Like all other remote control commands this also supports the @@ -672,7 +672,7 @@ reply to the request: .. code-block:: bash - $ celery inspect active_queues -d worker1.local + $ celery -A proj inspect active_queues -d worker1.local [...] From d618ee2681f603f4efc31e7542019f50e672a1e0 Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Fri, 6 Jun 2014 18:36:14 +0700 Subject: [PATCH 0211/1103] Added ability to change task name generation behavior --- celery/app/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/celery/app/base.py b/celery/app/base.py index e99affd36..b00db07e7 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -268,7 +268,7 @@ def _create_task_cls(fun): def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): if not self.finalized and not self.autofinalize: raise RuntimeError('Contract breach: app not finalized') - name = name or gen_task_name(self, fun.__name__, fun.__module__) + name = name or self.gen_task_name(self, fun.__name__, fun.__module__) base = base or self.Task if name not in self._tasks: @@ -286,6 +286,9 @@ def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): task = self._tasks[name] return task + def gen_task_name(self, name, module): + return gen_task_name(self, name, module) + def finalize(self, auto=False): with self._finalize_mutex: if not self.finalized: From 131f60130d671e97c4c817676e167981dc6a5bbd Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Fri, 6 Jun 2014 22:28:13 +0700 Subject: [PATCH 0212/1103] Removed unnecessary self --- celery/app/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/base.py b/celery/app/base.py index b00db07e7..45546af70 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -268,7 +268,7 @@ def _create_task_cls(fun): def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): if not self.finalized and not self.autofinalize: raise RuntimeError('Contract breach: app not finalized') - name = name or self.gen_task_name(self, fun.__name__, fun.__module__) + name = name or self.gen_task_name(fun.__name__, fun.__module__) base = base or self.Task if name not in self._tasks: From ca3d280e3f5203fd6f636a96dc6ae19483920869 Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Sat, 7 Jun 2014 16:35:56 +0700 Subject: [PATCH 0213/1103] Use app.gen_task_name everywhere & updated docs --- celery/app/__init__.py | 3 +-- celery/task/base.py | 3 +-- docs/userguide/tasks.rst | 49 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/celery/app/__init__.py b/celery/app/__init__.py index 22a9700ae..3053a59b9 100644 --- a/celery/app/__init__.py +++ b/celery/app/__init__.py @@ -17,7 +17,6 @@ get_current_task as current_task, connect_on_app_finalize, set_default_app, _get_active_apps, _task_stack, ) -from celery.utils import gen_task_name from .base import Celery, AppPickler @@ -140,7 +139,7 @@ def __inner(fun): def task_by_cons(): app = current_app() return app.tasks[ - name or gen_task_name(app, fun.__name__, fun.__module__) + name or app.gen_task_name(fun.__name__, fun.__module__) ] return Proxy(task_by_cons) return __inner diff --git a/celery/task/base.py b/celery/task/base.py index 8412b9517..35f8877ad 100644 --- a/celery/task/base.py +++ b/celery/task/base.py @@ -18,7 +18,6 @@ from celery.five import class_property, reclassmethod, with_metaclass from celery.local import Proxy from celery.schedules import maybe_schedule -from celery.utils import gen_task_name from celery.utils.log import get_task_logger __all__ = ['Context', 'Task', 'TaskType', 'PeriodicTask', 'task'] @@ -86,7 +85,7 @@ def __new__(cls, name, bases, attrs): # - Automatically generate missing/empty name. task_name = attrs.get('name') if not task_name: - attrs['name'] = task_name = gen_task_name(app, name, task_module) + attrs['name'] = task_name = app.gen_task_name(name, task_module) if not attrs.get('_decorated'): # non decorated tasks must also be shared in case diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 24df1cdb9..1b4594e6f 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -215,6 +215,55 @@ on the automatic naming: def add(x, y): return x + y +.. _task-name-generator-info: + +Changing the automatic naming behavior +-------------------------------------- + +.. versionadded:: 3.2 + +There are some cases when the default automatic naming is not suitable. +Consider you have many tasks within many different modules:: + + project/ + /__init__.py + /celery.py + /moduleA/ + /__init__.py + /tasks.py + /moduleB/ + /__init__.py + /tasks.py + +Using the default automatic naming, each task will have a generated name +like `moduleA.tasks.taskA`, `moduleA.tasks.taskB`, `moduleB.tasks.test` +and so on. You may want to get rid of having `tasks` in all task names. +As pointed above, you can explicitly give names for all tasks, or you +can change the automatic naming behavior by overriding +:meth:`~@Celery.gen_task_name`. Continuing with the example, `celery.py` +may contain: + +.. code-block:: python + + from celery import Celery + + class MyCelery(Celery): + + def gen_task_name(self, name, module): + if module.endswith('.tasks'): + module = module[:-6] + return super(MyCelery, self).gen_task_name(name, module) + + app = MyCelery('main') + +So each task will have a name like `moduleA.taskA`, `moduleA.taskB` and +`moduleB.test`. + +.. warning:: + + Make sure that your `gen_task_name` is a pure function, which means + that for the same input it must always return the same output. + .. _task-request-info: Context From ad86f405164afbfdc363bb5b824791161f3ec778 Mon Sep 17 00:00:00 2001 From: Mher Movsisyan Date: Mon, 9 Jun 2014 00:05:31 +0400 Subject: [PATCH 0214/1103] Update README.rst Updates readme --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 1e9b64ffb..10b6d8684 100644 --- a/README.rst +++ b/README.rst @@ -44,7 +44,7 @@ What do I need? Celery version 3.0 runs on, -- Python (2.5, 2.6, 2.7, 3.2, 3.3) +- Python (2.6, 2.7, 3.3, 3.4) - PyPy (1.8, 1.9) - Jython (2.5, 2.7). From 52b50e8cf453ba64838fc1398b23ffb0ef674cbe Mon Sep 17 00:00:00 2001 From: illes Date: Fri, 6 Jun 2014 19:01:38 +0200 Subject: [PATCH 0215/1103] ignore unknown events in consumer Custom event types (not starting with either 'task-' or 'worker-') break Consumer, as it tries to split a `None` returned by state.event() --- celery/worker/consumer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 2dfc5b399..6ac929e8c 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -827,7 +827,7 @@ def on_message(self, prepare, message): message.payload['hostname']) if hostname != self.hostname: type, event = prepare(message.payload) - obj, subject = self.update_state(event) + self.update_state(event) else: self.clock.forward() From 372de9cb05507965d5bc796866e9559369a4a744 Mon Sep 17 00:00:00 2001 From: Polina Giralt Date: Wed, 4 Jun 2014 15:11:54 -0400 Subject: [PATCH 0216/1103] Update first-steps-with-celery.rst fixed grammar, missing word "is" in "this how" --- docs/getting-started/first-steps-with-celery.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/first-steps-with-celery.rst b/docs/getting-started/first-steps-with-celery.rst index b3ea28706..86ddf38bb 100644 --- a/docs/getting-started/first-steps-with-celery.rst +++ b/docs/getting-started/first-steps-with-celery.rst @@ -350,7 +350,7 @@ contain any syntax errors, you can try to import it: For a complete reference of configuration options, see :ref:`configuration`. -To demonstrate the power of configuration files, this how you would +To demonstrate the power of configuration files, this is how you would route a misbehaving task to a dedicated queue: :file:`celeryconfig.py`: From 2473f5d503d0829b878ba3828d05478d95b128a3 Mon Sep 17 00:00:00 2001 From: Mher Movsisyan Date: Tue, 10 Jun 2014 15:17:43 +0400 Subject: [PATCH 0217/1103] Fixes cpython version requirements --- docs/includes/introduction.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 4cbc2627f..e178f0422 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -38,7 +38,7 @@ What do I need? Celery version 3.0 runs on, -- Python (2.5, 2.6, 2.7, 3.2, 3.3) +- Python (2.6, 2.7, 3.3, 3.4) - PyPy (1.8, 1.9) - Jython (2.5, 2.7). From 2a5ef7c55b10fb8c26d54fef64cceb6d0ddfbe05 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 10 Jun 2014 15:43:58 +0100 Subject: [PATCH 0218/1103] Tests passing --- celery/tests/events/test_events.py | 2 -- celery/tests/worker/test_control.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/celery/tests/events/test_events.py b/celery/tests/events/test_events.py index 791f4167e..0c78a4f4d 100644 --- a/celery/tests/events/test_events.py +++ b/celery/tests/events/test_events.py @@ -91,8 +91,6 @@ def test_send(self): for ev in evs: self.assertTrue(producer.has_event(ev)) - buf = eventer._outbound_buffer = Mock() - buf.popleft.side_effect = IndexError() eventer.flush() def test_enter_exit(self): diff --git a/celery/tests/worker/test_control.py b/celery/tests/worker/test_control.py index 829bd9c9d..4aa7531a0 100644 --- a/celery/tests/worker/test_control.py +++ b/celery/tests/worker/test_control.py @@ -119,7 +119,7 @@ class test_ControlPanel(AppCase): def setup(self): self.panel = self.create_panel(consumer=Consumer(self.app)) - @self.app.task(rate_limit=200, shared=False) + @self.app.task(name='c.unittest.mytask', rate_limit=200, shared=False) def mytask(): pass self.mytask = mytask From f05a95726eef5c798b6404d7bbc6753123886318 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 13 Jun 2014 13:49:09 +0100 Subject: [PATCH 0219/1103] Task: Do not requeue if retry fails. --- celery/app/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/task.py b/celery/app/task.py index 6bc1f2b54..32b16787f 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -580,7 +580,7 @@ def retry(self, args=None, kwargs=None, exc=None, throw=True, except Exception as exc: if is_eager: raise - raise Reject(exc, requeue=True) + raise Reject(exc, requeue=False) ret = Retry(exc=exc, when=eta or countdown) if throw: raise ret From ca4c9b00e0f98f89aab9243d4ab0d96bc8cbd277 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 13 Jun 2014 15:17:03 +0100 Subject: [PATCH 0220/1103] Gossip: Now sets x-message-ttl for event queue to heartbeat_interval s. (Issue #2005) --- celery/events/__init__.py | 28 ++++++++++++++++++---------- celery/worker/consumer.py | 7 +++++-- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/celery/events/__init__.py b/celery/events/__init__.py index c5cc03df1..d21df35a8 100644 --- a/celery/events/__init__.py +++ b/celery/events/__init__.py @@ -297,7 +297,7 @@ class EventReceiver(ConsumerMixin): def __init__(self, channel, handlers=None, routing_key='#', node_id=None, app=None, queue_prefix='celeryev', - accept=None): + accept=None, queue_ttl=None, queue_expires=None): self.app = app_or_default(app or self.app) self.channel = maybe_channel(channel) self.handlers = {} if handlers is None else handlers @@ -305,12 +305,15 @@ def __init__(self, channel, handlers=None, routing_key='#', self.node_id = node_id or uuid() self.queue_prefix = queue_prefix self.exchange = get_exchange(self.connection or self.app.connection()) - self.queue = Queue('.'.join([self.queue_prefix, self.node_id]), - exchange=self.exchange, - routing_key=self.routing_key, - auto_delete=True, - durable=False, - queue_arguments=self._get_queue_arguments()) + self.queue = Queue( + '.'.join([self.queue_prefix, self.node_id]), + exchange=self.exchange, + routing_key=self.routing_key, + auto_delete=True, durable=False, + queue_arguments=self._get_queue_arguments( + ttl=queue_ttl, expires=queue_expires, + ), + ) self.clock = self.app.clock self.adjust_clock = self.clock.adjust self.forward_clock = self.clock.forward @@ -318,11 +321,16 @@ def __init__(self, channel, handlers=None, routing_key='#', accept = {self.app.conf.CELERY_EVENT_SERIALIZER, 'json'} self.accept = accept - def _get_queue_arguments(self): + def _get_queue_arguments(self, ttl=None, expires=None): conf = self.app.conf return dictfilter({ - 'x-message-ttl': maybe_s_to_ms(conf.CELERY_EVENT_QUEUE_TTL), - 'x-expires': maybe_s_to_ms(conf.CELERY_EVENT_QUEUE_EXPIRES), + 'x-message-ttl': maybe_s_to_ms( + ttl if ttl is not None else conf.CELERY_EVENT_QUEUE_TTL, + ), + 'x-expires': maybe_s_to_ms( + expires if expires is not None + else conf.CELERY_EVENT_QUEUE_EXPIRES, + ), }) def process(self, type, event): diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 6ac929e8c..8afa85b5b 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -685,7 +685,8 @@ class Gossip(bootsteps.ConsumerStep): ) compatible_transports = {'amqp', 'redis'} - def __init__(self, c, without_gossip=False, interval=5.0, **kwargs): + def __init__(self, c, without_gossip=False, + interval=5.0, heartbeat_interval=2.0, **kwargs): self.enabled = not without_gossip and self.compatible_transport(c.app) self.app = c.app c.gossip = self @@ -704,6 +705,7 @@ def __init__(self, c, without_gossip=False, interval=5.0, **kwargs): c._mutex = DummyLock() self.update_state = self.state.event self.interval = interval + self.heartbeat_interval = heartbeat_interval self._tref = None self.consensus_requests = defaultdict(list) self.consensus_replies = {} @@ -802,7 +804,8 @@ def periodic(self): def get_consumers(self, channel): self.register_timer() - ev = self.Receiver(channel, routing_key='worker.#') + ev = self.Receiver(channel, routing_key='worker.#', + queue_ttl=self.heartbeat_interval) return [kombu.Consumer( channel, queues=[ev.queue], From ee46d0b78d8ffc068d5b80e9568a5a050c61d1a8 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 13 Jun 2014 16:34:39 +0100 Subject: [PATCH 0221/1103] Removes outdated assert (Issue #2086) --- celery/concurrency/asynpool.py | 1 - 1 file changed, 1 deletion(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index d86cfabba..4a024d508 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -186,7 +186,6 @@ def _recv_message(self, add_reader, fd, callback, else: buf = bufv = BytesIO() # header - assert not isblocking(fd) while Hr < 4: try: From d5029501ab762c4fb6b7baab7af4ecf96b106fbe Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 26 Jun 2014 11:44:33 +0100 Subject: [PATCH 0222/1103] Fixes problem with celery module type. Closes #2109 --- celery/five.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/five.py b/celery/five.py index 4418a4ac3..bfa42caf6 100644 --- a/celery/five.py +++ b/celery/five.py @@ -174,7 +174,7 @@ def create_module(name, attrs, cls_attrs=None, pkg=None, attr_name: (prepare_attr(attr) if prepare_attr else attr) for attr_name, attr in items(attrs) } - module = sys.modules[fqdn] = type(modname, (base, ), cls_attrs)(fqdn) + module = sys.modules[fqdn] = type(modname, (base, ), cls_attrs)(name) module.__dict__.update(attrs) return module From 2c541284b2a6a3c2ce42dec95f1cfdfdafadc2c9 Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Fri, 27 Jun 2014 13:07:18 +0700 Subject: [PATCH 0223/1103] Move base64 to utils.serialization --- celery/security/serialization.py | 11 +---------- celery/utils/serialization.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/celery/security/serialization.py b/celery/security/serialization.py index f1cab2914..278bfb9e9 100644 --- a/celery/security/serialization.py +++ b/celery/security/serialization.py @@ -8,26 +8,17 @@ """ from __future__ import absolute_import -import base64 - from kombu.serialization import registry, dumps, loads from kombu.utils.encoding import bytes_to_str, str_to_bytes, ensure_bytes from .certificate import Certificate, FSCertStore from .key import PrivateKey from .utils import reraise_errors +from celery.utils.serialization import b64encode, b64decode __all__ = ['SecureSerializer', 'register_auth'] -def b64encode(s): - return bytes_to_str(base64.b64encode(str_to_bytes(s))) - - -def b64decode(s): - return base64.b64decode(str_to_bytes(s)) - - class SecureSerializer(object): def __init__(self, key=None, cert=None, cert_store=None, diff --git a/celery/utils/serialization.py b/celery/utils/serialization.py index d5509f1c2..9861dd6cf 100644 --- a/celery/utils/serialization.py +++ b/celery/utils/serialization.py @@ -8,6 +8,7 @@ """ from __future__ import absolute_import +from base64 import b64encode as base64encode, b64decode as base64decode from inspect import getmro from itertools import takewhile @@ -16,6 +17,8 @@ except ImportError: import pickle # noqa +from kombu.utils.encoding import bytes_to_str, str_to_bytes + from .encoding import safe_repr __all__ = ['UnpickleableExceptionWrapper', 'subclass_exception', @@ -165,3 +168,11 @@ def get_pickled_exception(exc): if isinstance(exc, UnpickleableExceptionWrapper): return exc.restore() return exc + + +def b64encode(s): + return bytes_to_str(base64encode(str_to_bytes(s))) + + +def b64decode(s): + return base64decode(str_to_bytes(s)) From 20c5933bde5e072e750e9a8789e3b2f111aa8230 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 27 Jun 2014 13:40:35 +0100 Subject: [PATCH 0224/1103] Update FreeBSD faq. Closes #2104 --- docs/faq.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 54e9df8d1..bf7cb9480 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -275,9 +275,15 @@ most systems), it usually contains a message describing the reason. Does it work on FreeBSD? ------------------------ -**Answer:** The prefork pool requires a working POSIX semaphore -implementation which isn't enabled in FreeBSD by default. You have to enable -POSIX semaphores in the kernel and manually recompile multiprocessing. +**Answer:** Depends + +When using the RabbitMQ (AMQP) and Redis transports it should work +out of the box. + +For other transports the compatibility prefork pool is +used which requires a working POSIX semaphore implementation, and this isn't +enabled in FreeBSD by default. You have to enable +POSIX semaphores in the kernel and manually recompile billiard. Luckily, Viktor Petersson has written a tutorial to get you started with Celery on FreeBSD here: From 20340d79b55137643d5ac0df063614075385daaa Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 27 Jun 2014 13:43:50 +0100 Subject: [PATCH 0225/1103] Beat: Wake up 0.010 seconds earlier to adjust lateness --- celery/beat.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/celery/beat.py b/celery/beat.py index 372441221..72f7c3ac1 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -212,7 +212,7 @@ def apply_entry(self, entry, producer=None): def is_due(self, entry): return entry.is_due() - def tick(self, event_t=event_t, min=min, + def tick(self, drift=-0.010, event_t=event_t, min=min, heappop=heapq.heappop, heappush=heapq.heappush, heapify=heapq.heapify): """Run a tick, that is one iteration of the scheduler. @@ -223,7 +223,7 @@ def tick(self, event_t=event_t, min=min, max_interval = self.max_interval H = self._heap if H is None: - H = self._heap = [event_t(e.is_due()[1] or 0, 5, e) + H = self._heap = [event_t(e.is_due()[1] + drift or 0, 5, e) for e in values(self.schedule)] heapify(H) event = H[0] @@ -239,7 +239,8 @@ def tick(self, event_t=event_t, min=min, else: heappush(H, verify) return min(verify[0], max_interval) - return min(next_time_to_run or max_interval, max_interval) + return min(next_time_to_run + drift if next_time_to_run + else max_interval, max_interval) def should_sync(self): return ( @@ -476,7 +477,7 @@ def start(self, embedded_process=False): try: while not self._is_shutdown.is_set(): interval = self.scheduler.tick() - if interval: + if interval and interval > 0.0: debug('beat: Waking up %s.', humanize_seconds(interval, prefix='in ')) time.sleep(interval) From 75246714dd11e6c463b9dc67f4311690643bff24 Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Fri, 27 Jun 2014 19:48:18 +0700 Subject: [PATCH 0226/1103] Added autoretry decorator --- celery/contrib/autoretry.py | 52 +++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 celery/contrib/autoretry.py diff --git a/celery/contrib/autoretry.py b/celery/contrib/autoretry.py new file mode 100644 index 000000000..19254d2a1 --- /dev/null +++ b/celery/contrib/autoretry.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +""" +celery.contrib.autoretry +======================== + +.. versionadded:: 3.2 + +Decorator that enables autoretrying when one of specified exceptions +are raised in a task body. + + +Examples +-------- + +.. code-block:: python + + from celery.contrib.autoretry import autoretry + + @autoretry(on=(ZeroDivisionError,)) + @app.task + def div + +.. note:: + + `autoretry` decorator must be applied **before** `app.task` decorator. +""" + +from __future__ import absolute_import + +from functools import wraps + + +def autoretry(on=None, retry_kwargs=None): + + def decorator(task): + if not on: + return task.run + + autoretry_exceptions = tuple(on) # except only works with tuples + _retry_kwargs = retry_kwargs or {} + + @wraps(task.run) + def inner(*args, **kwargs): + try: + return task._orig_run(*args, **kwargs) + except autoretry_exceptions as exc: + raise task.retry(exc=exc, **_retry_kwargs) + + task._orig_run = task.run + task.run = inner + return inner + return decorator From c73bb64f13a9dd571c7939af97af6bb7783c7e69 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 27 Jun 2014 13:51:00 +0100 Subject: [PATCH 0227/1103] Tests passing --- celery/beat.py | 13 +++++++++---- celery/tests/app/test_beat.py | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/celery/beat.py b/celery/beat.py index 72f7c3ac1..9b8c4653c 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -209,10 +209,15 @@ def apply_entry(self, entry, producer=None): else: debug('%s sent. id->%s', entry.task, result.id) + def adjust(self, n, drift=-0.010): + if n and n > 0: + return n + drift + return n + def is_due(self, entry): return entry.is_due() - def tick(self, drift=-0.010, event_t=event_t, min=min, + def tick(self, event_t=event_t, min=min, heappop=heapq.heappop, heappush=heapq.heappush, heapify=heapq.heapify): """Run a tick, that is one iteration of the scheduler. @@ -220,10 +225,11 @@ def tick(self, drift=-0.010, event_t=event_t, min=min, Executes all due tasks. """ + adjust = self.adjust max_interval = self.max_interval H = self._heap if H is None: - H = self._heap = [event_t(e.is_due()[1] + drift or 0, 5, e) + H = self._heap = [event_t(adjust(e.is_due()[1]) or 0, 5, e) for e in values(self.schedule)] heapify(H) event = H[0] @@ -239,8 +245,7 @@ def tick(self, drift=-0.010, event_t=event_t, min=min, else: heappush(H, verify) return min(verify[0], max_interval) - return min(next_time_to_run + drift if next_time_to_run - else max_interval, max_interval) + return min(adjust(next_time_to_run) or max_interval, max_interval) def should_sync(self): return ( diff --git a/celery/tests/app/test_beat.py b/celery/tests/app/test_beat.py index 04a610df0..362fbf9b4 100644 --- a/celery/tests/app/test_beat.py +++ b/celery/tests/app/test_beat.py @@ -287,7 +287,7 @@ def test_pending_tick(self): scheduler = mScheduler(app=self.app) scheduler.add(name='test_pending_tick', schedule=always_pending) - self.assertEqual(scheduler.tick(), 1) + self.assertEqual(scheduler.tick(), 1 - 0.010) def test_honors_max_interval(self): scheduler = mScheduler(app=self.app) @@ -303,7 +303,7 @@ def test_ticks(self): {'schedule': mocked_schedule(False, j)}) for i, j in enumerate(nums)) scheduler.update_from_dict(s) - self.assertEqual(scheduler.tick(), min(nums)) + self.assertEqual(scheduler.tick(), min(nums) - 0.010) def test_schedule_no_remain(self): scheduler = mScheduler(app=self.app) From 7a11dc6f2ee8d26e709082608f74d45a1045083b Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Sat, 28 Jun 2014 01:38:45 +0700 Subject: [PATCH 0228/1103] Added ability to specify autoretry with app.task decorator --- celery/app/base.py | 7 +++++++ celery/contrib/autoretry.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/celery/app/base.py b/celery/app/base.py index 45546af70..da71617c6 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -38,6 +38,7 @@ from celery.utils.functional import first, maybe_list from celery.utils.imports import instantiate, symbol_by_name from celery.utils.objects import FallbackContext, mro_lookup +from celery.contrib.autoretry import autoretry from .annotations import prepare as prepare_annotations from .defaults import DEFAULTS, find_deprecated_settings @@ -282,6 +283,12 @@ def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): '__wrapped__': fun}, **options))() self._tasks[task.name] = task task.bind(self) # connects task to this app + + autoretry_on = options.get('autoretry_on') + retry_kwargs = options.get('retry_kwargs') + + if autoretry_on: + task = autoretry(autoretry_on, retry_kwargs)(task) else: task = self._tasks[name] return task diff --git a/celery/contrib/autoretry.py b/celery/contrib/autoretry.py index 19254d2a1..467aa7854 100644 --- a/celery/contrib/autoretry.py +++ b/celery/contrib/autoretry.py @@ -33,7 +33,7 @@ def div def autoretry(on=None, retry_kwargs=None): def decorator(task): - if not on: + if not on or hasattr(task, '_orig_run'): return task.run autoretry_exceptions = tuple(on) # except only works with tuples From 852f8380e33517475d5772c771bfc9eeb9d11c30 Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Sat, 28 Jun 2014 01:39:19 +0700 Subject: [PATCH 0229/1103] Added test cases for autoretry --- celery/tests/contrib/test_autoretry.py | 39 ++++++++++++++++++++++++++ celery/tests/tasks/test_tasks.py | 25 +++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 celery/tests/contrib/test_autoretry.py diff --git a/celery/tests/contrib/test_autoretry.py b/celery/tests/contrib/test_autoretry.py new file mode 100644 index 000000000..40e1b3938 --- /dev/null +++ b/celery/tests/contrib/test_autoretry.py @@ -0,0 +1,39 @@ +from __future__ import absolute_import + +from celery.contrib.autoretry import autoretry + +from celery.tests.case import AppCase + + +class TasksCase(AppCase): + + def setup(self): + + @autoretry(on=(ZeroDivisionError,)) + @self.app.task(shared=False) + def autoretry_task_no_kwargs(a, b): + self.iterations += 1 + return a/b + self.autoretry_task_no_kwargs = autoretry_task_no_kwargs + + @autoretry(on=(ZeroDivisionError,), retry_kwargs={'max_retries': 5}) + @self.app.task(shared=False) + def autoretry_task(a, b): + self.iterations += 1 + return a/b + self.autoretry_task = autoretry_task + + +class test_autoretry(TasksCase): + + def test_autoretry_no_kwargs(self): + self.autoretry_task_no_kwargs.max_retries = 3 + self.autoretry_task_no_kwargs.iterations = 0 + self.autoretry_task_no_kwargs.apply((1, 0)) + self.assertEqual(self.autoretry_task_no_kwargs.iterations, 4) + + def test_autoretry(self): + self.autoretry_task_no_kwargs.max_retries = 3 + self.autoretry_task_no_kwargs.iterations = 0 + self.autoretry_task_no_kwargs.apply((1, 0)) + self.assertEqual(self.autoretry_task_no_kwargs.iterations, 6) diff --git a/celery/tests/tasks/test_tasks.py b/celery/tests/tasks/test_tasks.py index 8d9da1f46..145495f1f 100644 --- a/celery/tests/tasks/test_tasks.py +++ b/celery/tests/tasks/test_tasks.py @@ -100,6 +100,19 @@ def retry_task_customexc(self, arg1, arg2, kwarg=1, **kwargs): raise self.retry(countdown=0, exc=exc) self.retry_task_customexc = retry_task_customexc + @self.app.task(on=(ZeroDivisionError,), shared=False) + def autoretry_task_no_kwargs(a, b): + self.iterations += 1 + return a/b + self.autoretry_task_no_kwargs + + @self.app.task(on=(ZeroDivisionError,), + retry_kwargs={'max_retries': 5}, shared=False) + def autoretry_task(a, b): + self.iterations += 1 + return a/b + self.autoretry_task + class MyCustomException(Exception): """Random custom exception.""" @@ -193,6 +206,18 @@ def test_max_retries_exceeded(self): result.get() self.assertEqual(self.retry_task.iterations, 2) + def test_autoretry_no_kwargs(self): + self.autoretry_task_no_kwargs.max_retries = 3 + self.autoretry_task_no_kwargs.iterations = 0 + self.autoretry_task_no_kwargs.apply((1, 0)) + self.assertEqual(self.autoretry_task_no_kwargs.iterations, 4) + + def test_autoretry(self): + self.autoretry_task_no_kwargs.max_retries = 3 + self.autoretry_task_no_kwargs.iterations = 0 + self.autoretry_task_no_kwargs.apply((1, 0)) + self.assertEqual(self.autoretry_task_no_kwargs.iterations, 6) + class test_canvas_utils(TasksCase): From a129efa7a2dd3403be68ed44cff2dfaf8a7a4d2f Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Sat, 28 Jun 2014 02:17:50 +0700 Subject: [PATCH 0230/1103] Fixed typo in tests & updated docs --- celery/tests/contrib/test_autoretry.py | 8 ++--- celery/tests/tasks/test_tasks.py | 8 ++--- docs/userguide/tasks.rst | 49 ++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 8 deletions(-) diff --git a/celery/tests/contrib/test_autoretry.py b/celery/tests/contrib/test_autoretry.py index 40e1b3938..09d5e7fe8 100644 --- a/celery/tests/contrib/test_autoretry.py +++ b/celery/tests/contrib/test_autoretry.py @@ -33,7 +33,7 @@ def test_autoretry_no_kwargs(self): self.assertEqual(self.autoretry_task_no_kwargs.iterations, 4) def test_autoretry(self): - self.autoretry_task_no_kwargs.max_retries = 3 - self.autoretry_task_no_kwargs.iterations = 0 - self.autoretry_task_no_kwargs.apply((1, 0)) - self.assertEqual(self.autoretry_task_no_kwargs.iterations, 6) + self.autoretry_tasks.max_retries = 3 + self.autoretry_task.iterations = 0 + self.autoretry_task.apply((1, 0)) + self.assertEqual(self.autoretry_task.iterations, 6) diff --git a/celery/tests/tasks/test_tasks.py b/celery/tests/tasks/test_tasks.py index 145495f1f..1cf9e868e 100644 --- a/celery/tests/tasks/test_tasks.py +++ b/celery/tests/tasks/test_tasks.py @@ -213,10 +213,10 @@ def test_autoretry_no_kwargs(self): self.assertEqual(self.autoretry_task_no_kwargs.iterations, 4) def test_autoretry(self): - self.autoretry_task_no_kwargs.max_retries = 3 - self.autoretry_task_no_kwargs.iterations = 0 - self.autoretry_task_no_kwargs.apply((1, 0)) - self.assertEqual(self.autoretry_task_no_kwargs.iterations, 6) + self.autoretry_task.max_retries = 3 + self.autoretry_task.iterations = 0 + self.autoretry_task.apply((1, 0)) + self.assertEqual(self.autoretry_task.iterations, 6) class test_canvas_utils(TasksCase): diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index d0ce0f291..a8f3ff28c 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -470,6 +470,55 @@ override this default. raise self.retry(exc=exc, countdown=60) # override the default and # retry in 1 minute +Autoretrying +------------ + +.. versionadded:: 3.2 + +Sometimes you may want to retry a task on particular exception. To do so, +you should wrap a task body with `try-except` statement, for example: + +.. code-block:: python + + @app.task + def div(a, b): + try: + return a / b + except ZeroDivisionError as exc: + raise div.retry(exc=exc) + +This may not be acceptable all the time, since you may have a lot of such +tasks. + +Fortunately, you can tell Celery to automatically retry a task using +:func:`autoretry <~celery.contrib.autoretry.autoretry>` decorator: + +.. code-block:: python + + @autoretry(on=(ZeroDivisionError,)) + @app.task + def div(a, b): + return a / b + +Also you can specify autoretry directly in `~@Celery.task` decorator: + +.. code-block:: python + + @app.task(autoretry_on=(ZeroDivisionError,)) + def div(a, b): + return a / b + +If you want to specify custom arguments for internal `~@Task.retry` +call, pass `retry_kwargs` argument to :func:`autoretry +<~celery.contrib.autoretry.autoretry>` or `~@Celery.task` decorators: + +.. code-block:: python + + @app.task(autoretry_on=(ZeroDivisionError,), + retry_kwargs={'max_retries': 5}) + def div(a, b): + return a / b + .. _task-options: List of Options From eb4069554c94e9a09e2b125f4ebbadf5dc678396 Mon Sep 17 00:00:00 2001 From: Tocho Tochev Date: Mon, 30 Jun 2014 13:14:57 +0300 Subject: [PATCH 0231/1103] Docs: Fix typos. --- docs/configuration.rst | 6 +++--- docs/contributing.rst | 2 +- docs/getting-started/introduction.rst | 2 +- docs/getting-started/next-steps.rst | 2 +- docs/glossary.rst | 2 +- docs/userguide/application.rst | 2 +- docs/userguide/calling.rst | 4 ++-- docs/userguide/canvas.rst | 4 ++-- docs/userguide/extending.rst | 2 +- docs/userguide/routing.rst | 2 +- docs/userguide/tasks.rst | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 3d8f2f2d8..6a5e29a45 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -335,7 +335,7 @@ AMQP backend settings .. note:: The AMQP backend requires RabbitMQ 1.1.0 or higher to automatically - expire results. If you are running an older version of RabbitmQ + expire results. If you are running an older version of RabbitMQ you should disable result expiration like this: CELERY_TASK_RESULT_EXPIRES = None @@ -455,7 +455,7 @@ which is the same as:: CELERY_RESULT_BACKEND = 'redis://' -The fields of the URL is defined as folows: +The fields of the URL are defined as follows: - *host* @@ -657,7 +657,7 @@ which is the same as:: CELERY_RESULT_BACKEND = "riak://" -The fields of the URL is defined as folows: +The fields of the URL are defined as follows: - *host* diff --git a/docs/contributing.rst b/docs/contributing.rst index ce8c3efb5..26cc0f04b 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -639,7 +639,7 @@ Edit the file using your favorite editor: $ vim celery.worker.awesome.rst - # change every occurance of ``celery.schedules`` to + # change every occurrence of ``celery.schedules`` to # ``celery.worker.awesome`` diff --git a/docs/getting-started/introduction.rst b/docs/getting-started/introduction.rst index afd31a5e4..874ba109e 100644 --- a/docs/getting-started/introduction.rst +++ b/docs/getting-started/introduction.rst @@ -281,7 +281,7 @@ Quickjump - :ref:`see a list of running workers ` - :ref:`purge all messages ` - :ref:`inspect what the workers are doing ` - - :ref:`see what tasks a worker has registerd ` + - :ref:`see what tasks a worker has registered ` - :ref:`migrate tasks to a new broker ` - :ref:`see a list of event message types ` - :ref:`contribute to Celery ` diff --git a/docs/getting-started/next-steps.rst b/docs/getting-started/next-steps.rst index e90e7688f..b2ea6132d 100644 --- a/docs/getting-started/next-steps.rst +++ b/docs/getting-started/next-steps.rst @@ -92,7 +92,7 @@ When the worker starts you should see a banner and some messages:: [2012-06-08 16:23:51,078: WARNING/MainProcess] celery@halcyon.local has started. --- The *broker* is the URL you specifed in the broker argument in our ``celery`` +-- The *broker* is the URL you specified in the broker argument in our ``celery`` module, you can also specify a different broker on the command-line by using the :option:`-b` option. diff --git a/docs/glossary.rst b/docs/glossary.rst index ecc4561a2..d3158c543 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -51,7 +51,7 @@ Glossary Idempotence is a mathematical property that describes a function that can be called multiple times without changing the result. Practically it means that a function can be repeated many times without - unintented effects, but not necessarily side-effect free in the pure + unintended effects, but not necessarily side-effect free in the pure sense (compare to :term:`nullipotent`). nullipotent diff --git a/docs/userguide/application.rst b/docs/userguide/application.rst index 4ebc142ef..5cdc74fa4 100644 --- a/docs/userguide/application.rst +++ b/docs/userguide/application.rst @@ -182,7 +182,7 @@ Example 2: Using a configuration module .. tip:: - Using the name of a module is recomended + Using the name of a module is recommended as this means that the module doesn't need to be serialized when the prefork pool is used. If you're experiencing configuration pickle errors then please try using diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index 7fc2d8d72..92eb6eef5 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -47,7 +47,7 @@ The API defines a standard set of execution options, as well as three methods: executes 10 seconds from now. - ``T.apply_async(eta=now + timedelta(seconds=10))`` - executes 10 seconds from now, specifed using ``eta`` + executes 10 seconds from now, specified using ``eta`` - ``T.apply_async(countdown=60, expires=120)`` executes in one minute from now, but expires after 2 minutes. @@ -447,7 +447,7 @@ Though this particular example is much better expressed as a group: >>> from celery import group >>> numbers = [(2, 2), (4, 4), (8, 8), (16, 16)] - >>> res = group(add.s(n) for i in numbers).apply_async() + >>> res = group(add.s(n) for n in numbers).apply_async() >>> res.get() [4, 8, 16, 32] diff --git a/docs/userguide/canvas.rst b/docs/userguide/canvas.rst index 5f86032f0..f285f6755 100644 --- a/docs/userguide/canvas.rst +++ b/docs/userguide/canvas.rst @@ -109,7 +109,7 @@ creates partials: - Any arguments added will be prepended to the args in the signature:: >>> partial = add.s(2) # incomplete signature - >>> partial.delay(4) # 2 + 4 + >>> partial.delay(4) # 4 + 2 >>> partial.apply_async((4, )) # same - Any keyword arguments added will be merged with the kwargs in the signature, @@ -125,7 +125,7 @@ creates partials: >>> s = add.signature((2, 2), countdown=10) >>> s.apply_async(countdown=1) # countdown is now 1 -You can also clone signatures to create derivates: +You can also clone signatures to create derivatives: >>> s = add.s(2) proj.tasks.add(2) diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index a4417c78a..50ee1a514 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -768,7 +768,7 @@ will take some time so other transports still use a threading-based solution. Add callback to be called when ``fd`` is readable. - The callback will stay registered until explictly removed using + The callback will stay registered until explicitly removed using :meth:`hub.remove(fd) `, or the fd is automatically discarded because it's no longer valid. diff --git a/docs/userguide/routing.rst b/docs/userguide/routing.rst index 9ea1e503c..2de3226cd 100644 --- a/docs/userguide/routing.rst +++ b/docs/userguide/routing.rst @@ -402,7 +402,7 @@ using the ``basic.publish`` command:: Now that the message is sent you can retrieve it again. You can use the ``basic.get``` command here, which polls for new messages on the queue -(which is alright for maintainence tasks, for services you'd want to use +(which is alright for maintenance tasks, for services you'd want to use ``basic.consume`` instead) Pop a message off the queue:: diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index d0ce0f291..5ce487c6b 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -19,7 +19,7 @@ many messages in advance and even if the worker is killed -- caused by power fai or otherwise -- the message will be redelivered to another worker. Ideally task functions should be :term:`idempotent`, which means that -the function will not cause unintented effects even if called +the function will not cause unintended effects even if called multiple times with the same arguments. Since the worker cannot detect if your tasks are idempotent, the default behavior is to acknowledge the message in advance, before it's executed, From 34082df653485d62a72d31f8258f46ec00b330c1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 30 Jun 2014 15:51:00 +0100 Subject: [PATCH 0232/1103] Task.signature_from_request must propagate reply_to for RPC backend. Closes #2113 --- celery/app/task.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/app/task.py b/celery/app/task.py index 32b16787f..b65803b82 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -476,6 +476,7 @@ def signature_from_request(self, request=None, args=None, kwargs=None, 'chord': request.chord, 'soft_time_limit': limit_soft, 'time_limit': limit_hard, + 'reply_to': request.reply_to, } options.update( {'queue': queue} if queue else (request.delivery_info or {}) From ea0124cb52805f4822191eb4fe077157a4159066 Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Tue, 1 Jul 2014 22:10:09 +0700 Subject: [PATCH 0233/1103] Deleted unnecesarry decorator & updated docs and tests --- celery/app/base.py | 18 ++++++--- celery/contrib/autoretry.py | 52 -------------------------- celery/tests/contrib/test_autoretry.py | 39 ------------------- celery/tests/tasks/test_tasks.py | 8 ++-- docs/userguide/tasks.rst | 18 ++------- 5 files changed, 21 insertions(+), 114 deletions(-) delete mode 100644 celery/contrib/autoretry.py delete mode 100644 celery/tests/contrib/test_autoretry.py diff --git a/celery/app/base.py b/celery/app/base.py index da71617c6..36622c093 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -15,6 +15,7 @@ from collections import defaultdict, deque from copy import deepcopy from operator import attrgetter +from functools import wraps from amqp import promise from billiard.util import register_after_fork @@ -38,7 +39,6 @@ from celery.utils.functional import first, maybe_list from celery.utils.imports import instantiate, symbol_by_name from celery.utils.objects import FallbackContext, mro_lookup -from celery.contrib.autoretry import autoretry from .annotations import prepare as prepare_annotations from .defaults import DEFAULTS, find_deprecated_settings @@ -284,11 +284,19 @@ def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): self._tasks[task.name] = task task.bind(self) # connects task to this app - autoretry_on = options.get('autoretry_on') - retry_kwargs = options.get('retry_kwargs') + autoretry_for = tuple(options.get('autoretry_for', ())) + retry_kwargs = options.get('retry_kwargs', {}) - if autoretry_on: - task = autoretry(autoretry_on, retry_kwargs)(task) + if autoretry_for and not hasattr(task, '_orig_run'): + + @wraps(task.run) + def run(*args, **kwargs): + try: + return task._orig_run(*args, **kwargs) + except autoretry_for as exc: + raise task.retry(exc=exc, **retry_kwargs) + + task._orig_run, task.run = task.run, run else: task = self._tasks[name] return task diff --git a/celery/contrib/autoretry.py b/celery/contrib/autoretry.py deleted file mode 100644 index 467aa7854..000000000 --- a/celery/contrib/autoretry.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- -""" -celery.contrib.autoretry -======================== - -.. versionadded:: 3.2 - -Decorator that enables autoretrying when one of specified exceptions -are raised in a task body. - - -Examples --------- - -.. code-block:: python - - from celery.contrib.autoretry import autoretry - - @autoretry(on=(ZeroDivisionError,)) - @app.task - def div - -.. note:: - - `autoretry` decorator must be applied **before** `app.task` decorator. -""" - -from __future__ import absolute_import - -from functools import wraps - - -def autoretry(on=None, retry_kwargs=None): - - def decorator(task): - if not on or hasattr(task, '_orig_run'): - return task.run - - autoretry_exceptions = tuple(on) # except only works with tuples - _retry_kwargs = retry_kwargs or {} - - @wraps(task.run) - def inner(*args, **kwargs): - try: - return task._orig_run(*args, **kwargs) - except autoretry_exceptions as exc: - raise task.retry(exc=exc, **_retry_kwargs) - - task._orig_run = task.run - task.run = inner - return inner - return decorator diff --git a/celery/tests/contrib/test_autoretry.py b/celery/tests/contrib/test_autoretry.py deleted file mode 100644 index 09d5e7fe8..000000000 --- a/celery/tests/contrib/test_autoretry.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import absolute_import - -from celery.contrib.autoretry import autoretry - -from celery.tests.case import AppCase - - -class TasksCase(AppCase): - - def setup(self): - - @autoretry(on=(ZeroDivisionError,)) - @self.app.task(shared=False) - def autoretry_task_no_kwargs(a, b): - self.iterations += 1 - return a/b - self.autoretry_task_no_kwargs = autoretry_task_no_kwargs - - @autoretry(on=(ZeroDivisionError,), retry_kwargs={'max_retries': 5}) - @self.app.task(shared=False) - def autoretry_task(a, b): - self.iterations += 1 - return a/b - self.autoretry_task = autoretry_task - - -class test_autoretry(TasksCase): - - def test_autoretry_no_kwargs(self): - self.autoretry_task_no_kwargs.max_retries = 3 - self.autoretry_task_no_kwargs.iterations = 0 - self.autoretry_task_no_kwargs.apply((1, 0)) - self.assertEqual(self.autoretry_task_no_kwargs.iterations, 4) - - def test_autoretry(self): - self.autoretry_tasks.max_retries = 3 - self.autoretry_task.iterations = 0 - self.autoretry_task.apply((1, 0)) - self.assertEqual(self.autoretry_task.iterations, 6) diff --git a/celery/tests/tasks/test_tasks.py b/celery/tests/tasks/test_tasks.py index 1cf9e868e..fb26ecd02 100644 --- a/celery/tests/tasks/test_tasks.py +++ b/celery/tests/tasks/test_tasks.py @@ -100,18 +100,18 @@ def retry_task_customexc(self, arg1, arg2, kwarg=1, **kwargs): raise self.retry(countdown=0, exc=exc) self.retry_task_customexc = retry_task_customexc - @self.app.task(on=(ZeroDivisionError,), shared=False) + @self.app.task(autoretry_for=(ZeroDivisionError,), shared=False) def autoretry_task_no_kwargs(a, b): self.iterations += 1 return a/b - self.autoretry_task_no_kwargs + self.autoretry_task_no_kwargs = autoretry_task_no_kwargs - @self.app.task(on=(ZeroDivisionError,), + @self.app.task(autoretry_for=(ZeroDivisionError,), retry_kwargs={'max_retries': 5}, shared=False) def autoretry_task(a, b): self.iterations += 1 return a/b - self.autoretry_task + self.autoretry_task = autoretry_task class MyCustomException(Exception): diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index a8f3ff28c..d0018b1b7 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -491,30 +491,20 @@ This may not be acceptable all the time, since you may have a lot of such tasks. Fortunately, you can tell Celery to automatically retry a task using -:func:`autoretry <~celery.contrib.autoretry.autoretry>` decorator: +`autoretry_for` argument in `~@Celery.task` decorator: .. code-block:: python - @autoretry(on=(ZeroDivisionError,)) - @app.task - def div(a, b): - return a / b - -Also you can specify autoretry directly in `~@Celery.task` decorator: - -.. code-block:: python - - @app.task(autoretry_on=(ZeroDivisionError,)) + @app.task(autoretry_for(ZeroDivisionError,)) def div(a, b): return a / b If you want to specify custom arguments for internal `~@Task.retry` -call, pass `retry_kwargs` argument to :func:`autoretry -<~celery.contrib.autoretry.autoretry>` or `~@Celery.task` decorators: +call, pass `retry_kwargs` argument to `~@Celery.task` decorator: .. code-block:: python - @app.task(autoretry_on=(ZeroDivisionError,), + @app.task(autoretry_for=(ZeroDivisionError,), retry_kwargs={'max_retries': 5}) def div(a, b): return a / b From 1fe8e281fee1377263809a239a1f835e898f521b Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Wed, 2 Jul 2014 17:22:31 +0700 Subject: [PATCH 0234/1103] Tests passed --- celery/tests/tasks/test_tasks.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/celery/tests/tasks/test_tasks.py b/celery/tests/tasks/test_tasks.py index fb26ecd02..dc7775bee 100644 --- a/celery/tests/tasks/test_tasks.py +++ b/celery/tests/tasks/test_tasks.py @@ -100,15 +100,16 @@ def retry_task_customexc(self, arg1, arg2, kwarg=1, **kwargs): raise self.retry(countdown=0, exc=exc) self.retry_task_customexc = retry_task_customexc - @self.app.task(autoretry_for=(ZeroDivisionError,), shared=False) - def autoretry_task_no_kwargs(a, b): + @self.app.task(bind=True, autoretry_for=(ZeroDivisionError,), + shared=False) + def autoretry_task_no_kwargs(self, a, b): self.iterations += 1 return a/b self.autoretry_task_no_kwargs = autoretry_task_no_kwargs - @self.app.task(autoretry_for=(ZeroDivisionError,), + @self.app.task(bind=True, autoretry_for=(ZeroDivisionError,), retry_kwargs={'max_retries': 5}, shared=False) - def autoretry_task(a, b): + def autoretry_task(self, a, b): self.iterations += 1 return a/b self.autoretry_task = autoretry_task From 6d0d4ddbf1c3dcd4c6a791dea9f6cc869e3118b7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Jul 2014 11:29:53 +0100 Subject: [PATCH 0235/1103] Fixes beat when empty schedule --- celery/beat.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/celery/beat.py b/celery/beat.py index 9b8c4653c..972579759 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -232,6 +232,8 @@ def tick(self, event_t=event_t, min=min, H = self._heap = [event_t(adjust(e.is_due()[1]) or 0, 5, e) for e in values(self.schedule)] heapify(H) + if not H: + return max_interval event = H[0] entry = event[2] is_due, next_time_to_run = self.is_due(entry) From 590efe42118733218466365ee3f43d91879755b9 Mon Sep 17 00:00:00 2001 From: Lucas Wiman Date: Tue, 1 Jul 2014 14:39:49 -0700 Subject: [PATCH 0236/1103] Correct arithmetic and add note about rate_limit --- docs/userguide/tasks.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 5ce487c6b..91d4b4b8e 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -567,11 +567,16 @@ General distributed over the specified time frame. Example: `"100/m"` (hundred tasks a minute). This will enforce a minimum - delay of 10ms between starting two tasks. + delay of 600ms between starting two tasks on the same worker instance. Default is the :setting:`CELERY_DEFAULT_RATE_LIMIT` setting, which if not specified means rate limiting for tasks is disabled by default. + Note that this is a *per worker instance* rate limit, and not a global + rate limit. To enforce a global rate limit (e.g. for an API with a + maximum number of requests per second), you must restrict to a given + queue. + .. attribute:: Task.time_limit The hard time limit, in seconds, for this task. If not set then the workers default From 3a6780b44bdb96c39e42662db3fb7220725446d7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Jul 2014 12:23:26 +0100 Subject: [PATCH 0237/1103] Removes tag_date --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 68aa7cdd1..682cb7d93 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,6 +16,3 @@ requires = pytz >= 2011b [wheel] universal = 1 - -[egg_info] -tag_date = true From 732b2364e4ea167b4982774837197783b6974f8c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Jul 2014 12:35:41 +0100 Subject: [PATCH 0238/1103] chunks/map/starmap tasks now routes based on the target task (introducing Signature.route_name_for) --- celery/app/base.py | 4 ++-- celery/canvas.py | 39 ++++++++++++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 45546af70..52b749bc8 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -355,7 +355,7 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, publisher=None, link=None, link_error=None, add_to_parent=True, group_id=None, retries=0, chord=None, reply_to=None, time_limit=None, soft_time_limit=None, - root_id=None, parent_id=None, **options): + root_id=None, parent_id=None, route_name=None, **options): amqp = self.amqp task_id = task_id or uuid() producer = producer or publisher # XXX compat @@ -365,7 +365,7 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, warnings.warn(AlwaysEagerIgnored( 'CELERY_ALWAYS_EAGER has no effect on send_task', ), stacklevel=2) - options = router.route(options, name, args, kwargs) + options = router.route(options, route_name or name, args, kwargs) message = amqp.create_task_message( task_id, name, args, kwargs, countdown, eta, group_id, diff --git a/celery/canvas.py b/celery/canvas.py index 0384a4d8b..38043d004 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -99,6 +99,10 @@ def maybe_unroll_group(g): return g.tasks[0] if size == 1 else g +def task_name_from(task): + return getattr(task, 'name', task) + + class Signature(dict): """Class that wraps the arguments and execution options for a single task invocation. @@ -230,7 +234,7 @@ def set(self, immutable=None, **options): def set_immutable(self, immutable): self.immutable = immutable - def apply_async(self, args=(), kwargs={}, **options): + def apply_async(self, args=(), kwargs={}, route_name=None, **options): try: _apply = self._apply_async except IndexError: # no tasks for chain, etc to find type @@ -240,7 +244,17 @@ def apply_async(self, args=(), kwargs={}, **options): args, kwargs, options = self._merge(args, kwargs, options) else: args, kwargs, options = self.args, self.kwargs, self.options - return _apply(args, kwargs, **options) + route_name = route_name or self.route_name_for(args, kwargs, options) + return _apply(args, kwargs, route_name=route_name, **options) + + def route_name_for(self, args, kwargs, options): + """Can be used to override the name used for routing the task + to a queue. + + If this returns :const:`None` the name of the task will be used. + + """ + pass def append_to_list_option(self, key, value): items = self.options.setdefault(key, []) @@ -309,6 +323,11 @@ def election(self): def __repr__(self): return self.reprcall() + @property + def name(self): + # for duck typing compatibility with Task.name + return self.task + @cached_property def type(self): return self._type or self.app.tasks[self['task']] @@ -485,11 +504,15 @@ def __init__(self, task, it, **options): {'task': task, 'it': regen(it)}, immutable=True, **options ) + def route_name_for(self, args, kwargs, options): + return task_name_from(self.kwargs.get('task')) + def apply_async(self, args=(), kwargs={}, **opts): # need to evaluate generators task, it = self._unpack_args(self.kwargs) return self.type.apply_async( - (), {'task': task, 'it': list(it)}, **opts + (), {'task': task, 'it': list(it)}, + route_name=self.route_name_for(args, kwargs, opts), **opts ) @classmethod @@ -532,11 +555,17 @@ def __init__(self, task, it, n, **options): def from_dict(self, d, app=None): return chunks(*self._unpack_args(d['kwargs']), app=app, **d['options']) + def route_name_for(self, args, kwargs, options): + return task_name_from(self.kwargs.get('task')) + def apply_async(self, args=(), kwargs={}, **opts): - return self.group().apply_async(args, kwargs, **opts) + return self.group().apply_async( + args, kwargs, + route_name=self.route_name_for(args, kwargs, opts), **opts + ) def __call__(self, **options): - return self.group()(**options) + return self.apply_async(**options) def group(self): # need to evaluate generators From 792a23184e55a14ba197fdcf5f1b1f8a768ee226 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Jul 2014 12:36:37 +0100 Subject: [PATCH 0239/1103] flakes --- celery/app/builtins.py | 2 -- celery/backends/riak.py | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/celery/app/builtins.py b/celery/app/builtins.py index 4144607b9..b633aeb81 100644 --- a/celery/app/builtins.py +++ b/celery/app/builtins.py @@ -166,13 +166,11 @@ def chain(*args, **kwargs): return chain - @connect_on_app_finalize def add_chord_task(app): """No longer used, but here for backwards compatibility.""" from celery import group, chord as _chord from celery.canvas import maybe_signature - _app = app @app.task(name='celery.chord', bind=True, ignore_result=False, shared=False, lazy=False) diff --git a/celery/backends/riak.py b/celery/backends/riak.py index 944ef5b8a..f25ae4f3e 100644 --- a/celery/backends/riak.py +++ b/celery/backends/riak.py @@ -26,6 +26,7 @@ Riak bucket names must be composed of ASCII characters only, not: {0!r}\ """ + def is_ascii(s): try: s.decode('ascii') From f90a84da4c7a0bdc729bd3e15989853d04d27393 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Jul 2014 13:06:09 +0100 Subject: [PATCH 0240/1103] App: Backend argument must also set CELERY_RESULT_BACKEND config --- celery/app/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/app/base.py b/celery/app/base.py index 52b749bc8..07738571b 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -140,7 +140,6 @@ def __init__(self, main=None, loader=None, backend=None, self.clock = LamportClock() self.main = main self.amqp_cls = amqp or self.amqp_cls - self.backend_cls = backend or self.backend_cls self.events_cls = events or self.events_cls self.loader_cls = loader or self.loader_cls self.log_cls = log or self.log_cls @@ -173,6 +172,8 @@ def __init__(self, main=None, loader=None, backend=None, self._preconf = changes or {} if broker: self._preconf['BROKER_URL'] = broker + if backend: + self._preconf['CELERY_RESULT_BACKEND'] = backend if include: self._preconf['CELERY_IMPORTS'] = include From c9bbe64afbc711557f5899b8b7cec9010b08b500 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Jul 2014 19:51:07 +0100 Subject: [PATCH 0241/1103] Removes Signature.route_name_for, fixing tests --- celery/backends/base.py | 11 ++++++----- celery/backends/redis.py | 4 ++-- celery/canvas.py | 28 ++++++++-------------------- celery/tests/app/test_builtins.py | 1 - celery/tests/tasks/test_canvas.py | 7 ++++--- 5 files changed, 20 insertions(+), 31 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index b69e9e238..1dd5ff1f1 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -351,9 +351,10 @@ def fallback_chord_unlock(self, group_id, body, result=None, (group_id, body, ), kwargs, countdown=countdown, ) - def apply_chord(self, header, partial_args, group_id, body, **options): - result = header(*partial_args, task_id=group_id) - self.fallback_chord_unlock(group_id, body, **options) + def apply_chord(self, header, partial_args, group_id, body, + options={}, **kwargs): + result = header(*partial_args, task_id=group_id, **options or {}) + self.fallback_chord_unlock(group_id, body, **kwargs) return result def current_task_children(self, request=None): @@ -516,9 +517,9 @@ def _restore_group(self, group_id): return meta def _apply_chord_incr(self, header, partial_args, group_id, body, - result=None, **options): + result=None, options={}, **kwargs): self.save_group(group_id, self.app.GroupResult(group_id, result)) - return header(*partial_args, task_id=group_id) + return header(*partial_args, task_id=group_id, **options or {}) def on_chord_part_return(self, task, state, result, propagate=None): if not self.implements_incr: diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 314b1d2b8..0c62c7411 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -185,9 +185,9 @@ def _unpack_chord_result(self, tup, decode, return retval def _new_chord_apply(self, header, partial_args, group_id, body, - result=None, **options): + result=None, options={}, **kwargs): # avoids saving the group in the redis db. - return header(*partial_args, task_id=group_id) + return header(*partial_args, task_id=group_id, **options or {}) def _new_chord_return(self, task, state, result, propagate=None, PROPAGATE_STATES=states.PROPAGATE_STATES): diff --git a/celery/canvas.py b/celery/canvas.py index 38043d004..e5f870eee 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -244,17 +244,7 @@ def apply_async(self, args=(), kwargs={}, route_name=None, **options): args, kwargs, options = self._merge(args, kwargs, options) else: args, kwargs, options = self.args, self.kwargs, self.options - route_name = route_name or self.route_name_for(args, kwargs, options) - return _apply(args, kwargs, route_name=route_name, **options) - - def route_name_for(self, args, kwargs, options): - """Can be used to override the name used for routing the task - to a queue. - - If this returns :const:`None` the name of the task will be used. - - """ - pass + return _apply(args, kwargs, **options) def append_to_list_option(self, key, value): items = self.options.setdefault(key, []) @@ -504,15 +494,12 @@ def __init__(self, task, it, **options): {'task': task, 'it': regen(it)}, immutable=True, **options ) - def route_name_for(self, args, kwargs, options): - return task_name_from(self.kwargs.get('task')) - def apply_async(self, args=(), kwargs={}, **opts): # need to evaluate generators task, it = self._unpack_args(self.kwargs) return self.type.apply_async( (), {'task': task, 'it': list(it)}, - route_name=self.route_name_for(args, kwargs, opts), **opts + route_name=task_name_from(self.kwargs.get('task')), **opts ) @classmethod @@ -555,13 +542,10 @@ def __init__(self, task, it, n, **options): def from_dict(self, d, app=None): return chunks(*self._unpack_args(d['kwargs']), app=app, **d['options']) - def route_name_for(self, args, kwargs, options): - return task_name_from(self.kwargs.get('task')) - def apply_async(self, args=(), kwargs={}, **opts): return self.group().apply_async( args, kwargs, - route_name=self.route_name_for(args, kwargs, opts), **opts + route_name=task_name_from(self.kwargs.get('task')), **opts ) def __call__(self, **options): @@ -833,6 +817,9 @@ def run(self, header, body, partial_args, app=None, interval=None, root_id = body.options.get('root_id') if 'chord_size' not in body: body['chord_size'] = self.__length_hint__() + options = dict(self.options, **options) if options else self.options + if options: + body.options.update(options) results = header.freeze( group_id=group_id, chord=body, root_id=root_id).results @@ -841,7 +828,8 @@ def run(self, header, body, partial_args, app=None, interval=None, parent = app.backend.apply_chord( header, partial_args, group_id, body, interval=interval, countdown=countdown, - max_retries=max_retries, propagate=propagate, result=results) + options=options, max_retries=max_retries, + propagate=propagate, result=results) bodyres.parent = parent return bodyres diff --git a/celery/tests/app/test_builtins.py b/celery/tests/app/test_builtins.py index 89e23c6e9..5642cbcd0 100644 --- a/celery/tests/app/test_builtins.py +++ b/celery/tests/app/test_builtins.py @@ -89,7 +89,6 @@ def test_apply_async_eager(self): def test_apply(self): x = group([self.add.s(4, 4), self.add.s(8, 8)]) - x.name = self.task.name res = x.apply() self.assertEqual(res.get(), [8, 16]) diff --git a/celery/tests/tasks/test_canvas.py b/celery/tests/tasks/test_canvas.py index 25080252b..393cda69b 100644 --- a/celery/tests/tasks/test_canvas.py +++ b/celery/tests/tasks/test_canvas.py @@ -171,6 +171,7 @@ def test_apply(self): s.apply_async(foo=1) s.type.apply_async.assert_called_with( (), {'task': self.add.s(), 'it': args}, foo=1, + route_name=self.add.name, ) self.assertEqual(type.from_dict(dict(s)), s) @@ -192,10 +193,10 @@ def test_chunks(self): gr = x.group.return_value = Mock() x.apply_async() - gr.apply_async.assert_called_with((), {}) - + gr.apply_async.assert_called_with((), {}, route_name=self.add.name) + gr.apply_async.reset_mock() x() - gr.assert_called_with() + gr.apply_async.assert_called_with((), {}, route_name=self.add.name) self.app.conf.CELERY_ALWAYS_EAGER = True chunks.apply_chunks(app=self.app, **x['kwargs']) From e4a8419046623f21815e7189ba1ad24d588e22c6 Mon Sep 17 00:00:00 2001 From: flyingfoxlee Date: Sat, 5 Jul 2014 18:38:46 +0800 Subject: [PATCH 0242/1103] Fix typo. There is no `self` in the context, `__name__` seems reasonable. --- docs/tutorials/task-cookbook.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/task-cookbook.rst b/docs/tutorials/task-cookbook.rst index c47c09a7d..16bad2724 100644 --- a/docs/tutorials/task-cookbook.rst +++ b/docs/tutorials/task-cookbook.rst @@ -43,7 +43,7 @@ The cache key expires after some time in case something unexpected happens # The cache key consists of the task name and the MD5 digest # of the feed URL. feed_url_digest = md5(feed_url).hexdigest() - lock_id = '{0}-lock-{1}'.format(self.name, feed_url_hexdigest) + lock_id = '{0}-lock-{1}'.format(__name__, feed_url_hexdigest) # cache.add fails if the key already exists acquire_lock = lambda: cache.add(lock_id, 'true', LOCK_EXPIRE) From 85a924a12aa5212a08e34e1600b9cd4a23b7cf95 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 7 Jul 2014 15:28:51 +0100 Subject: [PATCH 0243/1103] Fix for #2120 --- docs/tutorials/task-cookbook.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/task-cookbook.rst b/docs/tutorials/task-cookbook.rst index 16bad2724..ca3fa5065 100644 --- a/docs/tutorials/task-cookbook.rst +++ b/docs/tutorials/task-cookbook.rst @@ -38,12 +38,12 @@ The cache key expires after some time in case something unexpected happens LOCK_EXPIRE = 60 * 5 # Lock expires in 5 minutes - @task - def import_feed(feed_url): + @task(bind=True) + def import_feed(self, feed_url): # The cache key consists of the task name and the MD5 digest # of the feed URL. feed_url_digest = md5(feed_url).hexdigest() - lock_id = '{0}-lock-{1}'.format(__name__, feed_url_hexdigest) + lock_id = '{0}-lock-{1}'.format(self.name, feed_url_hexdigest) # cache.add fails if the key already exists acquire_lock = lambda: cache.add(lock_id, 'true', LOCK_EXPIRE) From 4b531b4dfd52c1cedc7765eee3f0069aef61eb94 Mon Sep 17 00:00:00 2001 From: nicholsonjf Date: Fri, 13 Jun 2014 22:10:48 -0700 Subject: [PATCH 0244/1103] A few small grammar changes Fixed a couple inconsistencies and run on sentences to improve the flow of the introduction. --- docs/getting-started/introduction.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/getting-started/introduction.rst b/docs/getting-started/introduction.rst index 874ba109e..ca8b480e0 100644 --- a/docs/getting-started/introduction.rst +++ b/docs/getting-started/introduction.rst @@ -14,19 +14,19 @@ What is a Task Queue? Task queues are used as a mechanism to distribute work across threads or machines. -A task queue's input is a unit of work, called a task, dedicated worker -processes then constantly monitor the queue for new work to perform. +A task queue's input is a unit of work called a task. Dedicated worker +processes constantly monitor task queues for new work to perform. Celery communicates via messages, usually using a broker -to mediate between clients and workers. To initiate a task a client puts a -message on the queue, the broker then delivers the message to a worker. +to mediate between clients and workers. To initiate a task, a client adds a +message to the queue, which the broker then delivers to a worker. A Celery system can consist of multiple workers and brokers, giving way to high availability and horizontal scaling. Celery is written in Python, but the protocol can be implemented in any language. So far there's RCelery_ for the Ruby programming language, -node-celery_ for Node.js and a `PHP client`_, but language interoperability can also be achieved +node-celery_ for Node.js and a `PHP client`_. Language interoperability can also be achieved by :ref:`using webhooks `. .. _RCelery: http://leapfrogdevelopment.github.com/rcelery/ From 9e3ba44b979bb21f5db26c201a4362378cf769b5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 7 Jul 2014 17:48:33 +0100 Subject: [PATCH 0245/1103] Tests passing Conflicts: celery/app/base.py celery/tests/app/test_app.py --- celery/app/base.py | 7 ++----- celery/tests/app/test_app.py | 3 +-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 07738571b..c72a87406 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -481,17 +481,14 @@ def _get_config(self): self.on_configure() if self._config_source: self.loader.config_from_object(self._config_source) + defaults = dict(deepcopy(DEFAULTS), **self._preconf) self.configured = True s = Settings({}, [self.prepare_config(self.loader.conf), - deepcopy(DEFAULTS)]) - + defaults]) # load lazy config dict initializers. pending = self._pending_defaults while pending: s.add_defaults(maybe_evaluate(pending.popleft()())) - if self._preconf: - for key, value in items(self._preconf): - setattr(s, key, value) self.on_after_configure.send(sender=self, source=s) return s diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index 1fcdf3ffc..977928537 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -432,12 +432,11 @@ def test_config_from_cmdline(self): {'foo': 'bar'}) def test_compat_setting_CELERY_BACKEND(self): - self.app.conf.defaults[0]['CELERY_RESULT_BACKEND'] = None + self.app._preconf = {} # removes result backend set by AppCase self.app.config_from_object(Object(CELERY_BACKEND='set_by_us')) self.assertEqual(self.app.conf.CELERY_RESULT_BACKEND, 'set_by_us') def test_setting_BROKER_TRANSPORT_OPTIONS(self): - _args = {'foo': 'bar', 'spam': 'baz'} self.app.config_from_object(Object()) From 6298c911a88258e5b415c71973d209a659cf24b3 Mon Sep 17 00:00:00 2001 From: Yuval Greenfield Date: Mon, 7 Jul 2014 12:38:01 -0700 Subject: [PATCH 0246/1103] missing an 'all' because 'call' ate it --- docs/reference/celery.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/celery.rst b/docs/reference/celery.rst index 75541f059..ec7966fd4 100644 --- a/docs/reference/celery.rst +++ b/docs/reference/celery.rst @@ -416,7 +416,7 @@ See :ref:`guide-canvas` for more about creating task workflows. A group is lazy so you must call it to take action and evaluate the group. - Will return a `group` task that when called will then call of the + Will return a `group` task that when called will then call all of the tasks in the group (and return a :class:`GroupResult` instance that can be used to inspect the state of the group). From 1457343cb86d67cefd0dbc2a37085d8e7bdef72c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 9 Jul 2014 13:55:58 +0100 Subject: [PATCH 0247/1103] Tests passing --- celery/tests/app/test_app.py | 4 +++- celery/tests/case.py | 8 +++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index 977928537..a58c27fb7 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -432,11 +432,13 @@ def test_config_from_cmdline(self): {'foo': 'bar'}) def test_compat_setting_CELERY_BACKEND(self): - self.app._preconf = {} # removes result backend set by AppCase + self.app._preconf = {} + self.app.conf.defaults[0]['CELERY_RESULT_BACKEND'] = None self.app.config_from_object(Object(CELERY_BACKEND='set_by_us')) self.assertEqual(self.app.conf.CELERY_RESULT_BACKEND, 'set_by_us') def test_setting_BROKER_TRANSPORT_OPTIONS(self): + _args = {'foo': 'bar', 'spam': 'baz'} self.app.config_from_object(Object()) diff --git a/celery/tests/case.py b/celery/tests/case.py index 50bb87a36..ad9951afa 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -84,9 +84,6 @@ """ CELERY_TEST_CONFIG = { - 'BROKER_URL': 'memory://', - 'CELERY_RESULT_BACKEND': 'cache+memory://', - #: Don't want log output when running suite. 'CELERYD_HIJACK_ROOT_LOGGER': False, 'CELERY_SEND_TASK_ERROR_EMAILS': False, @@ -127,10 +124,11 @@ def __init__(self, *args, **kwargs): self.already_setup = True -def UnitApp(name=None, set_as_current=False, log=UnitLogging, **kwargs): +def UnitApp(name=None, set_as_current=False, log=UnitLogging, + broker='memory://', backend='cache+memory://', **kwargs): app = Celery(name or 'celery.tests', set_as_current=set_as_current, - log=log, + log=log, broker=broker, backend=backend, **kwargs) app.add_defaults(deepcopy(CELERY_TEST_CONFIG)) return app From e1f59b29f6300d234cc411487262edc054329a05 Mon Sep 17 00:00:00 2001 From: Ross Deane Date: Thu, 10 Jul 2014 15:14:58 +0100 Subject: [PATCH 0248/1103] fix for #2076 --- celery/canvas.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/celery/canvas.py b/celery/canvas.py index e5f870eee..3c9370dff 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -466,6 +466,8 @@ def apply(self, args=(), kwargs={}, **options): def from_dict(self, d, app=None): tasks = d['kwargs']['tasks'] if d['args'] and tasks: + # make sure that tasks are made into signatures (Issue #2076) + tasks[0] = signature(tasks[0]) # partial args passed on to first task in chain (Issue #1057). tasks[0]['args'] = tasks[0]._merge(d['args'])[0] return chain(*d['kwargs']['tasks'], app=app, **d['options']) @@ -591,6 +593,8 @@ def from_dict(self, d, app=None): if d['args'] and tasks: # partial args passed on to all tasks in the group (Issue #1057). for task in tasks: + # make sure that tasks are made into signatures (Issue #2076) + task = signature(task) task['args'] = task._merge(d['args'])[0] return group(tasks, app=app, **d['options']) From 7ae69fb1d9e9c8f74beb9e2e284c1844d8464d7c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Jul 2014 16:56:49 +0100 Subject: [PATCH 0249/1103] Change default mask --- celery/app/base.py | 2 +- celery/bin/base.py | 5 +++-- celery/bin/celeryd_detach.py | 2 +- celery/platforms.py | 8 +++++--- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index c72a87406..f823ae4bd 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -30,7 +30,7 @@ _announce_app_finalized, ) from celery.exceptions import AlwaysEagerIgnored, ImproperlyConfigured -from celery.five import items, values +from celery.five import values from celery.loaders import get_loader_cls from celery.local import PromiseProxy, maybe_evaluate from celery.utils import gen_task_name diff --git a/celery/bin/base.py b/celery/bin/base.py index 79bdb5c8a..e9beb15ee 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -56,7 +56,8 @@ .. cmdoption:: --umask - Effective umask of the process after detaching. Default is 0. + Effective umask (in octal) of the process after detaching. Inherits + the umask of the parent process by default. .. cmdoption:: --workdir @@ -649,5 +650,5 @@ def daemon_options(default_pidfile=None, default_logfile=None): Option('--pidfile', default=default_pidfile), Option('--uid', default=None), Option('--gid', default=None), - Option('--umask', default=0, type='int'), + Option('--umask', default=None), ) diff --git a/celery/bin/celeryd_detach.py b/celery/bin/celeryd_detach.py index 8a6d339d4..862fc8979 100644 --- a/celery/bin/celeryd_detach.py +++ b/celery/bin/celeryd_detach.py @@ -38,7 +38,7 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, - gid=None, umask=0, working_directory=None, fake=False, app=None): + gid=None, umask=None, working_directory=None, fake=False, app=None): fake = 1 if C_FAKEFORK else fake with detached(logfile, pidfile, uid, gid, umask, working_directory, fake, after_forkers=False): diff --git a/celery/platforms.py b/celery/platforms.py index 0ddc3d6ae..766a30e74 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -56,7 +56,6 @@ IS_OSX = SYSTEM == 'Darwin' IS_WINDOWS = SYSTEM == 'Windows' -DAEMON_UMASK = 0 DAEMON_WORKDIR = '/' PIDFILE_FLAGS = os.O_CREAT | os.O_EXCL | os.O_WRONLY @@ -296,8 +295,10 @@ class DaemonContext(object): def __init__(self, pidfile=None, workdir=None, umask=None, fake=False, after_chdir=None, after_forkers=True, **kwargs): + if isinstance(umask, string_t): + umask = int(umask, 8) # convert str -> octal self.workdir = workdir or DAEMON_WORKDIR - self.umask = DAEMON_UMASK if umask is None else umask + self.umask = umask self.fake = fake self.after_chdir = after_chdir self.after_forkers = after_forkers @@ -314,7 +315,8 @@ def open(self): self._detach() os.chdir(self.workdir) - os.umask(self.umask) + if self.umask is not None: + os.umask(self.umask) if self.after_chdir: self.after_chdir() From 452d388bab0bde5c05473ab8eee6acaef8cd1463 Mon Sep 17 00:00:00 2001 From: Ross Deane Date: Thu, 10 Jul 2014 17:19:48 +0100 Subject: [PATCH 0250/1103] pass tests --- celery/canvas.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 3c9370dff..d0a824e01 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -467,7 +467,8 @@ def from_dict(self, d, app=None): tasks = d['kwargs']['tasks'] if d['args'] and tasks: # make sure that tasks are made into signatures (Issue #2076) - tasks[0] = signature(tasks[0]) + if not isinstance(tasks[0], Signature): + tasks[0] = signature(tasks[0]) # partial args passed on to first task in chain (Issue #1057). tasks[0]['args'] = tasks[0]._merge(d['args'])[0] return chain(*d['kwargs']['tasks'], app=app, **d['options']) @@ -594,7 +595,8 @@ def from_dict(self, d, app=None): # partial args passed on to all tasks in the group (Issue #1057). for task in tasks: # make sure that tasks are made into signatures (Issue #2076) - task = signature(task) + if not isinstance(tasks[0], Signature): + task = signature(task) task['args'] = task._merge(d['args'])[0] return group(tasks, app=app, **d['options']) From 34c43244b1681a59540936748800aaa504786a35 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Jul 2014 17:52:49 +0100 Subject: [PATCH 0251/1103] Tests passing Conflicts: celery/tests/bin/test_celeryd_detach.py --- celery/tests/bin/test_celeryd_detach.py | 4 ++-- celery/tests/utils/test_platforms.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/celery/tests/bin/test_celeryd_detach.py b/celery/tests/bin/test_celeryd_detach.py index 9e1b4a04d..6c529e9c4 100644 --- a/celery/tests/bin/test_celeryd_detach.py +++ b/celery/tests/bin/test_celeryd_detach.py @@ -25,7 +25,7 @@ def test_execs(self, setup_logs, logger, execv, detached): detach('/bin/boo', ['a', 'b', 'c'], logfile='/var/log', pidfile='/var/pid') detached.assert_called_with( - '/var/log', '/var/pid', None, None, 0, None, False, + '/var/log', '/var/pid', None, None, None, None, False, after_forkers=False, ) execv.assert_called_with('/bin/boo', ['/bin/boo', 'a', 'b', 'c']) @@ -86,7 +86,7 @@ def test_execute_from_commandline(self, detach, exit): self.assertTrue(exit.called) detach.assert_called_with( path=x.execv_path, uid=None, gid=None, - umask=0, fake=False, logfile='/var/log', pidfile='celeryd.pid', + umask=None, fake=False, logfile='/var/log', pidfile='celeryd.pid', working_directory=None, argv=x.execv_argv + [ '-c', '1', '-lDEBUG', diff --git a/celery/tests/utils/test_platforms.py b/celery/tests/utils/test_platforms.py index 4e27efd7b..aae0b38a0 100644 --- a/celery/tests/utils/test_platforms.py +++ b/celery/tests/utils/test_platforms.py @@ -372,7 +372,7 @@ class test_DaemonContext(Case): @patch('os.dup2') def test_open(self, dup2, open, close, closer, umask, chdir, _exit, setsid, fork): - x = DaemonContext(workdir='/opt/workdir') + x = DaemonContext(workdir='/opt/workdir', umask=0o22) x.stdfds = [0, 1, 2] fork.return_value = 0 @@ -385,7 +385,7 @@ def test_open(self, dup2, open, close, closer, umask, chdir, self.assertFalse(_exit.called) chdir.assert_called_with(x.workdir) - umask.assert_called_with(x.umask) + umask.assert_called_with(0o22) self.assertTrue(dup2.called) fork.reset_mock() From 12590a5aba7ff38069583b97cbbd5ed6e7006f6f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 9 Jul 2014 15:43:23 +0100 Subject: [PATCH 0252/1103] Updates Changelog Conflicts: Changelog --- Changelog | 161 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) diff --git a/Changelog b/Changelog index c523701e4..4b0543457 100644 --- a/Changelog +++ b/Changelog @@ -8,6 +8,167 @@ This document contains change notes for bugfix releases in the 3.1.x series (Cipater), please see :ref:`whatsnew-3.1` for an overview of what's new in Celery 3.1. +.. _version-3.1.13: + +3.1.13 +====== + +- **Requirements** + + - Now depends on :ref:`Kombu 3.0.21 `. + + - Now depends on :mod:`billiard` 3.3.0.18. + + +- **App**: ``backend`` argument now also sets the :setting:`CELERY_RESULT_BACKEND` + setting. + +- **Task**: ``signature_from_request`` now propagates ``reply_to`` so that + the RPC backend works with retried tasks (Issue #2113). + +- **Task**: ``retry`` will no longer attempt to requeue the task if sending + the retry message fails. + + Unrelated exceptions being raised could cause a message loop, so it was + better to remove this behavior. + +- **Beat**: Accounts for standard 1ms drift by always waking up 0.010s + earlier. + + This will adjust the latency so that the periodic tasks will not move + 1ms after every invocation. + +- Documentation fixes + + Contributed by Yuval Greenfield, Lucas Wiman, nicholsonjf + +- **Worker**: Removed an outdated assert statement that could lead to errors + being masked (Issue #2086). + + + +.. _version-3.1.12: + +3.1.12 +====== +:release-date: 2014-06-09 10:12 P.M UTC +:release-by: Ask Solem + +- **Requirements** + + Now depends on :ref:`Kombu 3.0.19 `. + +- **App**: Connections were not being closed after fork due to an error in the + after fork handler (Issue #2055). + + This could manifest itself by causing framing errors when using RabbitMQ. + (``Unexpected frame``). + +- **Django**: ``django.setup()`` was being called too late when + using Django 1.7 (Issue #1802). + +- **Django**: Fixed problems with event timezones when using Django + (``Substantial drift``). + + Celery did not take into account that Django modifies the + ``time.timeone`` attributes and friends. + +- **Canvas**: ``Signature.link`` now works when the link option is a scalar + value (Issue #2019). + +- **Prefork pool**: Fixed race conditions for when file descriptors are + removed from the event loop. + + Fix contributed by Roger Hu. + +- **Prefork pool**: Improved solution for dividing tasks between child + processes. + + This change should improve performance when there are many child + processes, and also decrease the chance that two subsequent tasks are + written to the same child process. + +- **Worker**: Now ignores unknown event types, instead of crashing. + + Fix contributed by Illes Solt. + +- **Programs**: :program:`celery worker --detach` no longer closes open file + descriptors when :envvar:`C_FAKEFORK` is used so that the workers output + can be seen. + +- **Programs**: The default working directory for :program:`celery worker + --detach` is now the current working directory, not ``/``. + +- **Canvas**: ``signature(s, app=app)`` did not upgrade serialized signatures + to their original class (``subtask_type``) when the ``app`` keyword argument + was used. + +- **Control**: The ``duplicate nodename`` warning emitted by control commands + now shows the duplicate node name. + +- **Tasks**: Can now call ``ResultSet.get()`` on a result set without members. + + Fix contributed by Alexey Kotlyarov. + +- **App**: Fixed strange traceback mangling issue for + ``app.connection_or_acquire``. + +- **Programs**: The :program:`celery multi stopwait` command is now documented + in usage. + +- **Other**: Fixed cleanup problem with ``PromiseProxy`` when an error is + raised while trying to evaluate the promise. + +- **Other**: The utility used to censor configuration values now handles + non-string keys. + + Fix contributed by Luke Pomfrey. + +- **Other**: The ``inspect conf`` command did not handle non-string keys well. + + Fix contributed by Jay Farrimond. + +- **Programs**: Fixed argument handling problem in + :program:`celery worker --detach`. + + Fix contributed by Dmitry Malinovsky. + +- **Programs**: :program:`celery worker --detach` did not forward working + directory option (Issue #2003). + +- **Programs**: :program:`celery inspect registered` no longer includes + the list of built-in tasks. + +- **Worker**: The ``requires`` attribute for boot steps were not being handled + correctly (Issue #2002). + +- **Eventlet**: The eventlet pool now supports the ``pool_grow`` and + ``pool_shrink`` remote control commands. + + Contributed by Mher Movsisyan. + +- **Eventlet**: The eventlet pool now implements statistics for + :program:``celery inspect stats``. + + Contributed by Mher Movsisyan. + +- **Documentation**: Clarified ``Task.rate_limit`` behavior. + + Contributed by Jonas Haag. + +- **Documentation**: ``AbortableTask`` examples now updated to use the new + API (Issue #1993). + +- **Documentation**: The security documentation examples used an out of date + import. + + Fix contributed by Ian Dees. + +- **Init scripts**: The CentOS init scripts did not quote + :envvar:`CELERY_CHDIR`. + + Fix contributed by ffeast. + .. _version-3.1.11: 3.1.11 From 765a59d87a44bc7e7feec0d37a76ae5656b12f03 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Jul 2014 17:33:16 +0100 Subject: [PATCH 0253/1103] Updates Changelog --- Changelog | 23 ++++++++++ docs/sec/CELERYSA-0002.txt | 90 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 docs/sec/CELERYSA-0002.txt diff --git a/Changelog b/Changelog index 4b0543457..4fd02f286 100644 --- a/Changelog +++ b/Changelog @@ -13,6 +13,29 @@ new in Celery 3.1. 3.1.13 ====== +Security Fixes +-------------- + +* [Security: `CELERYSA-0002`_] Insecure default umask. + + The built-in utility used to daemonize the Celery worker service sets + an insecure umask by default (umask 0). + + This means that any files or directories created by the worker will + end up having world-writable permissions. + + Special thanks to Red Hat for originally discovering and reporting the + issue! + + This version will no longer set a default umask by default, so if unset + the umask of the parent process will be used. + +.. _`CELERYSA-0002`: + http://github.com/celery/celery/tree/master/docs/sec/CELERYSA-0002.txt + +News +---- + - **Requirements** - Now depends on :ref:`Kombu 3.0.21 `. diff --git a/docs/sec/CELERYSA-0002.txt b/docs/sec/CELERYSA-0002.txt new file mode 100644 index 000000000..dd600b018 --- /dev/null +++ b/docs/sec/CELERYSA-0002.txt @@ -0,0 +1,90 @@ +========================================= + CELERYSA-0002: Celery Security Advisory +========================================= +:contact: security@celeryproject.org +:CVE id: TBA +:date: 2014-07-10 05:00:00 P.M UTC + +Details +======= + +:package: celery +:vulnerability: Environment error +:problem type: local +:risk: low +:versions-affected: 2.5, 3.0, 3.1 + +Description +=========== + +The built-in utility used to daemonize the Celery worker service sets +an insecure umask by default (umask 0). + +This means that any files or directories created by the worker will +end up having world-writable permissions. + +In practice this means that local users will be able to modify and possibly +corrupt the files created by user tasks. + +This is not immediately exploitable but can be if those files are later +evaluated as a program, for example a task that creates Python program files +that are later executed. + +Patches are now available for all maintained versions (see below), +and users are urged to upgrade, even if not directly +affected. + +Acknowledgements +================ + +Special thanks to Red Hat for originally discovering and reporting the issue. + +Systems affected +================ + +Users of Celery versions 3.0, and 3.1, except the recently +released 3.1.13, are affected if daemonizing the +Celery programs using the `--detach` argument or using the `celery multi` program +to start workers in the background, without setting a custom `--umask` +argument. + +Solution +======== + +NOTE: + Not all users of Celery will use it to create files, but if you do + then files may already have been created with insecure permissions. + + So after upgrading, or using the workaround, then please make sure + that files already created are not world writable. + +To work around the issue you can set a custom umask using the ``--umask`` +argument: + + $ celery worker -l info --detach --umask=16 # (022) + +Or you can upgrade to a more recent version: + +- Users of the 3.1 series should upgrade to 3.1.13: + + * ``pip install -U celery``, or + * ``easy_install -U celery``, or + * http://pypi.python.org/pypi/celery/3.1.13 + +- Users of the 3.0 series should upgrade to 3.0.25: + + * ``pip install -U celery==3.0.25``, or + * ``easy_install -U celery==3.0.25``, or + * http://pypi.python.org/pypi/celery/3.0.25 + +Distribution package maintainers are urged to provide their users +with updated packages. + +Please direct questions to the celery-users mailing-list: +http://groups.google.com/group/celery-users/, + +or if you are planning to report a new security related issue we request that +you keep the information confidential by contacting +security@celeryproject.org instead. + +Thank you! From 953c9141f664eeec7a245af2a5559bf8b65d4faa Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Jul 2014 22:24:36 +0100 Subject: [PATCH 0254/1103] Preserve old umask parsing behavior so that --umask starting with 0 is octal, otherwise decimal --- celery/platforms.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/platforms.py b/celery/platforms.py index 766a30e74..b9d39177c 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -296,7 +296,8 @@ def __init__(self, pidfile=None, workdir=None, umask=None, fake=False, after_chdir=None, after_forkers=True, **kwargs): if isinstance(umask, string_t): - umask = int(umask, 8) # convert str -> octal + # octal or decimal, depending on initial zero. + umask = int(umask, 8 if umask.startswith('0') else 10) self.workdir = workdir or DAEMON_WORKDIR self.umask = umask self.fake = fake From c4dc73dbb8e75c41eb955b091b2e8517e2df1ac3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Jul 2014 15:35:50 +0100 Subject: [PATCH 0255/1103] Improvements for #2134 (which fixes Issue #2076) --- celery/canvas.py | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index d0a824e01..0be4e7a96 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -397,7 +397,8 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, return result def prepare_steps(self, args, tasks, - root_id=None, link_error=None, app=None): + root_id=None, link_error=None, app=None, + from_dict=Signature.from_dict): app = app or self.app steps = deque(tasks) next_step = prev_task = prev_res = None @@ -405,6 +406,8 @@ def prepare_steps(self, args, tasks, i = 0 while steps: task = steps.popleft() + if not isinstance(task, Signature): + task = from_dict(task, app=app) if not i: # first task # first task gets partial args from chain task = task.clone(args) @@ -466,9 +469,6 @@ def apply(self, args=(), kwargs={}, **options): def from_dict(self, d, app=None): tasks = d['kwargs']['tasks'] if d['args'] and tasks: - # make sure that tasks are made into signatures (Issue #2076) - if not isinstance(tasks[0], Signature): - tasks[0] = signature(tasks[0]) # partial args passed on to first task in chain (Issue #1057). tasks[0]['args'] = tasks[0]._merge(d['args'])[0] return chain(*d['kwargs']['tasks'], app=app, **d['options']) @@ -590,17 +590,9 @@ def __init__(self, *tasks, **options): @classmethod def from_dict(self, d, app=None): - tasks = d['kwargs']['tasks'] - if d['args'] and tasks: - # partial args passed on to all tasks in the group (Issue #1057). - for task in tasks: - # make sure that tasks are made into signatures (Issue #2076) - if not isinstance(tasks[0], Signature): - task = signature(task) - task['args'] = task._merge(d['args'])[0] - return group(tasks, app=app, **d['options']) - - def _prepared(self, tasks, partial_args, group_id, root_id, dict=dict, + return group(d['kwargs']['tasks'], app=app, **d['options']) + + def _prepared(self, tasks, partial_args, group_id, root_id, app, dict=dict, Signature=Signature, from_dict=Signature.from_dict): for task in tasks: if isinstance(task, dict): @@ -610,11 +602,11 @@ def _prepared(self, tasks, partial_args, group_id, root_id, dict=dict, task = task.clone() else: # serialized sigs must be converted to Signature. - task = from_dict(task) + task = from_dict(task, app=app) if isinstance(task, group): # needs yield_from :( unroll = task._prepared( - task.tasks, partial_args, group_id, root_id, + task.tasks, partial_args, group_id, root_id, app, ) for taskN, resN in unroll: yield taskN, resN @@ -648,7 +640,7 @@ def apply_async(self, args=(), kwargs=None, add_to_parent=True, return self.freeze() options, group_id, root_id = self._freeze_gid(options) - tasks = self._prepared(self.tasks, args, group_id, root_id) + tasks = self._prepared(self.tasks, args, group_id, root_id, app) result = self.app.GroupResult( group_id, list(self._apply_tasks(tasks, producer, app, **options)), ) @@ -662,7 +654,7 @@ def apply(self, args=(), kwargs={}, **options): if not self.tasks: return self.freeze() # empty group returns GroupResult options, group_id, root_id = self._freeze_gid(options) - tasks = self._prepared(self.tasks, args, group_id, root_id) + tasks = self._prepared(self.tasks, args, group_id, root_id, app) return app.GroupResult(group_id, [ sig.apply(**options) for sig, _ in tasks ]) From 0cc391e452538c6963043eece1751d8bfd616d8e Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 14 Jul 2014 15:52:26 +0100 Subject: [PATCH 0256/1103] ghettomq is discontinued --- docs/userguide/routing.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/userguide/routing.rst b/docs/userguide/routing.rst index 2de3226cd..0656a8515 100644 --- a/docs/userguide/routing.rst +++ b/docs/userguide/routing.rst @@ -88,9 +88,9 @@ A queue named `"video"` will be created with the following settings: 'exchange_type': 'direct', 'routing_key': 'video'} -The non-AMQP backends like `ghettoq` does not support exchanges, so they -require the exchange to have the same name as the queue. Using this design -ensures it will work for them as well. +The non-AMQP backends like `Redis` or `Django-models` do not support exchanges, +so they require the exchange to have the same name as the queue. Using this +design ensures it will work for them as well. .. _routing-manual: From 0762057678ca27144c6a3ffb80dae8cd2c82c462 Mon Sep 17 00:00:00 2001 From: ocean1 Date: Wed, 23 Jul 2014 14:29:41 +0200 Subject: [PATCH 0257/1103] added groupmeta default --- celery/backends/mongodb.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 70af35aae..7b7b3b71e 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -20,7 +20,7 @@ from bson.binary import Binary except ImportError: # pragma: no cover from pymongo.binary import Binary # noqa - from pymongo.errors import InvalidDocument # noqa + from pymongo.errors import InvalidDocument # noqa else: # pragma: no cover Binary = None # noqa InvalidDocument = None # noqa @@ -39,13 +39,16 @@ __all__ = ['MongoBackend'] -BINARY_CODECS = frozenset(['pickle','msgpack']) +BINARY_CODECS = frozenset(['pickle', 'msgpack']) + +# register a fake bson serializer which will return the document as it is + -#register a fake bson serializer which will return the document as it is class bson_serializer(): + @staticmethod def loads(obj, *args, **kwargs): - if isinstance(obj,string_t): + if isinstance(obj, string_t): try: from anyjson import loads return loads(obj) @@ -58,10 +61,11 @@ def dumps(obj, *args, **kwargs): return obj register('bson', bson_serializer.loads, bson_serializer.dumps, - content_type='application/data', - content_encoding='utf-8') + content_type='application/data', + content_encoding='utf-8') + +disable_insecure_serializers(['json', 'bson']) -disable_insecure_serializers(['json','bson']) class Bunch(object): @@ -77,6 +81,7 @@ class MongoBackend(BaseBackend): password = None database_name = 'celery' taskmeta_collection = 'celery_taskmeta' + groupmeta_collection = 'celery_groupmeta' max_pool_size = 10 options = None @@ -123,7 +128,7 @@ def __init__(self, *args, **kwargs): 'taskmeta_collection', self.taskmeta_collection, ) self.groupmeta_collection = config.pop( - 'groupmeta_collection', self.taskmeta_collection, + 'groupmeta_collection', self.groupmeta_collection, ) self.options = dict(config, **config.pop('options', None) or {}) @@ -138,7 +143,6 @@ def __init__(self, *args, **kwargs): # Specifying backend as an URL self.host = url - def _get_connection(self): """Connect to the MongoDB server.""" if self._connection is None: @@ -172,7 +176,7 @@ def process_cleanup(self): def encode(self, data): payload = super(MongoBackend, self).encode(data) - #serializer which are in a unsupported format (pickle/binary) + # serializer which are in a unsupported format (pickle/binary) if self.serializer in BINARY_CODECS: payload = Binary(payload) @@ -213,7 +217,7 @@ def _get_task_meta_for(self, task_id): # if collection don't contain it try searching in the # group_collection it could be a groupresult instead obj = self.collection.find_one({'_id': task_id}) or \ - self.group_collection.find_one({'_id': task_id}) + self.group_collection.find_one({'_id': task_id}) if not obj: return {'status': states.PENDING, 'result': None} @@ -231,7 +235,7 @@ def _get_task_meta_for(self, task_id): def _save_group(self, group_id, result): """Save the group result.""" - task_ids = [ i.id for i in result ] + task_ids = [i.id for i in result] meta = {'_id': group_id, 'result': self.encode(task_ids), @@ -248,7 +252,7 @@ def _restore_group(self, group_id): tasks = self.decode(obj['result']) - tasks = [ AsyncResult(task) for task in tasks ] + tasks = [AsyncResult(task) for task in tasks] meta = { 'task_id': obj['_id'], From 3f0ce3004dcab0663e44c11b63f1549ea2ccfd2a Mon Sep 17 00:00:00 2001 From: Kevin McCarthy Date: Thu, 24 Jul 2014 09:21:49 -1000 Subject: [PATCH 0258/1103] Clarify the prefetch multiplier For years, I thought CELERYD_PREFETCH_MULTIPLIER turned off prefetching, and I had a lot of weird problems. Today I finally figured out 1 means disabled, 0 means PREFETCH ALL THE THINGS. This fixes the documentation bug. --- docs/configuration.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/configuration.rst b/docs/configuration.rst index 6a5e29a45..f237da89f 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -164,6 +164,12 @@ workers, note that the first worker to start will receive four times the number of messages initially. Thus the tasks may not be fairly distributed to the workers. +To disable prefetching, set CELERYD_PREFETCH_MULTIPLIER to 1. Setting +CELERYD_PREFETCH_MULTIPLIER to 0 will allow the worker to keep consuming +as many messages as it wants. + +For more on prefetching, read :ref:`optimizing-prefetch-limit` + .. note:: Tasks with ETA/countdown are not affected by prefetch limits. From 3751a06182fb6c95c5599b94669f40444d6a1a6b Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 30 Jul 2014 20:26:17 +0300 Subject: [PATCH 0259/1103] Added PyPy 3 to tox.ini. --- tox.ini | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tox.ini b/tox.ini index 9cfbd75ce..6b2e2ae10 100644 --- a/tox.ini +++ b/tox.ini @@ -4,6 +4,7 @@ envlist = 3.3, 3.4, pypy + pypy3 [testenv] sitepackages = False @@ -49,6 +50,17 @@ setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -U -r{toxinidir}/requirements/dev.txt nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + +[testenv:pypy3] +basepython = pypy3 +deps = -r{toxinidir}/requirements/default.txt +-r{toxinidir}/requirements/test.txt +-r{toxinidir}/requirements/test-ci.txt +-r{toxinidir}/requirements/dev.txt +setenv = C_DEBUG_TEST = 1 +commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} +pip install -U -r{toxinidir}/requirements/dev.txt +nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:docs] deps = -r{toxinidir}/requirements/docs.txt From 913640da8639ce875402d6b2348edd0330879b50 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 30 Jul 2014 20:28:40 +0300 Subject: [PATCH 0260/1103] Added PyPy 3 to the build matrix. Also, I removed the custom PPA since they are available by default. --- .travis.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index f0c96caa6..11179dc00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,15 +8,9 @@ env: - TOXENV=3.3 - TOXENV=3.4 - TOXENV=pypy + - TOXENV=pypy3 before_install: - | - if [[ $TOXENV = pypy ]]; then - deactivate - sudo apt-add-repository --yes ppa:pypy/ppa - sudo apt-get update - sudo apt-get install pypy - source ~/virtualenv/pypy/bin/activate - fi python --version uname -a lsb_release -a From d01214fdec63609153ef4ab1a8ca23d9b9c1ce6e Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 30 Jul 2014 20:43:20 +0300 Subject: [PATCH 0261/1103] Fixed copy/paste typos. --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index 6b2e2ae10..44e76316e 100644 --- a/tox.ini +++ b/tox.ini @@ -54,13 +54,13 @@ commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} [testenv:pypy3] basepython = pypy3 deps = -r{toxinidir}/requirements/default.txt --r{toxinidir}/requirements/test.txt --r{toxinidir}/requirements/test-ci.txt --r{toxinidir}/requirements/dev.txt + -r{toxinidir}/requirements/test.txt + -r{toxinidir}/requirements/test-ci.txt + -r{toxinidir}/requirements/dev.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} -pip install -U -r{toxinidir}/requirements/dev.txt -nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + pip install -U -r{toxinidir}/requirements/dev.txt + nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:docs] deps = -r{toxinidir}/requirements/docs.txt From 8c116df82a3ed289e13d6f5b871a23748c92419e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 31 Jul 2014 15:06:46 +0100 Subject: [PATCH 0262/1103] cert for test has expired --- celery/tests/security/test_certificate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/tests/security/test_certificate.py b/celery/tests/security/test_certificate.py index 4b07b5a98..f9678f947 100644 --- a/celery/tests/security/test_certificate.py +++ b/celery/tests/security/test_certificate.py @@ -6,7 +6,7 @@ from . import CERT1, CERT2, KEY1 from .case import SecurityCase -from celery.tests.case import Mock, mock_open, patch +from celery.tests.case import Mock, SkipTest, mock_open, patch class test_Certificate(SecurityCase): @@ -23,6 +23,7 @@ def test_invalid_certificate(self): self.assertRaises(SecurityError, Certificate, KEY1) def test_has_expired(self): + raise SkipTest('cert expired') self.assertFalse(Certificate(CERT1).has_expired()) From 4eed4c2a916a3b323f82d745dfb5fde1763291a9 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 31 Jul 2014 15:07:19 +0100 Subject: [PATCH 0263/1103] Worker: Changed loglevel for unrecoverable error to critical. --- celery/worker/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/worker/__init__.py b/celery/worker/__init__.py index 74c981d82..afd262b2d 100644 --- a/celery/worker/__init__.py +++ b/celery/worker/__init__.py @@ -209,7 +209,7 @@ def start(self): except WorkerTerminate: self.terminate() except Exception as exc: - logger.error('Unrecoverable error: %r', exc, exc_info=True) + logger.critical('Unrecoverable error: %r', exc, exc_info=True) self.stop(exitcode=EX_FAILURE) except SystemExit as exc: self.stop(exitcode=exc.code) From 4778c15d03ca189d9373dee6504e2a3075f6dbcf Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 31 Jul 2014 15:08:07 +0100 Subject: [PATCH 0264/1103] Improve rate limit accuracy --- celery/worker/consumer.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 8afa85b5b..7bf4576ca 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -182,6 +182,7 @@ def __init__(self, on_task_request, self._restart_state = restart_state(maxR=5, maxT=1) self._does_info = logger.isEnabledFor(logging.INFO) + self._limit_order = 0 self.on_task_request = on_task_request self.on_task_message = set() self.amqheartbeat_rate = self.app.conf.BROKER_HEARTBEAT_CHECKRATE @@ -252,11 +253,17 @@ def _update_qos_eventually(self, index): else self.qos.increment_eventually)( abs(index) * self.prefetch_multiplier) + def _limit_move_to_pool(self, request): + task_reserved(request) + self.on_task_request(request) + def _limit_task(self, request, bucket, tokens): if not bucket.can_consume(tokens): hold = bucket.expected_time(tokens) + pri = self._limit_order = (self._limit_order + 1) % 10 self.timer.call_after( - hold, self._limit_task, (request, bucket, tokens), + hold, self._limit_move_to_pool, (request, ), + priority=pri, ) else: task_reserved(request) From dea4eac28a09cda289b586d6bc66ed9515e9b8e4 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 31 Jul 2014 15:28:53 +0100 Subject: [PATCH 0265/1103] Tests passing --- celery/tests/worker/test_consumer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/celery/tests/worker/test_consumer.py b/celery/tests/worker/test_consumer.py index b9962a49c..db2d47eff 100644 --- a/celery/tests/worker/test_consumer.py +++ b/celery/tests/worker/test_consumer.py @@ -83,10 +83,13 @@ def test_limit_task(self): with patch('celery.worker.consumer.task_reserved') as reserved: bucket.can_consume.return_value = False bucket.expected_time.return_value = 3.33 + limit_order = c._limit_order c._limit_task(request, bucket, 4) + self.assertEqual(c._limit_order, limit_order + 1) bucket.can_consume.assert_called_with(4) c.timer.call_after.assert_called_with( - 3.33, c._limit_task, (request, bucket, 4), + 3.33, c._limit_move_to_pool, (request, ), + priority=c._limit_order, ) bucket.expected_time.assert_called_with(4) self.assertFalse(reserved.called) From e19b8898f6577608825b3c0cd63aa46b4ecb9598 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 31 Jul 2014 15:29:34 +0100 Subject: [PATCH 0266/1103] Fixes MongoDB result backend URL parsing problem. Closes celery/kombu#375 --- celery/backends/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/celery/backends/__init__.py b/celery/backends/__init__.py index 9b3d4c1ec..9870a58f1 100644 --- a/celery/backends/__init__.py +++ b/celery/backends/__init__.py @@ -10,7 +10,7 @@ import sys -from kombu.utils.url import _parse_url +from kombu.utils.url import urlparse from celery.local import Proxy from celery._state import current_app @@ -57,8 +57,9 @@ def get_backend_by_url(backend=None, loader=None): url = None if backend and '://' in backend: url = backend - if '+' in url[:url.index('://')]: + scheme, _, _ = url.partition('://') + if '+' in scheme: backend, url = url.split('+', 1) else: - backend, _, _, _, _, _, _ = _parse_url(url) + backend = scheme return get_backend_cls(backend, loader), url From bb4193b26b95e07dd9182b736809317b667df848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomek=20=C5=9Awi=C4=99cicki?= Date: Thu, 7 Aug 2014 16:09:02 +0200 Subject: [PATCH 0267/1103] Update configuration.rst I removed duplicates (BROKER_FAILOVER_STRATEGY) --- docs/configuration.rst | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 6a5e29a45..fc1e5b5b2 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -961,26 +961,6 @@ Example:: .. setting:: BROKER_TRANSPORT -BROKER_FAILOVER_STRATEGY -~~~~~~~~~~~~~~~~~~~~~~~~ - -Default failover strategy for the broker Connection object. If supplied, -may map to a key in 'kombu.connection.failover_strategies', or be a reference -to any method that yields a single item from a supplied list. - -Example:: - - # Random failover strategy - def random_failover_strategy(servers): - it = list(it) # don't modify callers list - shuffle = random.shuffle - for _ in repeat(None): - shuffle(it) - yield it[0] - - BROKER_FAILOVER_STRATEGY=random_failover_strategy - - BROKER_TRANSPORT ~~~~~~~~~~~~~~~~ :Aliases: ``BROKER_BACKEND`` From 2cdd450f63f919833156957d0549ab72cdbcc54a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tadej=20Jane=C5=BE?= Date: Fri, 8 Aug 2014 19:40:50 +0200 Subject: [PATCH 0268/1103] Docs: Improvements to example configuration file /etc/default/celeryd. --- docs/tutorials/daemonizing.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/daemonizing.rst b/docs/tutorials/daemonizing.rst index a6ad47ea3..776de1987 100644 --- a/docs/tutorials/daemonizing.rst +++ b/docs/tutorials/daemonizing.rst @@ -78,11 +78,13 @@ This is an example configuration for a Python project. .. code-block:: bash # Names of nodes to start - # most will only start one node: + # most people will only start one node: CELERYD_NODES="worker1" # but you can also start multiple and configure settings - # for each in CELERYD_OPTS (see `celery multi --help` for examples). - CELERYD_NODES="worker1 worker2 worker3" + # for each in CELERYD_OPTS (see `celery multi --help` for examples): + #CELERYD_NODES="worker1 worker2 worker3" + # alternatively, you can specify the number of nodes to start: + #CELERYD_NODES=10 # Absolute or relative path to the 'celery' command: CELERY_BIN="/usr/local/bin/celery" @@ -100,6 +102,9 @@ This is an example configuration for a Python project. # Extra command-line arguments to the worker CELERYD_OPTS="--time-limit=300 --concurrency=8" + # Set logging level to DEBUG + #CELERYD_LOG_LEVEL="DEBUG" + # %n will be replaced with the first part of the nodename. CELERYD_LOG_FILE="/var/log/celery/%n%I.log" CELERYD_PID_FILE="/var/run/celery/%n.pid" From ba7f500f6b3f79f8e023da5990cb4a55187d7594 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 8 Aug 2014 19:12:38 +0100 Subject: [PATCH 0269/1103] Django: Child processes must call django.setup if on Windows (Issue #2126) --- celery/fixups/django.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index f9ad33119..23e51bfc8 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -136,9 +136,11 @@ def __init__(self, app): def validate_models(self): import django try: - django.setup() + django_setup = django.setup except AttributeError: pass + else: + django_setup() s = io.StringIO() try: from django.core.management.validation import get_validation_errors @@ -166,6 +168,12 @@ def install(self): return self def on_worker_process_init(self, **kwargs): + # Child process must validate models again if on Windows, + # or if they were started using execv. + if os.environ.get('FORKED_BY_MULTIPROCESSING'): + self.validate_models() + + # close connections: # the parent process may have established these, # so need to close them. From ce2252d3a389b57e1630c38689aad4783129c8d5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 11 Aug 2014 15:24:21 +0100 Subject: [PATCH 0270/1103] Cosmetics --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 44e76316e..b5e7d9f30 100644 --- a/tox.ini +++ b/tox.ini @@ -50,7 +50,7 @@ setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -U -r{toxinidir}/requirements/dev.txt nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] - + [testenv:pypy3] basepython = pypy3 deps = -r{toxinidir}/requirements/default.txt @@ -60,7 +60,7 @@ deps = -r{toxinidir}/requirements/default.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -U -r{toxinidir}/requirements/dev.txt - nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:docs] deps = -r{toxinidir}/requirements/docs.txt From 4ae2f1d6310941aa1a66fac4ea5c4a2c517c437e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 11 Aug 2014 16:12:32 +0100 Subject: [PATCH 0271/1103] AMQP result backend: Call on_interval every second (always) to propagate parent errors --- celery/backends/amqp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index 62cf2034e..c443e232b 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -213,7 +213,10 @@ def callback(meta, message): # Total time spent may exceed a single call to wait() if timeout and now() - time_start >= timeout: raise socket.timeout() - wait(timeout=timeout) + try: + wait(timeout=1) + except socket.timeout: + pass if on_interval: on_interval() if results: # got event on the wanted channel. From 751ac8d7a153f53aa1f2c807fc48e320fce89cd1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 11 Aug 2014 18:33:30 +0100 Subject: [PATCH 0272/1103] Py3: tests should not depend on mock --- requirements/test3.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements/test3.txt b/requirements/test3.txt index f7ca6c013..f3c7e8e6f 100644 --- a/requirements/test3.txt +++ b/requirements/test3.txt @@ -1,3 +1 @@ nose -# FIXME required by kombu.tests.case -mock >=1.0.1 From 4eac177b88a54f2eb0943cf9e932feb904be0ef8 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 11 Aug 2014 18:34:50 +0100 Subject: [PATCH 0273/1103] Tox: pypy3 must use test3.txt, not test.txt --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b5e7d9f30..4a3a7cd6c 100644 --- a/tox.ini +++ b/tox.ini @@ -54,7 +54,7 @@ commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} [testenv:pypy3] basepython = pypy3 deps = -r{toxinidir}/requirements/default.txt - -r{toxinidir}/requirements/test.txt + -r{toxinidir}/requirements/test3.txt -r{toxinidir}/requirements/test-ci.txt -r{toxinidir}/requirements/dev.txt setenv = C_DEBUG_TEST = 1 From 6dcd2c3296e2f4795be00f538962cc5296c4eb65 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 13 Aug 2014 15:46:17 +0100 Subject: [PATCH 0274/1103] Task: Do not raise retry exception when retry executed in eager mode. Partially fixes #2164 --- celery/app/task.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index b65803b82..c21347822 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -574,15 +574,18 @@ def retry(self, args=None, kwargs=None, exc=None, throw=True, "Can't retry {0}[{1}] args:{2} kwargs:{3}".format( self.name, request.id, S.args, S.kwargs)) - # If task was executed eagerly using apply(), - # then the retry must also be executed eagerly. + ret = Retry(exc=exc, when=eta or countdown) + + if is_eager: + # if task was executed eagerly using apply(), + # then the retry must also be executed eagerly. + S.apply().get() + return ret + try: - S.apply().get() if is_eager else S.apply_async() + S.apply_async() except Exception as exc: - if is_eager: - raise raise Reject(exc, requeue=False) - ret = Retry(exc=exc, when=eta or countdown) if throw: raise ret return ret From 9c7e72f8f64e294f19f1a5776208f0af58b23410 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 13 Aug 2014 16:01:25 +0100 Subject: [PATCH 0275/1103] Fixes tests on py3 --- celery/tests/backends/test_riak.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/celery/tests/backends/test_riak.py b/celery/tests/backends/test_riak.py index 07d8bf802..b3323e35c 100644 --- a/celery/tests/backends/test_riak.py +++ b/celery/tests/backends/test_riak.py @@ -2,13 +2,12 @@ from __future__ import absolute_import, with_statement -from mock import MagicMock, Mock, patch, sentinel -from nose import SkipTest - from celery.backends import riak as module from celery.backends.riak import RiakBackend, riak from celery.exceptions import ImproperlyConfigured -from celery.tests.case import AppCase +from celery.tests.case import ( + AppCase, MagicMock, Mock, SkipTest, patch, sentinel, +) RIAK_BUCKET = 'riak_bucket' From c3adc32da6fc2a7af720316135d60f6ad4f4dd5a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 13 Aug 2014 17:20:25 +0100 Subject: [PATCH 0276/1103] Added logging example to redirect stdouts to custom logger in task --- docs/userguide/tasks.rst | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 91d4b4b8e..a048fd30f 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -376,9 +376,35 @@ for which documentation can be found in the :mod:`logging` module. You can also use :func:`print`, as anything written to standard -out/-err will be redirected to logging system (you can disable this, +out/-err will be redirected to the logging system (you can disable this, see :setting:`CELERY_REDIRECT_STDOUTS`). +.. note:: + + The worker will not update the redirection if you create a logger instance + somewhere in your task or task module. + + If you want to redirect ``sys.stdout`` and ``sys.stderr`` to a custom + logger you have to enable this manually, for example: + + .. code-block:: python + + import sys + + logger = get_task_logger(__name__) + + @app.task(bind=True) + def add(self, x, y): + old_outs = sys.stdout, sys.stderr + rlevel = self.app.conf.CELERY_REDIRECT_STDOUTS_LEVEL + try: + self.app.log.redirect_stdouts_to_logger(logger, rlevel) + print('Adding {0} + {1}'.format(x, y)) + return x + y + finally: + sys.stdout, sys.stderr = old_outs + + .. _task-retry: Retrying From a9725961d97e88b01e4b045bc4317172ea063d3f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 13 Aug 2014 18:18:54 +0100 Subject: [PATCH 0277/1103] Worker: autoreload should ignore not implemented pool.restart. Closes #2153 --- celery/worker/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/celery/worker/__init__.py b/celery/worker/__init__.py index afd262b2d..6f7cccc83 100644 --- a/celery/worker/__init__.py +++ b/celery/worker/__init__.py @@ -287,7 +287,10 @@ def reload(self, modules=None, reload=False, reloader=None): if self.consumer: self.consumer.update_strategies() self.consumer.reset_rate_limits() - self.pool.restart() + try: + self.pool.restart() + except NotImplementedError: + pass def info(self): return {'total': self.state.total_count, From 6ea6f24558df8037931345c1fbb299e947bfee46 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 14 Aug 2014 15:45:36 +0100 Subject: [PATCH 0278/1103] Sphinx extension resolved to celery.attr instead of app.attr, also faq referred to outdated celery.execute module --- celery/app/utils.py | 8 +++++- docs/_ext/celerydocs.py | 23 ++++++++++++++--- docs/faq.rst | 7 +++--- .../first-steps-with-celery.rst | 2 +- docs/history/changelog-3.0.rst | 20 +++++++-------- docs/userguide/application.rst | 25 ++++++++++--------- docs/userguide/extending.rst | 2 +- docs/userguide/tasks.rst | 6 ++--- docs/userguide/workers.rst | 2 +- docs/whatsnew-3.0.rst | 2 +- docs/whatsnew-3.1.rst | 6 ++--- 11 files changed, 63 insertions(+), 40 deletions(-) diff --git a/celery/app/utils.py b/celery/app/utils.py index f8bd9837f..62006c3e4 100644 --- a/celery/app/utils.py +++ b/celery/app/utils.py @@ -51,7 +51,13 @@ def appstr(app): class Settings(ConfigurationView): - """Celery settings object.""" + """Celery settings object. + + .. seealso: + + :ref:`configuration` for a full list of configuration keys. + + """ @property def CELERY_RESULT_BACKEND(self): diff --git a/docs/_ext/celerydocs.py b/docs/_ext/celerydocs.py index 2cbc97f60..e89462aa6 100644 --- a/docs/_ext/celerydocs.py +++ b/docs/_ext/celerydocs.py @@ -5,6 +5,7 @@ APPATTRS = { 'amqp': 'celery.app.amqp.AMQP', 'backend': 'celery.backends.base.BaseBackend', + 'conf': 'celery.app.utils.Settings', 'control': 'celery.app.control.Control', 'events': 'celery.events.Events', 'loader': 'celery.app.loaders.base.BaseLoader', @@ -13,15 +14,31 @@ 'tasks': 'celery.app.registry.Registry', 'AsyncResult': 'celery.result.AsyncResult', + 'ResultSet': 'celery.result.ResultSet', 'GroupResult': 'celery.result.GroupResult', 'Worker': 'celery.apps.worker.Worker', 'WorkController': 'celery.worker.WorkController', 'Beat': 'celery.apps.beat.Beat', 'Task': 'celery.app.task.Task', - 'send_task': 'celery.Celery.send_task', - 'connection': 'celery.Celery.connection', + 'signature': 'celery.canvas.Signature', } +APPDIRECT = { + 'on_configure', 'on_after_configure', 'on_after_finalize', + 'set_current', 'set_default', 'close', 'on_init', 'start', + 'worker_main', 'task', 'gen_task_name', 'finalize', + 'add_defaults', 'config_from_object', 'config_from_envvar', + 'config_from_cmdline', 'setup_security', 'autodiscover_tasks', + 'send_task', 'connection', 'connection_or_acquire', + 'producer_or_acquire', 'prepare_config', 'now', 'mail_admins', + 'select_queues', 'either', 'bugreport', 'create_task_cls', + 'subclass_with_self', 'annotations', 'current_task', 'oid', + 'timezone', '__reduce_keys__', 'fixups', 'finalized', 'configured', + 'autofinalize', 'steps', 'user_options', 'main', 'clock', +} + +APPATTRS.update({x: 'celery.Celery.{0}'.format(x) for x in APPDIRECT}) + ABBRS = { 'Celery': 'celery.Celery', } @@ -43,7 +60,7 @@ def shorten(S, newtarget, src_dict): return S[2:] elif S.startswith('@'): if src_dict is APPATTRS: - return '.'.join([pkg_of(newtarget), S[1:]]) + return '.'.join(['app', S[1:]]) return S[1:] return S diff --git a/docs/faq.rst b/docs/faq.rst index bf7cb9480..c0274ca2f 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -425,7 +425,7 @@ using the tasks current result backend. If you need to specify a custom result backend, or you want to use the current application's default backend you can use -:class:`@Celery.AsyncResult`: +:class:`@AsyncResult`: >>> result = app.AsyncResult(task_id) >>> result.get() @@ -600,12 +600,11 @@ queue for exchange, so that rejected messages is moved there. Can I call a task by name? ----------------------------- -**Answer**: Yes. Use :func:`celery.execute.send_task`. +**Answer**: Yes. Use :meth:`@send_task`. You can also call a task by name from any language that has an AMQP client. - >>> from celery.execute import send_task - >>> send_task("tasks.add", args=[2, 2], kwargs={}) + >>> app.send_task('tasks.add', args=[2, 2], kwargs={}) .. _faq-get-current-task-id: diff --git a/docs/getting-started/first-steps-with-celery.rst b/docs/getting-started/first-steps-with-celery.rst index 86ddf38bb..bb04f97d3 100644 --- a/docs/getting-started/first-steps-with-celery.rst +++ b/docs/getting-started/first-steps-with-celery.rst @@ -316,7 +316,7 @@ you can also imagine your SysAdmin making simple changes to the configuration in the event of system trouble. You can tell your Celery instance to use a configuration module, -by calling the :meth:`~@Celery.config_from_object` method: +by calling the :meth:`@config_from_object` method: .. code-block:: python diff --git a/docs/history/changelog-3.0.rst b/docs/history/changelog-3.0.rst index 76994ed37..25ee5cebb 100644 --- a/docs/history/changelog-3.0.rst +++ b/docs/history/changelog-3.0.rst @@ -353,7 +353,7 @@ If you're looking for versions prior to 3.0.x you should go to :ref:`history`. - The ``pool_restart`` remote control command now reports an error if the :setting:`CELERYD_POOL_RESTARTS` setting is not set. -- ``celery.conf.add_defaults`` can now be used with non-dict objects. +- :meth:`@add_defaults`` can now be used with non-dict objects. - Fixed compatibility problems in the Proxy class (Issue #1087). @@ -1108,7 +1108,7 @@ If you're looking for versions prior to 3.0.x you should go to :ref:`history`. - App instances now supports the with statement. - This calls the new :meth:`~celery.Celery.close` method at exit, which + This calls the new :meth:`@close` method at exit, which cleans up after the app like closing pool connections. Note that this is only necessary when dynamically creating apps, @@ -1411,16 +1411,16 @@ If you're looking for versions prior to 3.0.x you should go to :ref:`history`. } } -- New :meth:`@Celery.add_defaults` method can add new default configuration +- New :meth:`@add_defaults` method can add new default configuration dicts to the applications configuration. For example:: config = {'FOO': 10} - celery.add_defaults(config) + app.add_defaults(config) - is the same as ``celery.conf.update(config)`` except that data will not be + is the same as ``app.conf.update(config)`` except that data will not be copied, and that it will not be pickled when the worker spawns child processes. @@ -1429,7 +1429,7 @@ If you're looking for versions prior to 3.0.x you should go to :ref:`history`. def initialize_config(): # insert heavy stuff that can't be done at import time here. - celery.add_defaults(initialize_config) + app.add_defaults(initialize_config) which means the same as the above except that it will not happen until the celery configuration is actually used. @@ -1437,8 +1437,8 @@ If you're looking for versions prior to 3.0.x you should go to :ref:`history`. As an example, Celery can lazily use the configuration of a Flask app:: flask_app = Flask() - celery = Celery() - celery.add_defaults(lambda: flask_app.config) + app = Celery() + app.add_defaults(lambda: flask_app.config) - Revoked tasks were not marked as revoked in the result backend (Issue #871). @@ -1455,8 +1455,8 @@ If you're looking for versions prior to 3.0.x you should go to :ref:`history`. - New method names: - - ``Celery.default_connection()`` ➠ :meth:`~@Celery.connection_or_acquire`. - - ``Celery.default_producer()`` ➠ :meth:`~@Celery.producer_or_acquire`. + - ``Celery.default_connection()`` ➠ :meth:`~@connection_or_acquire`. + - ``Celery.default_producer()`` ➠ :meth:`~@producer_or_acquire`. The old names still work for backward compatibility. diff --git a/docs/userguide/application.rst b/docs/userguide/application.rst index 5cdc74fa4..a4653dfe5 100644 --- a/docs/userguide/application.rst +++ b/docs/userguide/application.rst @@ -67,7 +67,8 @@ This is only a problem in a limited set of use cases: #. If the module that the task is defined in is run as a program. #. If the application is created in the Python shell (REPL). -For example here, where the tasks module is also used to start a worker: +For example here, where the tasks module is also used to start a worker +with :meth:`@worker_main`: :file:`tasks.py`: @@ -114,7 +115,7 @@ There are several options you can set that will change how Celery works. These options can be set directly on the app instance, or you can use a dedicated configuration module. -The configuration is available as :attr:`@Celery.conf`:: +The configuration is available as :attr:`@conf`:: >>> app.conf.CELERY_TIMEZONE 'Europe/London' @@ -137,7 +138,7 @@ that are consulted in order: #. The configuration module (if any) #. The default configuration (:mod:`celery.app.defaults`). -You can even add new default sources by using the :meth:`@Celery.add_defaults` +You can even add new default sources by using the :meth:`@add_defaults` method. .. seealso:: @@ -148,13 +149,13 @@ method. ``config_from_object`` ---------------------- -The :meth:`@Celery.config_from_object` method loads configuration +The :meth:`@config_from_object` method loads configuration from a configuration object. This can be a configuration module, or any object with configuration attributes. Note that any configuration that was previous set will be reset when -:meth:`~@Celery.config_from_object` is called. If you want to set additional +:meth:`~@config_from_object` is called. If you want to set additional configuration you should do so after. Example 1: Using the name of a module @@ -216,7 +217,7 @@ Example 3: Using a configuration class/object ``config_from_envvar`` ---------------------- -The :meth:`@Celery.config_from_envvar` takes the configuration module name +The :meth:`@config_from_envvar` takes the configuration module name from an environment variable For example -- to load configuration from a module specified in the @@ -288,9 +289,9 @@ Creating a :class:`@Celery` instance will only do the following: #. Create the task registry. #. Set itself as the current app (but not if the ``set_as_current`` argument was disabled) - #. Call the :meth:`@Celery.on_init` callback (does nothing by default). + #. Call the :meth:`@on_init` callback (does nothing by default). -The :meth:`~@Celery.task` decorator does not actually create the +The :meth:`@task` decorator does not actually create the tasks at the point when it's called, instead it will defer the creation of the task to happen either when the task is used, or after the application has been *finalized*, @@ -317,7 +318,7 @@ you use the task, or access an attribute (in this case :meth:`repr`): True *Finalization* of the app happens either explicitly by calling -:meth:`@Celery.finalize` -- or implicitly by accessing the :attr:`~@Celery.tasks` +:meth:`@finalize` -- or implicitly by accessing the :attr:`@tasks` attribute. Finalizing the object will: @@ -464,8 +465,8 @@ chain breaks: Abstract Tasks ============== -All tasks created using the :meth:`~@Celery.task` decorator -will inherit from the applications base :attr:`~@Celery.Task` class. +All tasks created using the :meth:`~@task` decorator +will inherit from the applications base :attr:`~@Task` class. You can specify a different base class with the ``base`` argument: @@ -504,7 +505,7 @@ Once a task is bound to an app it will read configuration to set default values and so on. It's also possible to change the default base class for an application -by changing its :meth:`@Celery.Task` attribute: +by changing its :meth:`@Task` attribute: .. code-block:: python diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index 50ee1a514..3d64dc0ed 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -613,7 +613,7 @@ Command-specific options ~~~~~~~~~~~~~~~~~~~~~~~~ You can add additional command-line options to the ``worker``, ``beat`` and -``events`` commands by modifying the :attr:`~@Celery.user_options` attribute of the +``events`` commands by modifying the :attr:`~@user_options` attribute of the application instance. Celery commands uses the :mod:`optparse` module to parse command-line diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index a048fd30f..6342790bb 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -45,7 +45,7 @@ Basics ====== You can easily create a task from any callable by using -the :meth:`~@Celery.task` decorator: +the :meth:`~@task` decorator: .. code-block:: python @@ -240,7 +240,7 @@ like `moduleA.tasks.taskA`, `moduleA.tasks.taskB`, `moduleB.tasks.test` and so on. You may want to get rid of having `tasks` in all task names. As pointed above, you can explicitly give names for all tasks, or you can change the automatic naming behavior by overriding -:meth:`~@Celery.gen_task_name`. Continuing with the example, `celery.py` +:meth:`@gen_task_name`. Continuing with the example, `celery.py` may contain: .. code-block:: python @@ -261,7 +261,7 @@ So each task will have a name like `moduleA.taskA`, `moduleA.taskB` and .. warning:: - Make sure that your `gen_task_name` is a pure function, which means + Make sure that your :meth:`@gen_task_name` is a pure function, which means that for the same input it must always return the same output. .. _task-request-info: diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index cbfe81fb9..8d1eb2d86 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -810,7 +810,7 @@ Inspecting workers uses remote control commands under the hood. You can also use the ``celery`` command to inspect workers, -and it supports the same commands as the :class:`@Celery.control` interface. +and it supports the same commands as the :class:`@control` interface. .. code-block:: python diff --git a/docs/whatsnew-3.0.rst b/docs/whatsnew-3.0.rst index 8c2f83102..abadd7182 100644 --- a/docs/whatsnew-3.0.rst +++ b/docs/whatsnew-3.0.rst @@ -656,7 +656,7 @@ decorator executes inline like before, however for custom apps the @task decorator now returns a special PromiseProxy object that is only evaluated on access. -All promises will be evaluated when `app.finalize` is called, or implicitly +All promises will be evaluated when :meth:`@finalize` is called, or implicitly when the task registry is first used. diff --git a/docs/whatsnew-3.1.rst b/docs/whatsnew-3.1.rst index 49a75b511..62b16aa02 100644 --- a/docs/whatsnew-3.1.rst +++ b/docs/whatsnew-3.1.rst @@ -716,7 +716,7 @@ In Other News - New ability to specify additional command line options to the worker and beat programs. - The :attr:`@Celery.user_options` attribute can be used + The :attr:`@user_options` attribute can be used to add additional command-line arguments, and expects optparse-style options: @@ -1053,7 +1053,7 @@ In Other News This is the mapping of parsed command line arguments, and can be used to prepare new preload arguments (``app.user_options['preload']``). -- New callback: ``Celery.on_configure``. +- New callback: :meth:`@on_configure`. This callback is called when an app is about to be configured (a configuration key is required). @@ -1264,7 +1264,7 @@ Internal changes This removes a lot of duplicate functionality. - The ``Celery.with_default_connection`` method has been removed in favor - of ``with app.connection_or_acquire``. + of ``with app.connection_or_acquire`` (:meth:`@connection_or_acquire`) - The ``celery.results.BaseDictBackend`` class has been removed and is replaced by :class:`celery.results.BaseBackend`. From 1bcbca1a1159a7c00e022637c7efa6b27eeea454 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 15 Aug 2014 17:16:44 +0100 Subject: [PATCH 0279/1103] CELERYSA-02 contains invalid workaround advice --- docs/sec/CELERYSA-0002.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sec/CELERYSA-0002.txt b/docs/sec/CELERYSA-0002.txt index dd600b018..7938da59c 100644 --- a/docs/sec/CELERYSA-0002.txt +++ b/docs/sec/CELERYSA-0002.txt @@ -61,7 +61,7 @@ NOTE: To work around the issue you can set a custom umask using the ``--umask`` argument: - $ celery worker -l info --detach --umask=16 # (022) + $ celery worker -l info --detach --umask=18 # (022) Or you can upgrade to a more recent version: From 7ab2634341947b2ce4b25fb4abcf0d5e808447e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tadej=20Jane=C5=BE?= Date: Mon, 18 Aug 2014 16:14:19 +0200 Subject: [PATCH 0280/1103] Added myself as a contributor, based on 2cdd450. --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index a28946434..8a4f6b7f4 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -163,3 +163,4 @@ Brian Bouterse, 2014/04/10 Dmitry Malinovsky, 2014/04/28 Luke Pomfrey, 2014/05/06 Alexey Kotlyarov, 2014/05/16 +Tadej Janež, 2014/08/08 From 58ce751a5e6baed481f227469bbb6757579b9260 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Aug 2014 12:16:42 +0100 Subject: [PATCH 0281/1103] Cosmetics --- celery/concurrency/asynpool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 4a024d508..759861a21 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -1089,7 +1089,7 @@ def process_flush_queues(self, proc): all tasks that have not been started will be discarded. In Celery this is called whenever the transport connection is lost - (consumer restart). + (consumer restart), and when a process is terminated. """ resq = proc.outq._reader From a7e44097c0e54e11c4366c4329ef25ec2ec5f183 Mon Sep 17 00:00:00 2001 From: Balthazar Rouberol Date: Tue, 19 Aug 2014 10:33:12 +0200 Subject: [PATCH 0282/1103] Add side note about %h macro escaping in a supervisord config file Signed-off-by: Balthazar Rouberol --- docs/userguide/workers.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index 8d1eb2d86..953053fec 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -55,6 +55,9 @@ these will expand to: - ``worker1.%n`` -> ``worker1.george`` - ``worker1.%d`` -> ``worker1.example.com`` +.. note:: If you're storing the worker invokation command in a supervisord configuration file, the ``%`` sign must be escaped by adding a second one. + Thus, ``%h`` must be written ``%%h`` + .. _worker-stopping: Stopping the worker From d0a0b2eb8a35251c71a6509c404f0efd7022e6ea Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Aug 2014 12:18:40 +0100 Subject: [PATCH 0283/1103] Cosmetics for #2201 --- docs/userguide/workers.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index 953053fec..a0ad3cdfd 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -55,8 +55,9 @@ these will expand to: - ``worker1.%n`` -> ``worker1.george`` - ``worker1.%d`` -> ``worker1.example.com`` -.. note:: If you're storing the worker invokation command in a supervisord configuration file, the ``%`` sign must be escaped by adding a second one. - Thus, ``%h`` must be written ``%%h`` +.. admonition:: Note for :program:`supervisord` users. + + The ``%`` sign must be escaped by adding a second one: `%%h`. .. _worker-stopping: From f4b389d486cf5de9d2cf7430abdda01bf04a7aa0 Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Sat, 16 Aug 2014 08:00:46 +0400 Subject: [PATCH 0284/1103] Fix task scheduler internal heap building - use of absolute timestamps instead of delays --- celery/beat.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/celery/beat.py b/celery/beat.py index 972579759..04368e496 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -222,18 +222,25 @@ def tick(self, event_t=event_t, min=min, heapify=heapq.heapify): """Run a tick, that is one iteration of the scheduler. - Executes all due tasks. + Executes one due task per call. + Returns preferred delay in seconds for next call. """ + + def _when(entry, next_time_to_run): + return (time.mktime(entry.schedule.now().timetuple()) + + (adjust(next_time_to_run) or 0)) + adjust = self.adjust max_interval = self.max_interval H = self._heap if H is None: - H = self._heap = [event_t(adjust(e.is_due()[1]) or 0, 5, e) + H = self._heap = [event_t(_when(e, e.is_due()[1]) or 0, 5, e) for e in values(self.schedule)] heapify(H) if not H: return max_interval + event = H[0] entry = event[2] is_due, next_time_to_run = self.is_due(entry) @@ -242,7 +249,7 @@ def tick(self, event_t=event_t, min=min, if verify is event: next_entry = self.reserve(entry) self.apply_entry(entry, producer=self.producer) - heappush(H, event_t(next_time_to_run, event[1], next_entry)) + heappush(H, event_t(_when(next_entry, next_time_to_run), event[1], next_entry)) return 0 else: heappush(H, verify) From 922c98c4442eabb323469d7e5ce24dc6bda286b3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Aug 2014 12:22:20 +0100 Subject: [PATCH 0285/1103] Cosmetics for #2196 --- celery/beat.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/celery/beat.py b/celery/beat.py index 04368e496..aaa4df74a 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -219,7 +219,7 @@ def is_due(self, entry): def tick(self, event_t=event_t, min=min, heappop=heapq.heappop, heappush=heapq.heappush, - heapify=heapq.heapify): + heapify=heapq.heapify, mktime=time.mktime): """Run a tick, that is one iteration of the scheduler. Executes one due task per call. @@ -228,7 +228,7 @@ def tick(self, event_t=event_t, min=min, """ def _when(entry, next_time_to_run): - return (time.mktime(entry.schedule.now().timetuple()) + return (mktime(entry.schedule.now().timetuple()) + (adjust(next_time_to_run) or 0)) adjust = self.adjust @@ -249,7 +249,8 @@ def _when(entry, next_time_to_run): if verify is event: next_entry = self.reserve(entry) self.apply_entry(entry, producer=self.producer) - heappush(H, event_t(_when(next_entry, next_time_to_run), event[1], next_entry)) + heappush(H, event_t(_when(next_entry, next_time_to_run), + event[1], next_entry)) return 0 else: heappush(H, verify) From 6ef873c8cf3db179ef4c69b3db1bfc9b992035d3 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 5 Aug 2014 18:41:51 -0300 Subject: [PATCH 0286/1103] Fix typos on FAQ doc (misuses of doesn't -> don't) --- docs/faq.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index c0274ca2f..f46477490 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -99,7 +99,7 @@ many performance and stability improvements. It is an eventual goal that these improvements will be merged back into Python one day. It is also used for compatibility with older Python versions -that doesn't come with the multiprocessing module. +that don't come with the multiprocessing module. .. _`billiard`: http://pypi.python.org/pypi/billiard @@ -587,7 +587,7 @@ Why do workers delete tasks from the queue if they are unable to process them? **Answer**: The worker rejects unknown tasks, messages with encoding errors and messages -that doesn't contain the proper fields (as per the task message protocol). +that don't contain the proper fields (as per the task message protocol). If it did not reject them they could be redelivered again and again, causing a loop. From 68dd435fd56579bace3234e78161350fa2c9b987 Mon Sep 17 00:00:00 2001 From: Jasper Bryant-Greene Date: Tue, 15 Jul 2014 20:46:58 +1200 Subject: [PATCH 0287/1103] Pass None for now and timezone arguments which were missing. (Perhaps this method call should use kwargs to avoid this happening again?) --- celery/app/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/base.py b/celery/app/base.py index f823ae4bd..206ca102a 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -373,7 +373,7 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, expires, retries, chord, maybe_list(link), maybe_list(link_error), reply_to or self.oid, time_limit, soft_time_limit, - self.conf.CELERY_SEND_TASK_SENT_EVENT, + self.conf.CELERY_SEND_TASK_SENT_EVENT, None, None, root_id, parent_id, ) From 5328d3e2cdab933e9a0334ab16acac4765d0dfcb Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Aug 2014 12:38:03 +0100 Subject: [PATCH 0288/1103] Cosmetics for #2148 --- celery/app/amqp.py | 8 ++++---- celery/app/base.py | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 7fc59c43f..108e707ac 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -269,8 +269,8 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, expires=None, retries=0, chord=None, callbacks=None, errbacks=None, reply_to=None, time_limit=None, soft_time_limit=None, - create_sent_event=False, now=None, timezone=None, - root_id=None, parent_id=None): + create_sent_event=False, root_id=None, parent_id=None, + now=None, timezone=None): args = args or () kwargs = kwargs or {} utc = self.utc @@ -336,8 +336,8 @@ def as_task_v1(self, task_id, name, args=None, kwargs=None, expires=None, retries=0, chord=None, callbacks=None, errbacks=None, reply_to=None, time_limit=None, soft_time_limit=None, - create_sent_event=False, now=None, timezone=None, - root_id=None, parent_id=None): + create_sent_event=False, root_id=None, parent_id=None, + now=None, timezone=None): args = args or () kwargs = kwargs or {} utc = self.utc diff --git a/celery/app/base.py b/celery/app/base.py index 206ca102a..d5cc579da 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -373,8 +373,7 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, expires, retries, chord, maybe_list(link), maybe_list(link_error), reply_to or self.oid, time_limit, soft_time_limit, - self.conf.CELERY_SEND_TASK_SENT_EVENT, None, None, - root_id, parent_id, + self.conf.CELERY_SEND_TASK_SENT_EVENT, root_id, parent_id, ) if connection: From 28519beb577226c044d7469f155800f15fe23b0f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Aug 2014 12:46:17 +0100 Subject: [PATCH 0289/1103] CI: disable import_all_modules used for coverage on pypy3 --- celery/tests/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/celery/tests/__init__.py b/celery/tests/__init__.py index 966787270..24fc92c78 100644 --- a/celery/tests/__init__.py +++ b/celery/tests/__init__.py @@ -7,6 +7,8 @@ from importlib import import_module +PYPY3 = getattr(sys, 'pypy_version_info', None) and sys.version_info[0] > 3 + try: WindowsError = WindowsError # noqa except NameError: @@ -16,13 +18,16 @@ class WindowsError(Exception): def setup(): + using_coverage = ( + os.environ.get('COVER_ALL_MODULES') or '--with-coverage' in sys.argv + ) os.environ.update( # warn if config module not found C_WNOCONF='yes', KOMBU_DISABLE_LIMIT_PROTECTION='yes', ) - if os.environ.get('COVER_ALL_MODULES') or '--with-coverage' in sys.argv: + if using_coverage and not PYPY3: from warnings import catch_warnings with catch_warnings(record=True): import_all_modules() From a32debac00fe0ea34acafd737b8ab00345ca3998 Mon Sep 17 00:00:00 2001 From: mikemccabe Date: Mon, 14 Jul 2014 12:28:19 -0700 Subject: [PATCH 0290/1103] Apostrophe fix --- docs/getting-started/brokers/sqlalchemy.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started/brokers/sqlalchemy.rst b/docs/getting-started/brokers/sqlalchemy.rst index 0f8cb7b6a..47f9b96d0 100644 --- a/docs/getting-started/brokers/sqlalchemy.rst +++ b/docs/getting-started/brokers/sqlalchemy.rst @@ -32,7 +32,7 @@ an SQLAlchemy database URI. Please see `SQLAlchemy: Supported Databases`_ for a table of supported databases. -Here's a list of examples using a selection of other `SQLAlchemy Connection String`_'s: +Here's a list of examples using a selection of other `SQLAlchemy Connection Strings`_: .. code-block:: python @@ -51,7 +51,7 @@ Here's a list of examples using a selection of other `SQLAlchemy Connection Stri .. _`SQLAlchemy: Supported Databases`: http://www.sqlalchemy.org/docs/core/engines.html#supported-databases -.. _`SQLAlchemy Connection String`: +.. _`SQLAlchemy Connection Strings`: http://www.sqlalchemy.org/docs/core/engines.html#database-urls .. _sqlalchemy-results-configuration: From ac2918f6f49c9b642a90bd9ae7d6e3727d79e993 Mon Sep 17 00:00:00 2001 From: Ross Date: Fri, 11 Jul 2014 15:55:02 +0100 Subject: [PATCH 0291/1103] Update CONTRIBUTORS.txt :) Conflicts: CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 8a4f6b7f4..077c7719d 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -163,4 +163,5 @@ Brian Bouterse, 2014/04/10 Dmitry Malinovsky, 2014/04/28 Luke Pomfrey, 2014/05/06 Alexey Kotlyarov, 2014/05/16 +Ross Deane, 2014/07/11 Tadej Janež, 2014/08/08 From e28be9302fd71e37e54c2447c20f36688ec67d49 Mon Sep 17 00:00:00 2001 From: John Whitlock Date: Wed, 26 Mar 2014 11:06:11 -0500 Subject: [PATCH 0292/1103] Add /etc/init.d/celerybeat status Design copied from extra/generic-init.d/celeryd Tested on Ubuntu 12.04 LTS --- extra/generic-init.d/celerybeat | 35 ++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/extra/generic-init.d/celerybeat b/extra/generic-init.d/celerybeat index 00e8b35b3..c2e04c01b 100755 --- a/extra/generic-init.d/celerybeat +++ b/extra/generic-init.d/celerybeat @@ -247,6 +247,36 @@ start_beat () { } +check_status () { + local failed= + local pid_file=$CELERYBEAT_PID_FILE + if [ ! -e $pid_file ]; then + echo "${SCRIPT_NAME} is stopped: no pids were found" + failed=true + elif [ ! -r $pid_file ]; then + echo "${SCRIPT_NAME} is in unknown state, user cannot read pid file." + failed=true + else + local pid=`cat "$pid_file"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "${SCRIPT_NAME}: bad pid file ($pid_file)" + failed=true + else + local failed= + kill -0 $pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} (pid $pid) is stopped, but pid file exists!" + failed=true + else + echo "${SCRIPT_NAME} (pid $pid) is running..." + fi + fi + fi + + [ "$failed" ] && exit 1 || exit 0 +} + case "$1" in start) @@ -261,6 +291,9 @@ case "$1" in reload|force-reload) echo "Use start+stop" ;; + status) + check_status + ;; restart) echo "Restarting celery periodic task scheduler" check_paths @@ -277,7 +310,7 @@ case "$1" in check_paths ;; *) - echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|create-paths}" + echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|create-paths|status}" exit 64 # EX_USAGE ;; esac From 085af4bb31d6a8049061649bb179231777a1ad9b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Aug 2014 14:06:35 +0100 Subject: [PATCH 0293/1103] Multi: Now outputs to stdout instead of stderr for non-errors --- celery/bin/multi.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/celery/bin/multi.py b/celery/bin/multi.py index e61b19e7c..a7eb541d5 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -167,8 +167,10 @@ class MultiTool(object): retcode = 0 # Final exit code. def __init__(self, env=None, fh=None, quiet=False, verbose=False, - no_color=False, nosplash=False): - self.fh = fh or sys.stderr + no_color=False, nosplash=False, stdout=None, stderr=None): + """fh is an old alias to stdout.""" + self.stdout = self.fh = stdout or fh or sys.stdout + self.stderr = stderr or sys.stderr self.env = env self.nosplash = nosplash self.quiet = quiet @@ -213,8 +215,11 @@ def execute_from_commandline(self, argv, cmd='celery worker'): return self.retcode - def say(self, m, newline=True): - print(m, file=self.fh, end='\n' if newline else '') + def say(self, m, newline=True, file=None): + print(m, file=file or self.stdout, end='\n' if newline else '') + + def carp(self, m, newline=True, file=None): + return self.say(m, newline, file or self.stderr) def names(self, argv, cmd): p = NamespacedOptionParser(argv) @@ -424,7 +429,7 @@ def waitexec(self, argv, path=sys.executable): def error(self, msg=None): if msg: - self.say(msg) + self.carp(msg) self.usage() self.retcode = 1 return 1 From b63f9d4447b1c0a7b6229fd24ff1530a8db3a38f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Aug 2014 14:07:27 +0100 Subject: [PATCH 0294/1103] generic init: celeryd status now gets pidfiles using multi expand (Issue #1942) --- extra/generic-init.d/celeryd | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/extra/generic-init.d/celeryd b/extra/generic-init.d/celeryd index df918bca2..875f300f2 100755 --- a/extra/generic-init.d/celeryd +++ b/extra/generic-init.d/celeryd @@ -28,7 +28,7 @@ # # You can then configure this by manipulating /etc/default/little-worker. # -VERSION=10.0 +VERSION=10.1 echo "celery init v${VERSION}." if [ $(id -u) -ne 0 ]; then echo "Error: This program can only be used by the root user." @@ -199,15 +199,21 @@ create_paths() { export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" +_get_pidfiles () { + # note: multi < 3.1.14 output to stderr, not stdout, hence the redirect. + ${CELERYD_MULTI} expand "${CELERYD_PID_FILE}" ${CELERYD_NODES} 2>&1 +} + + _get_pids() { found_pids=0 my_exitcode=0 - for pid_file in "$CELERYD_PID_DIR"/*.pid; do - local pid=`cat "$pid_file"` + for pidfile in $(_get_pidfiles); do + local pid=`cat "$pidfile"` local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then - echo "bad pid file ($pid_file)" + echo "bad pid file ($pidfile)" one_failed=true my_exitcode=1 else @@ -267,6 +273,8 @@ kill_workers() { restart_workers_graceful () { + echo "WARNING: Use with caution in production" + echo "The workers will attempt to restart, but they may not be able to." local worker_pids= worker_pids=`_get_pids` [ "$one_failed" ] && exit 1 @@ -291,27 +299,27 @@ check_status () { found_pids=0 local one_failed= - for pid_file in "$CELERYD_PID_DIR"/*.pid; do - if [ ! -r $pid_file ]; then - echo "${SCRIPT_NAME} is stopped: no pids were found" + for pidfile in $(_get_pidfiles); do + if [ ! -r $pidfile ]; then + echo "${SCRIPT_NAME} down: no pidfiles found" one_failed=true break fi - local node=`basename "$pid_file" .pid` - local pid=`cat "$pid_file"` + local node=`basename "$pidfile" .pid` + local pid=`cat "$pidfile"` local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then - echo "bad pid file ($pid_file)" + echo "bad pid file ($pidfile)" one_failed=true else local failed= kill -0 $pid 2> /dev/null || failed=true if [ "$failed" ]; then - echo "${SCRIPT_NAME} (node $node) (pid $pid) is stopped, but pid file exists!" + echo "${SCRIPT_NAME} (node $node) (pid $pid) is down, but pidfile exists!" one_failed=true else - echo "${SCRIPT_NAME} (node $node) (pid $pid) is running..." + echo "${SCRIPT_NAME} (node $node) (pid $pid) is up..." fi fi done From 5ce97f1de36570a6b3c1eda84513cc6a8b3eb96e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Aug 2014 14:09:03 +0100 Subject: [PATCH 0295/1103] Bump celerybeat init script version and cosmetics for #1942 --- extra/generic-init.d/celerybeat | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extra/generic-init.d/celerybeat b/extra/generic-init.d/celerybeat index c2e04c01b..27f31111e 100755 --- a/extra/generic-init.d/celerybeat +++ b/extra/generic-init.d/celerybeat @@ -20,7 +20,7 @@ # Cannot use set -e/bash -e since the kill -0 command will abort # abnormally in the absence of a valid process ID. #set -e -VERSION=10.0 +VERSION=10.1 echo "celery init v${VERSION}." if [ $(id -u) -ne 0 ]; then @@ -251,7 +251,7 @@ check_status () { local failed= local pid_file=$CELERYBEAT_PID_FILE if [ ! -e $pid_file ]; then - echo "${SCRIPT_NAME} is stopped: no pids were found" + echo "${SCRIPT_NAME} is up: no pid file found" failed=true elif [ ! -r $pid_file ]; then echo "${SCRIPT_NAME} is in unknown state, user cannot read pid file." @@ -266,10 +266,10 @@ check_status () { local failed= kill -0 $pid 2> /dev/null || failed=true if [ "$failed" ]; then - echo "${SCRIPT_NAME} (pid $pid) is stopped, but pid file exists!" + echo "${SCRIPT_NAME} (pid $pid) is down, but pid file exists!" failed=true else - echo "${SCRIPT_NAME} (pid $pid) is running..." + echo "${SCRIPT_NAME} (pid $pid) is up..." fi fi fi From 5203d33a46b5621e025c29206a2c3e91359e3f44 Mon Sep 17 00:00:00 2001 From: Alexander Koshelev Date: Tue, 19 Aug 2014 17:26:32 +0400 Subject: [PATCH 0296/1103] Added @daevaorn to contributors --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 077c7719d..661a9ef4f 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -165,3 +165,4 @@ Luke Pomfrey, 2014/05/06 Alexey Kotlyarov, 2014/05/16 Ross Deane, 2014/07/11 Tadej Janež, 2014/08/08 +Akexander Koshelev, 2014/08/19 From 639b40f6308267312a1030bb3d6ac5805069510a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Aug 2014 14:30:17 +0100 Subject: [PATCH 0297/1103] Cosmetics for #1990 --- celery/backends/mongodb.py | 103 ++++++++++--------------------------- 1 file changed, 27 insertions(+), 76 deletions(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 7b7b3b71e..c05fe6cfa 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -28,44 +28,15 @@ from kombu.syn import detect_environment from kombu.utils import cached_property from kombu.exceptions import EncodeError -from kombu.serialization import register, disable_insecure_serializers from celery import states from celery.exceptions import ImproperlyConfigured from celery.five import string_t from celery.utils.timeutils import maybe_timedelta -from celery.result import AsyncResult from .base import BaseBackend __all__ = ['MongoBackend'] -BINARY_CODECS = frozenset(['pickle', 'msgpack']) - -# register a fake bson serializer which will return the document as it is - - -class bson_serializer(): - - @staticmethod - def loads(obj, *args, **kwargs): - if isinstance(obj, string_t): - try: - from anyjson import loads - return loads(obj) - except: - pass - return obj - - @staticmethod - def dumps(obj, *args, **kwargs): - return obj - -register('bson', bson_serializer.loads, bson_serializer.dumps, - content_type='application/data', - content_encoding='utf-8') - -disable_insecure_serializers(['json', 'bson']) - class Bunch(object): @@ -102,11 +73,6 @@ def __init__(self, *args, **kwargs): self.expires = kwargs.get('expires') or maybe_timedelta( self.app.conf.CELERY_TASK_RESULT_EXPIRES) - # little hack to get over standard kombu loads because - # mongo return strings which don't get decoded! - if self.serializer == 'bson': - self.decode = self.decode_bson - if not pymongo: raise ImproperlyConfigured( 'You need to install the pymongo library to use the ' @@ -175,21 +141,15 @@ def process_cleanup(self): self._connection = None def encode(self, data): - payload = super(MongoBackend, self).encode(data) - # serializer which are in a unsupported format (pickle/binary) - if self.serializer in BINARY_CODECS: - payload = Binary(payload) - - return payload - - def decode_bson(self, data): - return bson_serializer.loads(data) + if self.serializer == 'bson': + # mongodb handles serialization + return data + return super(MongoBackend, self).encode(data) - def encode_result(self, result, status): - if status in self.EXCEPTION_STATES and isinstance(result, Exception): - return self.prepare_exception(result) - else: - return self.prepare_value(result) + def decode(self, data): + if self.serializer == 'bson': + return data + return super(MongoBackend, self).decode(data) def _store_result(self, task_id, result, status, traceback=None, request=None, **kwargs): @@ -213,24 +173,20 @@ def _store_result(self, task_id, result, status, def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" - # if collection don't contain it try searching in the # group_collection it could be a groupresult instead obj = self.collection.find_one({'_id': task_id}) or \ self.group_collection.find_one({'_id': task_id}) - if not obj: - return {'status': states.PENDING, 'result': None} - - meta = { - 'task_id': obj['_id'], - 'status': obj['status'], - 'result': self.decode(obj['result']), - 'date_done': obj['date_done'], - 'traceback': self.decode(obj['traceback']), - 'children': self.decode(obj['children']), - } - - return meta + if obj: + return { + 'task_id': obj['_id'], + 'status': obj['status'], + 'result': self.decode(obj['result']), + 'date_done': obj['date_done'], + 'traceback': self.decode(obj['traceback']), + 'children': self.decode(obj['children']), + } + return {'status': states.PENDING, 'result': None} def _save_group(self, group_id, result): """Save the group result.""" @@ -247,20 +203,15 @@ def _save_group(self, group_id, result): def _restore_group(self, group_id): """Get the result for a group by id.""" obj = self.group_collection.find_one({'_id': group_id}) - if not obj: - return - - tasks = self.decode(obj['result']) - - tasks = [AsyncResult(task) for task in tasks] - - meta = { - 'task_id': obj['_id'], - 'result': tasks, - 'date_done': obj['date_done'], - } - - return meta + if obj: + tasks = [self.app.AsyncResult(task) + for task in self.decode(obj['result'])] + + return { + 'task_id': obj['_id'], + 'result': tasks, + 'date_done': obj['date_done'], + } def _delete_group(self, group_id): """Delete a group by id.""" From 181ad03abef827a6d29bb9d26d2211e357a5c102 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Aug 2014 14:34:20 +0100 Subject: [PATCH 0298/1103] Mongodb result backend: Should not return group results in _get_task_meta (re issue #1990) --- celery/backends/mongodb.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 735088a77..27f8fa3cc 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -172,10 +172,7 @@ def _store_result(self, task_id, result, status, def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" - # if collection don't contain it try searching in the - # group_collection it could be a groupresult instead - obj = self.collection.find_one({'_id': task_id}) or \ - self.group_collection.find_one({'_id': task_id}) + obj = self.collection.find_one({'_id': task_id}) if obj: return { 'task_id': obj['_id'], From 6bcd507e5f7f1fae68ea5603a174f54593c3388e Mon Sep 17 00:00:00 2001 From: Kouhei Maeda Date: Thu, 24 Apr 2014 10:49:57 +0900 Subject: [PATCH 0299/1103] Update calling.rst Fixed typo variable on example code of Connections. Conflicts: docs/userguide/calling.rst --- docs/userguide/calling.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index 92eb6eef5..bdaf94abb 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -447,7 +447,7 @@ Though this particular example is much better expressed as a group: >>> from celery import group >>> numbers = [(2, 2), (4, 4), (8, 8), (16, 16)] - >>> res = group(add.s(n) for n in numbers).apply_async() + >>> res = group(add.s(i) for i in numbers).apply_async() >>> res.get() [4, 8, 16, 32] From f31ddbe2ee69f59b327bad461de1b4117cc2a8ef Mon Sep 17 00:00:00 2001 From: John Whitlock Date: Tue, 19 Aug 2014 12:36:54 -0500 Subject: [PATCH 0300/1103] Add to contributors lists --- CONTRIBUTORS.txt | 1 + docs/AUTHORS.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 661a9ef4f..00db513ad 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -166,3 +166,4 @@ Alexey Kotlyarov, 2014/05/16 Ross Deane, 2014/07/11 Tadej Janež, 2014/08/08 Akexander Koshelev, 2014/08/19 +John Whitlock, 2014/08/19 diff --git a/docs/AUTHORS.txt b/docs/AUTHORS.txt index 8caea46a7..5c4f055db 100644 --- a/docs/AUTHORS.txt +++ b/docs/AUTHORS.txt @@ -69,6 +69,7 @@ Jeff Terrace Jerzy Kozera Jesper Noehr John Watson +John Whitlock Jonas Haag Jonas Obrist Jonatan Heyman From 2fddb876e1b966c3540aadbbc2257dfd493abb85 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 20 Aug 2014 15:08:39 +0100 Subject: [PATCH 0301/1103] Cosmetics --- docs/reference/celery.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/celery.rst b/docs/reference/celery.rst index ec7966fd4..d244e95e8 100644 --- a/docs/reference/celery.rst +++ b/docs/reference/celery.rst @@ -298,12 +298,12 @@ and creating Celery applications. .. attribute:: Celery.AsyncResult - Create new result instance. See :class:`~celery.result.AsyncResult`. + Create new result instance. See :class:`celery.result.AsyncResult`. .. attribute:: Celery.GroupResult Create new group result instance. - See :class:`~celery.result.GroupResult`. + See :class:`celery.result.GroupResult`. .. method:: Celery.worker_main(argv=None) From 5ca13990db03001eb0a2f61144f250162ed0c487 Mon Sep 17 00:00:00 2001 From: Marin Atanasov Nikolov Date: Fri, 22 Aug 2014 10:58:22 +0300 Subject: [PATCH 0302/1103] flake8 fixes --- celery/backends/__init__.py | 2 -- funtests/benchmarks/bench_worker.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/celery/backends/__init__.py b/celery/backends/__init__.py index 9870a58f1..83c570804 100644 --- a/celery/backends/__init__.py +++ b/celery/backends/__init__.py @@ -10,8 +10,6 @@ import sys -from kombu.utils.url import urlparse - from celery.local import Proxy from celery._state import current_app from celery.five import reraise diff --git a/funtests/benchmarks/bench_worker.py b/funtests/benchmarks/bench_worker.py index a85a7f1c2..77e743408 100644 --- a/funtests/benchmarks/bench_worker.py +++ b/funtests/benchmarks/bench_worker.py @@ -8,7 +8,7 @@ USE_FAST_LOCALS='yes', ) -from celery import Celery, group +from celery import Celery from celery.five import range from kombu.five import monotonic @@ -71,7 +71,6 @@ def bench_apply(n=DEFAULT_ITS): task = it._get_current_object() with app.producer_or_acquire() as producer: [task.apply_async((i, n), producer=producer) for i in range(n)] - #group(s(i, n) for i in range(n))() print('-- apply {0} tasks: {1}s'.format(n, monotonic() - time_start)) From cc371c47b27ba9508c5a8aa3e95988fa27ae8c81 Mon Sep 17 00:00:00 2001 From: ocean1 Date: Sun, 24 Aug 2014 16:44:18 +0200 Subject: [PATCH 0303/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 661a9ef4f..a5cb85846 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -166,3 +166,4 @@ Alexey Kotlyarov, 2014/05/16 Ross Deane, 2014/07/11 Tadej Janež, 2014/08/08 Akexander Koshelev, 2014/08/19 +Davide Quarta, 2014/08/19 From 768562ffd48859febd9a5b07329fa661136128a2 Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Sat, 30 Aug 2014 16:06:55 -0700 Subject: [PATCH 0304/1103] Fix failing test in test_multi The failure was introduced by a @ask's change on 8/19/14 in 085af4bb31d6a8049061649bb179231777a1ad9b: MultiTool.error calls carp instead of say. Fixed the test to reflect that change. --- celery/tests/bin/test_multi.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/celery/tests/bin/test_multi.py b/celery/tests/bin/test_multi.py index 76a6c1b64..653c8c126 100644 --- a/celery/tests/bin/test_multi.py +++ b/celery/tests/bin/test_multi.py @@ -176,15 +176,15 @@ def test_info_not_verbose(self): self.assertFalse(self.fh.getvalue()) def test_error(self): - self.t.say = Mock() + self.t.carp = Mock() self.t.usage = Mock() self.assertEqual(self.t.error('foo'), 1) - self.t.say.assert_called_with('foo') + self.t.carp.assert_called_with('foo') self.t.usage.assert_called_with() - self.t.say = Mock() + self.t.carp = Mock() self.assertEqual(self.t.error(), 1) - self.assertFalse(self.t.say.called) + self.assertFalse(self.t.carp.called) self.assertEqual(self.t.retcode, 1) From be0b620411a6adb7d05955d5f7411d16a01263d7 Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Sat, 30 Aug 2014 13:32:01 -0700 Subject: [PATCH 0305/1103] Fix issue #2225 Creating a chord no longer results in "TypeError: group object got multiple values for keyword argument 'task_id'". Chords now complete without hanging. --- celery/app/amqp.py | 1 + celery/backends/base.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 108e707ac..609dd53c2 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -371,6 +371,7 @@ def as_task_v1(self, task_id, name, args=None, kwargs=None, 'id': task_id, 'args': args, 'kwargs': kwargs, + 'group': group_id, 'retries': retries, 'eta': eta, 'expires': expires, diff --git a/celery/backends/base.py b/celery/backends/base.py index 1dd5ff1f1..17bf5a428 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -519,7 +519,11 @@ def _restore_group(self, group_id): def _apply_chord_incr(self, header, partial_args, group_id, body, result=None, options={}, **kwargs): self.save_group(group_id, self.app.GroupResult(group_id, result)) - return header(*partial_args, task_id=group_id, **options or {}) + + fixed_options = dict((k,v) for k,v in options.items() if k != 'task_id') + + return header(*partial_args, task_id=group_id, **fixed_options or {}) + def on_chord_part_return(self, task, state, result, propagate=None): if not self.implements_incr: From f7b29f637e1b83c6e756164d5396d8fdae882ab5 Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Sun, 31 Aug 2014 21:22:40 -0700 Subject: [PATCH 0306/1103] Fix issue mentioned in https://github.com/celery/celery/issues/1671 See the comment from @lance-burton on June 20, 2014. A nested group in an expression such as: c = (group(add.s(1,1),add.s(2,2)) | add.s(1) | add.s(1) | group(mul.s(1),mul.s(2))) res = c.apply_async().get() Causes an "AttributeError: 'dict' object has no attribute 'type'". --- celery/canvas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index 0be4e7a96..2c012f5ba 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -572,7 +572,7 @@ def _maybe_group(tasks): elif isinstance(tasks, Signature): tasks = [tasks] else: - tasks = regen(tasks) + tasks = map(signature, regen(tasks)) return tasks From 59ab502b4c3ef1c83fc716027e6b452cb9cd6280 Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Sun, 31 Aug 2014 21:57:53 -0700 Subject: [PATCH 0307/1103] Fix additional issue #2225 Earlier commit with the same title missed one of the cases causing the duplicate task_id argument error (i.e., when using AMQP). This commit addresses the issue. --- celery/backends/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 17bf5a428..1a586f423 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -353,7 +353,8 @@ def fallback_chord_unlock(self, group_id, body, result=None, def apply_chord(self, header, partial_args, group_id, body, options={}, **kwargs): - result = header(*partial_args, task_id=group_id, **options or {}) + fixed_options = dict((k,v) for k,v in options.items() if k!='task_id') + result = header(*partial_args, task_id=group_id, **fixed_options or {}) self.fallback_chord_unlock(group_id, body, **kwargs) return result From 43b970e148e45fcf68d55c7ce6951e4c0ea4e62b Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Mon, 1 Sep 2014 01:01:32 -0700 Subject: [PATCH 0308/1103] Fix issue #2228 Fixes the bug where the wrong result is returned when a chain contains a chord as the penultimate task. https://github.com/celery/celery/issues/2228 --- celery/canvas.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 2c012f5ba..ab24e6570 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -450,9 +450,9 @@ def prepare_steps(self, args, tasks, if link_error: task.set(link_error=link_error) - if not isinstance(prev_task, chord): - results.append(res) - tasks.append(task) + tasks.append(task) + results.append(res) + prev_task, prev_res = task, res return tasks, results From dc6a90d7a93182bd5143dcc3d587d034154e6b17 Mon Sep 17 00:00:00 2001 From: Gino Ledesma Date: Fri, 29 Aug 2014 11:03:08 -0700 Subject: [PATCH 0309/1103] Fixed MultiTool test case Updated the test case to match the change introduced in 085af4bb31d6a8049061649bb179231777a1ad9b. --- celery/tests/bin/test_multi.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/celery/tests/bin/test_multi.py b/celery/tests/bin/test_multi.py index 76a6c1b64..653c8c126 100644 --- a/celery/tests/bin/test_multi.py +++ b/celery/tests/bin/test_multi.py @@ -176,15 +176,15 @@ def test_info_not_verbose(self): self.assertFalse(self.fh.getvalue()) def test_error(self): - self.t.say = Mock() + self.t.carp = Mock() self.t.usage = Mock() self.assertEqual(self.t.error('foo'), 1) - self.t.say.assert_called_with('foo') + self.t.carp.assert_called_with('foo') self.t.usage.assert_called_with() - self.t.say = Mock() + self.t.carp = Mock() self.assertEqual(self.t.error(), 1) - self.assertFalse(self.t.say.called) + self.assertFalse(self.t.carp.called) self.assertEqual(self.t.retcode, 1) From 31b91f0618f34ab9c6161bfa48d3ab5b0ae27c08 Mon Sep 17 00:00:00 2001 From: Aaron Schumacher Date: Wed, 3 Sep 2014 09:46:07 -0400 Subject: [PATCH 0310/1103] some typos --- docs/getting-started/first-steps-with-celery.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/getting-started/first-steps-with-celery.rst b/docs/getting-started/first-steps-with-celery.rst index bb04f97d3..d02097ac8 100644 --- a/docs/getting-started/first-steps-with-celery.rst +++ b/docs/getting-started/first-steps-with-celery.rst @@ -16,7 +16,7 @@ In this tutorial you will learn the absolute basics of using Celery. You will learn about; - Choosing and installing a message transport (broker). -- Installing Celery and creating your first task +- Installing Celery and creating your first task. - Starting the worker and calling tasks. - Keeping track of tasks as they transition through different states, and inspecting return values. @@ -37,7 +37,7 @@ showcase Celery's capabilities. Choosing a Broker ================= -Celery requires a solution to send and receive messages, usually this +Celery requires a solution to send and receive messages; usually this comes in the form of a separate service called a *message broker*. There are several choices available, including: @@ -118,8 +118,8 @@ with standard Python tools like ``pip`` or ``easy_install``: Application =========== -The first thing you need is a Celery instance, this is called the celery -application or just app in short. Since this instance is used as +The first thing you need is a Celery instance, which is called the celery +application or just "app" for short. Since this instance is used as the entry-point for everything you want to do in Celery, like creating tasks and managing workers, it must be possible for other modules to import it. @@ -230,7 +230,7 @@ you choose to use a configuration module):: app = Celery('tasks', backend='amqp', broker='amqp://') -or if you want to use Redis as the result backend, but still use RabbitMQ as +Or if you want to use Redis as the result backend, but still use RabbitMQ as the message broker (a popular combination):: app = Celery('tasks', backend='redis://localhost', broker='amqp://') From 6c74fbff0adf5f2d6867cdb55b8c95b5bc589dc4 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 5 Sep 2014 14:20:50 +0100 Subject: [PATCH 0311/1103] Improves reference for CELERY_QUEUES. Closes #2231 --- docs/configuration.rst | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index cb372963d..1fd716b65 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -785,14 +785,21 @@ Message Routing CELERY_QUEUES ~~~~~~~~~~~~~ -The mapping of queues the worker consumes from. This is a dictionary -of queue name/options. See :ref:`guide-routing` for more information. +Most users will not want to specify this setting and should rather use +the :ref:`automatic routing facilities `. + +If you really want to configure advanced routing, this setting should +be a list of :class:`kombu.Queue` objects the worker will consume from. + +Note that workers can be overriden this setting via the `-Q` option, +or individual queues from this list (by name) can be excluded using +the `-X` option. + +Also see :ref:`routing-basics` for more information. The default is a queue/exchange/binding key of ``celery``, with exchange type ``direct``. -You don't have to care about this unless you want custom routing facilities. - .. setting:: CELERY_ROUTES CELERY_ROUTES From 2dc3de9dda03efeb04ed3c3de47ecb2518153ac7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 5 Sep 2014 14:28:35 +0100 Subject: [PATCH 0312/1103] Periodic task examples too easily confused --- docs/userguide/periodic-tasks.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/userguide/periodic-tasks.rst b/docs/userguide/periodic-tasks.rst index a81c45f1a..b539e1f71 100644 --- a/docs/userguide/periodic-tasks.rst +++ b/docs/userguide/periodic-tasks.rst @@ -220,20 +220,20 @@ The syntax of these crontab expressions are very flexible. Some examples: | ``crontab(minute=0, hour='*/3,8-17')`` | Execute every hour divisible by 3, and | | | every hour during office hours (8am-5pm). | +-----------------------------------------+--------------------------------------------+ -| ``crontab(day_of_month='2')`` | Execute on the second day of every month. | +| ``crontab(0, 0, 0, day_of_month='2')`` | Execute on the second day of every month. | | | | +-----------------------------------------+--------------------------------------------+ -| ``crontab(day_of_month='2-30/3')`` | Execute on every even numbered day. | -| | | +| ``crontab(0, 0, 0,'`` | Execute on every even numbered day. | +| ``day_of_month='2-30/3')`` | | +-----------------------------------------+--------------------------------------------+ -| ``crontab(day_of_month='1-7,15-21')`` | Execute on the first and third weeks of | -| | the month. | +| ``crontab(0, 0, 0,`` | Execute on the first and third weeks of | +| ``day_of_month='1-7,15-21')`` | the month. | +-----------------------------------------+--------------------------------------------+ -| ``crontab(day_of_month='11',`` | Execute on 11th of May every year. | -| ``month_of_year='5')`` | | +| ``crontab(0, 0, 0, day_of_month='11',`` | Execute on 11th of May every year. | +| ``month_of_year='5')`` | | +-----------------------------------------+--------------------------------------------+ -| ``crontab(month_of_year='*/3')`` | Execute on the first month of every | -| | quarter. | +| ``crontab(0, 0, 0,`` | Execute on the first month of every | +| ``month_of_year='*/3')`` | quarter. | +-----------------------------------------+--------------------------------------------+ See :class:`celery.schedules.crontab` for more documentation. From b42a844e916cfc24d97f5b073b8145ee1d6da37c Mon Sep 17 00:00:00 2001 From: Jonathan Sundqvist Date: Sun, 7 Sep 2014 20:23:27 +0200 Subject: [PATCH 0313/1103] Adding missing instruction on necessary path for RabbitMQ --- docs/getting-started/brokers/rabbitmq.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/getting-started/brokers/rabbitmq.rst b/docs/getting-started/brokers/rabbitmq.rst index 1df16560a..f599e756b 100644 --- a/docs/getting-started/brokers/rabbitmq.rst +++ b/docs/getting-started/brokers/rabbitmq.rst @@ -90,6 +90,12 @@ Finally, we can install rabbitmq using :program:`brew`: .. _rabbitmq-osx-system-hostname: +After you have installed rabbitmq with brew you need to add the following to your path to be able to start and stop the broker. Add it to your .bash_profile or .profile + +.. code-block:: bash + + `PATH=$PATH:/usr/local/sbin` + Configuring the system host name ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 9bc10151a43af51f8192c5e1c5534a0753aec91c Mon Sep 17 00:00:00 2001 From: dimlev Date: Mon, 8 Sep 2014 18:14:29 +0300 Subject: [PATCH 0314/1103] Fixed RCelery link --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 10b6d8684..7bffaab40 100644 --- a/README.rst +++ b/README.rst @@ -34,7 +34,7 @@ any language. So far there's RCelery_ for the Ruby programming language, and a `PHP client`, but language interoperability can also be achieved by using webhooks. -.. _RCelery: http://leapfrogdevelopment.github.com/rcelery/ +.. _RCelery: https://github.com/leapfrogonline/rcelery .. _`PHP client`: https://github.com/gjedeer/celery-php .. _`using webhooks`: http://docs.celeryproject.org/en/latest/userguide/remote-tasks.html From 28771544e14be477493037e156c45b7460f171ec Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 9 Sep 2014 12:01:24 +0100 Subject: [PATCH 0315/1103] Result: .get() should not call get_task_meta after receiving the result. Closes #2245 --- celery/backends/amqp.py | 11 ++------ celery/backends/base.py | 14 +++------- celery/result.py | 45 ++++++++++++++++++------------ celery/tests/backends/test_amqp.py | 11 ++++---- 4 files changed, 39 insertions(+), 42 deletions(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index c443e232b..96d8a3722 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -140,7 +140,7 @@ def store_result(self, task_id, result, status, def on_reply_declare(self, task_id): return [self._create_binding(task_id)] - def wait_for(self, task_id, timeout=None, cache=True, propagate=True, + def wait_for(self, task_id, timeout=None, cache=True, no_ack=True, on_interval=None, READY_STATES=states.READY_STATES, PROPAGATE_STATES=states.PROPAGATE_STATES, @@ -148,19 +148,14 @@ def wait_for(self, task_id, timeout=None, cache=True, propagate=True, cached_meta = self._cache.get(task_id) if cache and cached_meta and \ cached_meta['status'] in READY_STATES: - meta = cached_meta + return cached_meta else: try: - meta = self.consume(task_id, timeout=timeout, no_ack=no_ack, + return self.consume(task_id, timeout=timeout, no_ack=no_ack, on_interval=on_interval) except socket.timeout: raise TimeoutError('The operation timed out.') - if meta['status'] in PROPAGATE_STATES and propagate: - raise self.exception_to_python(meta['result']) - # consume() always returns READY_STATE. - return meta['result'] - def get_task_meta(self, task_id, backlog_limit=1000): # Polling and using basic_get with self.app.pool.acquire_channel(block=True) as (_, channel): diff --git a/celery/backends/base.py b/celery/backends/base.py index 1dd5ff1f1..ef833e55d 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -189,8 +189,7 @@ def decode(self, payload): accept=self.accept) def wait_for(self, task_id, - timeout=None, propagate=True, interval=0.5, no_ack=True, - on_interval=None): + timeout=None, interval=0.5, no_ack=True, on_interval=None): """Wait for task and return its result. If the task raises an exception, this exception @@ -205,14 +204,9 @@ def wait_for(self, task_id, time_elapsed = 0.0 while 1: - status = self.get_status(task_id) - if status == states.SUCCESS: - return self.get_result(task_id) - elif status in states.PROPAGATE_STATES: - result = self.get_result(task_id) - if propagate: - raise result - return result + meta = self.get_task_meta(task_id) + if meta['status'] in states.READY_STATES: + return meta if on_interval: on_interval() # avoid hammering the CPU checking status. diff --git a/celery/result.py b/celery/result.py index cab76bab4..cba8d86de 100644 --- a/celery/result.py +++ b/celery/result.py @@ -119,8 +119,10 @@ def revoke(self, connection=None, terminate=False, signal=None, terminate=terminate, signal=signal, reply=wait, timeout=timeout) - def get(self, timeout=None, propagate=True, interval=0.5, no_ack=True, - follow_parents=True): + def get(self, timeout=None, propagate=True, interval=0.5, + no_ack=True, follow_parents=True, + EXCEPTION_STATES=states.EXCEPTION_STATES, + PROPAGATE_STATES=states.PROPAGATE_STATES): """Wait until task is ready, and return its result. .. warning:: @@ -159,16 +161,21 @@ def get(self, timeout=None, propagate=True, interval=0.5, no_ack=True, self.maybe_reraise() return self.result - try: - return self.backend.wait_for( - self.id, timeout=timeout, - propagate=propagate, - interval=interval, - on_interval=on_interval, - no_ack=no_ack, - ) - finally: - self._get_task_meta() # update self._cache + meta = self.backend.wait_for( + self.id, timeout=timeout, + propagate=propagate, + interval=interval, + on_interval=on_interval, + no_ack=no_ack, + ) + if meta: + self._maybe_set_cache(meta) + status = meta['status'] + if status in EXCEPTION_STATES: + return self.backend.exception_to_python(meta['result']) + if status in PROPAGATE_STATES and propagate: + raise self.backend.exception_to_python(meta['result']) + return meta['result'] wait = get # deprecated alias to :meth:`get`. def _maybe_reraise_parent_error(self): @@ -322,14 +329,16 @@ def supports_native_join(self): def children(self): return self._get_task_meta().get('children') + def _maybe_set_cache(self, meta): + if meta: + state = meta['status'] + if state == states.SUCCESS or state in states.PROPAGATE_STATES: + return self._set_cache(meta) + return meta + def _get_task_meta(self): if self._cache is None: - meta = self.backend.get_task_meta(self.id) - if meta: - state = meta['status'] - if state == states.SUCCESS or state in states.PROPAGATE_STATES: - return self._set_cache(meta) - return meta + return self._maybe_set_cache(self.backend.get_task_meta(self.id)) return self._cache def _set_cache(self, d): diff --git a/celery/tests/backends/test_amqp.py b/celery/tests/backends/test_amqp.py index 37847a8f9..6ca5441de 100644 --- a/celery/tests/backends/test_amqp.py +++ b/celery/tests/backends/test_amqp.py @@ -234,15 +234,14 @@ def test_wait_for(self): with self.assertRaises(TimeoutError): b.wait_for(tid, timeout=0.1) b.store_result(tid, 42, states.SUCCESS) - self.assertEqual(b.wait_for(tid, timeout=1), 42) + self.assertEqual(b.wait_for(tid, timeout=1)['result'], 42) b.store_result(tid, 56, states.SUCCESS) - self.assertEqual(b.wait_for(tid, timeout=1), 42, + self.assertEqual(b.wait_for(tid, timeout=1)['result'], 42, 'result is cached') - self.assertEqual(b.wait_for(tid, timeout=1, cache=False), 56) + self.assertEqual(b.wait_for(tid, timeout=1, cache=False)['result'], 56) b.store_result(tid, KeyError('foo'), states.FAILURE) - with self.assertRaises(KeyError): - b.wait_for(tid, timeout=1, cache=False) - self.assertTrue(b.wait_for(tid, timeout=1, propagate=False)) + res = b.wait_for(tid, timeout=1, cache=False) + self.assertEqual(res['status'], states.FAILURE) b.store_result(tid, KeyError('foo'), states.PENDING) with self.assertRaises(TimeoutError): b.wait_for(tid, timeout=0.01, cache=False) From e4ca8dae43eaac79d2a2df367f6e093a898c55a2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 9 Sep 2014 13:39:54 +0100 Subject: [PATCH 0316/1103] Fixes problem with propagate in last commit --- celery/result.py | 1 - 1 file changed, 1 deletion(-) diff --git a/celery/result.py b/celery/result.py index cba8d86de..360f4abab 100644 --- a/celery/result.py +++ b/celery/result.py @@ -163,7 +163,6 @@ def get(self, timeout=None, propagate=True, interval=0.5, meta = self.backend.wait_for( self.id, timeout=timeout, - propagate=propagate, interval=interval, on_interval=on_interval, no_ack=no_ack, From 827c5b1ad68ff146819c02bfdd292802bbcccec3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 8 Sep 2014 16:27:39 +0100 Subject: [PATCH 0317/1103] Django: Always call django.setup() before task modules imported. Closes #2227 --- celery/fixups/django.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index 23e51bfc8..439921abf 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -45,6 +45,7 @@ class DjangoFixup(object): def __init__(self, app): self.app = app self.app.set_default() + self._worker_fixup = None def install(self): # Need to add project directory to path @@ -53,12 +54,20 @@ def install(self): self.app.loader.now = self.now self.app.loader.mail_admins = self.mail_admins + signals.import_modules.connect(self.on_import_modules) signals.worker_init.connect(self.on_worker_init) return self + @cached_property + def worker_fixup(self): + return DjangoWorkerFixup(self.app) + + def on_import_modules(self, **kwargs): + # call django.setup() before task modules are imported + self.worker_fixup.validate_models() + def on_worker_init(self, **kwargs): - # keep reference - self._worker_fixup = DjangoWorkerFixup(self.app).install() + self.worker_fixup.install() def now(self, utc=False): return datetime.utcnow() if utc else self._now() @@ -162,7 +171,6 @@ def install(self): signals.task_prerun.connect(self.on_task_prerun) signals.task_postrun.connect(self.on_task_postrun) signals.worker_process_init.connect(self.on_worker_process_init) - self.validate_models() self.close_database() self.close_cache() return self From 43eabfd4fac4e9a21546a3707a983cee75a0ed68 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 9 Sep 2014 13:43:06 +0100 Subject: [PATCH 0318/1103] Fixes tests --- celery/fixups/django.py | 4 +++- celery/result.py | 4 ++-- celery/tests/fixups/test_django.py | 4 +--- celery/tests/tasks/test_result.py | 7 +++++-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index 439921abf..c1ae62e21 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -60,7 +60,9 @@ def install(self): @cached_property def worker_fixup(self): - return DjangoWorkerFixup(self.app) + if self._worker_fixup is None: + self._worker_fixup = DjangoWorkerFixup(self.app) + return self._worker_fixup def on_import_modules(self, **kwargs): # call django.setup() before task modules are imported diff --git a/celery/result.py b/celery/result.py index 360f4abab..708fa69f4 100644 --- a/celery/result.py +++ b/celery/result.py @@ -170,10 +170,10 @@ def get(self, timeout=None, propagate=True, interval=0.5, if meta: self._maybe_set_cache(meta) status = meta['status'] - if status in EXCEPTION_STATES: - return self.backend.exception_to_python(meta['result']) if status in PROPAGATE_STATES and propagate: raise self.backend.exception_to_python(meta['result']) + if status in EXCEPTION_STATES: + return self.backend.exception_to_python(meta['result']) return meta['result'] wait = get # deprecated alias to :meth:`get`. diff --git a/celery/tests/fixups/test_django.py b/celery/tests/fixups/test_django.py index 1d4ec5cea..17990a6e8 100644 --- a/celery/tests/fixups/test_django.py +++ b/celery/tests/fixups/test_django.py @@ -93,9 +93,7 @@ def test_on_worker_init(self): f.on_worker_init() DWF.assert_called_with(f.app) DWF.return_value.install.assert_called_with() - self.assertIs( - f._worker_fixup, DWF.return_value.install.return_value, - ) + self.assertIs(f._worker_fixup, DWF.return_value) class test_DjangoWorkerFixup(FixupCase): diff --git a/celery/tests/tasks/test_result.py b/celery/tests/tasks/test_result.py index 559ce78fd..a92b22448 100644 --- a/celery/tests/tasks/test_result.py +++ b/celery/tests/tasks/test_result.py @@ -74,15 +74,18 @@ def test_children(self): def test_propagates_for_parent(self): x = self.app.AsyncResult(uuid()) - x.backend = Mock() + x.backend = Mock(name='backend') x.backend.get_task_meta.return_value = {} + x.backend.wait_for.return_value = { + 'status': states.SUCCESS, 'result': 84, + } x.parent = EagerResult(uuid(), KeyError('foo'), states.FAILURE) with self.assertRaises(KeyError): x.get(propagate=True) self.assertFalse(x.backend.wait_for.called) x.parent = EagerResult(uuid(), 42, states.SUCCESS) - x.get(propagate=True) + self.assertEqual(x.get(propagate=True), 84) self.assertTrue(x.backend.wait_for.called) def test_get_children(self): From 0ed72be050935e2793943d62a5db693e56e6ea4a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 9 Sep 2014 14:21:01 +0100 Subject: [PATCH 0319/1103] Fixes mongodb tests. Closes #1964 --- celery/tests/backends/test_mongodb.py | 50 +++++++++++++-------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index f3449f793..46628b05d 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -1,27 +1,28 @@ from __future__ import absolute_import import datetime -import uuid from pickle import loads, dumps +from celery import uuid from celery import states from celery.backends import mongodb as module from celery.backends.mongodb import MongoBackend, Bunch, pymongo from celery.exceptions import ImproperlyConfigured from celery.tests.case import ( AppCase, MagicMock, Mock, SkipTest, ANY, - depends_on_current_app, patch, sentinel, + call, depends_on_current_app, patch, sentinel, ) COLLECTION = 'taskmeta_celery' -TASK_ID = str(uuid.uuid1()) +TASK_ID = uuid() MONGODB_HOST = 'localhost' MONGODB_PORT = 27017 MONGODB_USER = 'mongo' MONGODB_PASSWORD = '1234' MONGODB_DATABASE = 'testing' MONGODB_COLLECTION = 'collection1' +MONGODB_GROUP_COLLECTION = 'group_collection1' class test_MongoBackend(AppCase): @@ -66,13 +67,6 @@ def test_init_settings_is_None(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None MongoBackend(app=self.app) - def test_restore_group_no_entry(self): - x = MongoBackend(app=self.app) - x.collection = Mock() - fo = x.collection.find_one = Mock() - fo.return_value = None - self.assertIsNone(x._restore_group('1f3fab')) - @depends_on_current_app def test_reduce(self): x = MongoBackend(app=self.app) @@ -220,29 +214,36 @@ def test_get_task_meta_for_no_result(self, mock_get_database): @patch('celery.backends.mongodb.MongoBackend._get_database') def test_save_group(self, mock_get_database): - self.backend.taskmeta_collection = MONGODB_COLLECTION + self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection - + res = [self.app.AsyncResult(i) for i in range(3)] ret_val = self.backend._save_group( - sentinel.taskset_id, sentinel.result) - + sentinel.taskset_id, res, + ) mock_get_database.assert_called_once_with() - mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) + mock_database.__getitem__.assert_called_once_with( + MONGODB_GROUP_COLLECTION, + ) mock_collection.save.assert_called_once_with(ANY) - self.assertEqual(sentinel.result, ret_val) + self.assertEqual(res, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_restore_group(self, mock_get_database): - self.backend.taskmeta_collection = MONGODB_COLLECTION + self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() - mock_collection.find_one.return_value = MagicMock() + mock_collection.find_one.return_value = { + '_id': sentinel.taskset_id, + 'result': [uuid(), uuid()], + 'date_done': 1, + } + self.backend.decode.side_effect = lambda r: r mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection @@ -250,12 +251,11 @@ def test_restore_group(self, mock_get_database): ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() - mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.find_one.assert_called_once_with( {'_id': sentinel.taskset_id}) self.assertEqual( - ['date_done', 'result', 'task_id'], - list(ret_val.keys()), + list(sorted(['date_done', 'result', 'task_id'])), + list(sorted(ret_val.keys())), ) @patch('celery.backends.mongodb.MongoBackend._get_database') @@ -271,7 +271,6 @@ def test_delete_group(self, mock_get_database): self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() - mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {'_id': sentinel.taskset_id}) @@ -297,19 +296,20 @@ def test_forget(self, mock_get_database): def test_cleanup(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION + self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION - mock_database = MagicMock(spec=['__getitem__', '__setitem__']) + mock_database = Mock(spec=['__getitem__', '__setitem__'], + name='MD') self.backend.collections = mock_collection = Mock() mock_get_database.return_value = mock_database + mock_database.__getitem__ = Mock(name='MD.__getitem__') mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() - mock_database.__getitem__.assert_called_once_with( - MONGODB_COLLECTION) self.assertTrue(mock_collection.remove.called) def test_get_database_authfailure(self): From a5f121a549d67824c94413817b1b29ec296736e2 Mon Sep 17 00:00:00 2001 From: Nathan Van Gheem Date: Wed, 10 Sep 2014 16:52:08 -0500 Subject: [PATCH 0320/1103] add couchdb support --- celery/backends/__init__.py | 1 + celery/backends/couchdb.py | 127 ++++++++++++++++++++++++++ celery/tests/backends/test_couchdb.py | 122 +++++++++++++++++++++++++ docs/configuration.rst | 52 +++++++++++ requirements/extras/couchdb.txt | 2 +- 5 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 celery/backends/couchdb.py create mode 100644 celery/tests/backends/test_couchdb.py diff --git a/celery/backends/__init__.py b/celery/backends/__init__.py index 83c570804..eec585227 100644 --- a/celery/backends/__init__.py +++ b/celery/backends/__init__.py @@ -31,6 +31,7 @@ 'database': 'celery.backends.database:DatabaseBackend', 'cassandra': 'celery.backends.cassandra:CassandraBackend', 'couchbase': 'celery.backends.couchbase:CouchBaseBackend', + 'couchdb': 'celery.backends.couchdb:CouchDBBackend', 'riak': 'celery.backends.riak:RiakBackend', 'disabled': 'celery.backends.base:DisabledBackend', } diff --git a/celery/backends/couchdb.py b/celery/backends/couchdb.py new file mode 100644 index 000000000..0d2a68d4b --- /dev/null +++ b/celery/backends/couchdb.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +""" + celery.backends.couchdb + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + CouchDB result store backend. + +""" +from __future__ import absolute_import + +import logging + +try: + import pycouchdb +except ImportError: + pycouchdb = None # noqa + +from kombu.utils.url import _parse_url + +from celery.exceptions import ImproperlyConfigured +from celery.utils.timeutils import maybe_timedelta + +from .base import KeyValueStoreBackend + +__all__ = ['CouchDBBackend'] + + +class CouchDBBackend(KeyValueStoreBackend): + container = 'default' + scheme = 'http' + host = 'localhost' + port = 5984 + username = None + password = None + quiet = False + conncache = None + unlock_gil = True + timeout = 2.5 + transcoder = None + # supports_autoexpire = False + + def __init__(self, url=None, *args, **kwargs): + """Initialize CouchDB backend instance. + + :raises celery.exceptions.ImproperlyConfigured: if + module :mod:`pycouchdb` is not available. + + """ + super(CouchDBBackend, self).__init__(*args, **kwargs) + + self.expires = kwargs.get('expires') or maybe_timedelta( + self.app.conf.CELERY_TASK_RESULT_EXPIRES) + + if pycouchdb is None: + raise ImproperlyConfigured( + 'You need to install the pycouchdb library to use the ' + 'CouchDB backend.', + ) + + uscheme = uhost = uport = uname = upass = ucontainer = None + if url: + _, uhost, uport, uname, upass, ucontainer , _ = _parse_url(url) # noqa + ucontainer = ucontainer.strip('/') if ucontainer else None + + config = self.app.conf.get('CELERY_COUCHDB_BACKEND_SETTINGS', None) + if config is not None: + if not isinstance(config, dict): + raise ImproperlyConfigured( + 'CouchDB backend settings should be grouped in a dict', + ) + else: + config = {} + + self.scheme = uscheme or config.get('scheme', self.scheme) + self.host = uhost or config.get('host', self.host) + self.port = int(uport or config.get('port', self.port)) + self.container = ucontainer or config.get('container', self.container) + self.username = uname or config.get('username', self.username) + self.password = upass or config.get('password', self.password) + + self._connection = None + + def _get_connection(self): + """Connect to the CouchDB server.""" + if self._connection is None: + if self.username and self.password: + conn_string = '%s://%s:%s@%s:%s' % ( + self.scheme, self.username, self.password, + self.host, str(self.port)) + server = pycouchdb.Server(conn_string, authmethod='basic') + else: + conn_string = '%s://%s:%s' % ( + self.scheme, self.host, str(self.port)) + server = pycouchdb.Server(conn_string) + + logging.debug('couchdb conn string: %s', conn_string) + try: + self._connection = server.database(self.container) + except pycouchdb.exceptions.NotFound: + self._connection = server.create(self.container) + return self._connection + + @property + def connection(self): + return self._get_connection() + + def get(self, key): + try: + return self.connection.get(key)['value'] + except pycouchdb.exceptions.NotFound: + return None + + def set(self, key, value): + data = {'_id': key, 'value': value} + try: + self.connection.save(data) + except pycouchdb.exceptions.Conflict: + # document already exists, update it + data = self.connection.get(key) + data['value'] = value + self.connection.save(data) + + def mget(self, keys): + return [self.get(key) for key in keys] + + def delete(self, key): + self.connection.delete(key) diff --git a/celery/tests/backends/test_couchdb.py b/celery/tests/backends/test_couchdb.py new file mode 100644 index 000000000..1df8eec46 --- /dev/null +++ b/celery/tests/backends/test_couchdb.py @@ -0,0 +1,122 @@ +from __future__ import absolute_import + +from celery.backends import couchdb as module +from celery.backends.couchdb import CouchDBBackend +from celery.exceptions import ImproperlyConfigured +from celery import backends +from celery.tests.case import ( + AppCase, Mock, SkipTest, patch, sentinel, +) + +try: + import pycouchdb +except ImportError: + pycouchdb = None # noqa + +COUCHDB_CONTAINER = 'celery_container' + + +class test_CouchDBBackend(AppCase): + + def setup(self): + if pycouchdb is None: + raise SkipTest('pycouchdb is not installed.') + self.backend = CouchDBBackend(app=self.app) + + def test_init_no_pycouchdb(self): + """test init no pycouchdb raises""" + prev, module.pycouchdb = module.pycouchdb, None + try: + with self.assertRaises(ImproperlyConfigured): + CouchDBBackend(app=self.app) + finally: + module.pycouchdb = prev + + def test_init_no_settings(self): + """test init no settings""" + self.app.conf.CELERY_COUCHDB_BACKEND_SETTINGS = [] + with self.assertRaises(ImproperlyConfigured): + CouchDBBackend(app=self.app) + + def test_init_settings_is_None(self): + """Test init settings is None""" + self.app.conf.CELERY_COUCHDB_BACKEND_SETTINGS = None + CouchDBBackend(app=self.app) + + def test_get_container_exists(self): + with patch('pycouchdb.client.Database') as mock_Connection: + self.backend._connection = sentinel._connection + + connection = self.backend._get_connection() + + self.assertEqual(sentinel._connection, connection) + self.assertFalse(mock_Connection.called) + + def test_get(self): + """test_get + + CouchDBBackend.get should return and take two params + db conn to couchdb is mocked. + TODO Should test on key not exists + + """ + self.app.conf.CELERY_COUCHDB_BACKEND_SETTINGS = {} + x = CouchDBBackend(app=self.app) + x._connection = Mock() + mocked_get = x._connection.get = Mock() + mocked_get.return_value = sentinel.retval + # should return None + self.assertEqual(x.get('1f3fab'), sentinel.retval) + x._connection.get.assert_called_once_with('1f3fab') + + def test_delete(self): + """test_delete + + CouchDBBackend.delete should return and take two params + db conn to pycouchdb is mocked. + TODO Should test on key not exists + + """ + self.app.conf.CELERY_COUCHDB_BACKEND_SETTINGS = {} + x = CouchDBBackend(app=self.app) + x._connection = Mock() + mocked_delete = x._connection.delete = Mock() + mocked_delete.return_value = None + # should return None + self.assertIsNone(x.delete('1f3fab')) + x._connection.delete.assert_called_once_with('1f3fab') + + def test_config_params(self): + """test_config_params + + celery.conf.CELERY_COUCHDB_BACKEND_SETTINGS is properly set + """ + self.app.conf.CELERY_COUCHDB_BACKEND_SETTINGS = { + 'container': 'mycoolcontainer', + 'host': ['here.host.com', 'there.host.com'], + 'username': 'johndoe', + 'password': 'mysecret', + 'port': '1234', + } + x = CouchDBBackend(app=self.app) + self.assertEqual(x.container, 'mycoolcontainer') + self.assertEqual(x.host, ['here.host.com', 'there.host.com'],) + self.assertEqual(x.username, 'johndoe',) + self.assertEqual(x.password, 'mysecret') + self.assertEqual(x.port, 1234) + + def test_backend_by_url(self, url='couchdb://myhost/mycoolcontainer'): + from celery.backends.couchdb import CouchDBBackend + backend, url_ = backends.get_backend_by_url(url, self.app.loader) + self.assertIs(backend, CouchDBBackend) + self.assertEqual(url_, url) + + def test_backend_params_by_url(self): + url = 'couchdb://johndoe:mysecret@myhost:123/mycoolcontainer' + with self.Celery(backend=url) as app: + x = app.backend + self.assertEqual(x.container, 'mycoolcontainer') + self.assertEqual(x.host, 'myhost') + self.assertEqual(x.username, 'johndoe') + self.assertEqual(x.password, 'mysecret') + self.assertEqual(x.port, 123) diff --git a/docs/configuration.rst b/docs/configuration.rst index 1fd716b65..e97f49c07 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -221,6 +221,10 @@ Can be one of the following: Use `Couchbase`_ to store the results. See :ref:`conf-couchbase-result-backend`. +* couchdb + Use `CouchDB`_ to store the results. + See :ref:`conf-couchdb-result-backend`. + .. warning: While the AMQP result backend is very efficient, you must make sure @@ -773,6 +777,54 @@ This is a dict supporting the following keys: Password to authenticate to the Couchbase server (optional). +.. _conf-couchdb-result-backend: + +CouchDB backend settings +------------------------ + +.. note:: + + The CouchDB backend requires the :mod:`pycouchdb` library: + https://pypi.python.org/pypi/pycouchdb + + To install the couchbase package use `pip` or `easy_install`: + + .. code-block:: bash + + $ pip install pycouchdb + +This backend can be configured via the :setting:`CELERY_RESULT_BACKEND` +set to a couchbase URL:: + + CELERY_RESULT_BACKEND = 'couchdb://username:password@host:port/container' + + +.. setting:: CELERY_COUCHDB_BACKEND_SETTINGS + +CELERY_COUCHDB_BACKEND_SETTINGS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This is a dict supporting the following keys: + +* scheme + http or https. Defaults to ``http``. + +* host + Host name of the CouchDB server. Defaults to ``localhost``. + +* port + The port the CouchDB server is listening to. Defaults to ``8091``. + +* container + The default container the CouchDB server is writing to. + Defaults to ``default``. + +* username + User name to authenticate to the CouchDB server as (optional). + +* password + Password to authenticate to the CouchDB server (optional). + .. _conf-messaging: Message Routing diff --git a/requirements/extras/couchdb.txt b/requirements/extras/couchdb.txt index 3e100d4b3..bc7a1a32b 100644 --- a/requirements/extras/couchdb.txt +++ b/requirements/extras/couchdb.txt @@ -1 +1 @@ -couchdb +pycouchdb From 133f2a1aecf5c6bb6e7e04978d14d041a9706c42 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 15 Sep 2014 17:07:21 +0100 Subject: [PATCH 0321/1103] Adds Task.add_to_chord and Task.replace_in_chord --- celery/app/task.py | 29 +++++++++++++++- celery/backends/base.py | 3 ++ celery/backends/redis.py | 15 ++++++-- celery/tests/backends/test_base.py | 5 +++ celery/tests/backends/test_redis.py | 18 +++++++--- celery/tests/tasks/test_chord.py | 53 ++++++++++++++++++++++++++++- funtests/stress/stress/app.py | 11 ++++++ funtests/stress/stress/templates.py | 2 +- 8 files changed, 126 insertions(+), 10 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index c21347822..9fce0c3c6 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -16,7 +16,7 @@ from celery import states from celery._state import _task_stack from celery.canvas import signature -from celery.exceptions import MaxRetriesExceededError, Reject, Retry +from celery.exceptions import Ignore, MaxRetriesExceededError, Reject, Retry from celery.five import class_property, items from celery.result import EagerResult from celery.utils import uuid, maybe_reraise @@ -686,6 +686,33 @@ def send_event(self, type_, **fields): with self.app.events.default_dispatcher(hostname=req.hostname) as d: return d.send(type_, uuid=req.id, **fields) + def replace_in_chord(self, sig): + sig.freeze(self.request.id, + group_id=self.request.group, + chord=self.request.chord, + root_id=self.request.root_id) + sig.delay() + raise Ignore('Chord member replaced by new task') + + def add_to_chord(self, sig, lazy=False): + """Add signature to the chord the current task is a member of. + + :param sig: Signature to extend chord with. + :param lazy: If enabled the new task will not actually be called, + and ``sig.delay()`` must be called manually. + + Currently only supported by the Redis result backend when + ``?new_join=1`` is enabled. + + """ + if not self.request.chord: + raise ValueError('Current task is not member of any chord') + result = sig.freeze(group_id=self.request.group, + chord=self.request.chord, + root_id=self.request.root_id) + self.backend.add_to_chord(self.request.group, result) + return sig.delay() if not lazy else sig + def update_state(self, task_id=None, state=None, meta=None): """Update task state. diff --git a/celery/backends/base.py b/celery/backends/base.py index ef833e55d..e75aaa7e0 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -335,6 +335,9 @@ def process_cleanup(self): def on_task_call(self, producer, task_id): return {} + def add_to_chord(self, chord_id, result): + raise NotImplementedError('Backend does not support add_to_chord') + def on_chord_part_return(self, task, state, result, propagate=False): pass diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 0c62c7411..6b8e4f12d 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -177,6 +177,9 @@ def incr(self, key): def expire(self, key, value): return self.client.expire(key, value) + def add_to_chord(self, group_id, result): + self.client.incr(self.get_key_for_group(group_id, '.t'), 1) + def _unpack_chord_result(self, tup, decode, PROPAGATE_STATES=states.PROPAGATE_STATES): _, tid, state, retval = decode(tup) @@ -201,21 +204,27 @@ def _new_chord_return(self, task, state, result, propagate=None, client = self.client jkey = self.get_key_for_group(gid, '.j') + tkey = self.get_key_for_group(gid, '.t') result = self.encode_result(result, state) - _, readycount, _ = client.pipeline() \ + _, readycount, totaldiff, _, _ = client.pipeline() \ .rpush(jkey, self.encode([1, tid, state, result])) \ .llen(jkey) \ + .get(tkey) \ .expire(jkey, 86400) \ + .expire(tkey, 86400) \ .execute() + totaldiff = int(totaldiff or 0) + try: callback = maybe_signature(request.chord, app=app) - total = callback['chord_size'] + total = callback['chord_size'] + totaldiff if readycount >= total: decode, unpack = self.decode, self._unpack_chord_result - resl, _ = client.pipeline() \ + resl, _, _ = client.pipeline() \ .lrange(jkey, 0, total) \ .delete(jkey) \ + .delete(tkey) \ .execute() try: callback.delay([unpack(tup, decode) for tup in resl]) diff --git a/celery/tests/backends/test_base.py b/celery/tests/backends/test_base.py index 6f626b76b..f1cde8984 100644 --- a/celery/tests/backends/test_base.py +++ b/celery/tests/backends/test_base.py @@ -188,6 +188,11 @@ def test_save_group(self): b.save_group('foofoo', 'xxx') b._save_group.assert_called_with('foofoo', 'xxx') + def test_add_to_chord_interface(self): + b = BaseBackend(self.app) + with self.assertRaises(NotImplementedError): + b.add_to_chord('group_id', 'sig') + def test_forget_interface(self): b = BaseBackend(self.app) with self.assertRaises(NotImplementedError): diff --git a/celery/tests/backends/test_redis.py b/celery/tests/backends/test_redis.py index ad8b50fc7..b2ebcd2a3 100644 --- a/celery/tests/backends/test_redis.py +++ b/celery/tests/backends/test_redis.py @@ -12,7 +12,8 @@ from celery.exceptions import ImproperlyConfigured from celery.tests.case import ( - AppCase, Mock, MockCallbacks, SkipTest, depends_on_current_app, patch, + AppCase, Mock, MockCallbacks, SkipTest, + call, depends_on_current_app, patch, ) @@ -194,6 +195,12 @@ def test_set_new_join_from_url_query(self): self.assertEqual(b.on_chord_part_return, b._new_chord_return) self.assertEqual(b.apply_chord, b._new_chord_apply) + def test_add_to_chord(self): + b = self.Backend('redis://?new_join=True', app=self.app) + gid = uuid() + b.add_to_chord(gid, 'sig') + b.client.incr.assert_called_with(b.get_key_for_group(gid, '.t'), 1) + def test_default_is_old_join(self): b = self.Backend(app=self.app) self.assertNotEqual(b.on_chord_part_return, b._new_chord_return) @@ -250,9 +257,12 @@ def create_task(): self.assertTrue(b.client.rpush.call_count) b.client.rpush.reset_mock() self.assertTrue(b.client.lrange.call_count) - gkey = b.get_key_for_group('group_id', '.j') - b.client.delete.assert_called_with(gkey) - b.client.expire.assert_called_witeh(gkey, 86400) + jkey = b.get_key_for_group('group_id', '.j') + tkey = b.get_key_for_group('group_id', '.t') + b.client.delete.assert_has_calls([call(jkey), call(tkey)]) + b.client.expire.assert_has_calls([ + call(jkey, 86400), call(tkey, 86400), + ]) def test_process_cleanup(self): self.Backend(app=self.app, new_join=True).process_cleanup() diff --git a/celery/tests/tasks/test_chord.py b/celery/tests/tasks/test_chord.py index 27424a30a..df06bdc4f 100644 --- a/celery/tests/tasks/test_chord.py +++ b/celery/tests/tasks/test_chord.py @@ -2,7 +2,7 @@ from contextlib import contextmanager -from celery import group +from celery import group, uuid from celery import canvas from celery import result from celery.exceptions import ChordError, Retry @@ -219,6 +219,57 @@ def test_apply(self): chord.run = prev +class test_add_to_chord(AppCase): + + def setup(self): + + @self.app.task(shared=False) + def add(x, y): + return x + y + self.add = add + + @self.app.task(shared=False, bind=True) + def adds(self, sig, lazy=False): + return self.add_to_chord(sig, lazy) + self.adds = adds + + def test_add_to_chord(self): + self.app.backend = Mock(name='backend') + + sig = self.add.s(2, 2) + sig.delay = Mock(name='sig.delay') + self.adds.request.group = uuid() + self.adds.request.id = uuid() + + with self.assertRaises(ValueError): + # task not part of chord + self.adds.run(sig) + self.adds.request.chord = self.add.s() + + res1 = self.adds.run(sig, True) + self.assertEqual(res1, sig) + self.assertTrue(sig.options['task_id']) + self.assertEqual(sig.options['group_id'], self.adds.request.group) + self.assertEqual(sig.options['chord'], self.adds.request.chord) + self.assertFalse(sig.delay.called) + self.app.backend.add_to_chord.assert_called_with( + self.adds.request.group, sig.freeze(), + ) + + self.app.backend.reset_mock() + sig2 = self.add.s(4, 4) + sig2.delay = Mock(name='sig2.delay') + res2 = self.adds.run(sig2) + self.assertEqual(res2, sig2.delay.return_value) + self.assertTrue(sig2.options['task_id']) + self.assertEqual(sig2.options['group_id'], self.adds.request.group) + self.assertEqual(sig2.options['chord'], self.adds.request.chord) + sig2.delay.assert_called_with() + self.app.backend.add_to_chord.assert_called_with( + self.adds.request.group, sig2.freeze(), + ) + + class test_Chord_task(ChordCase): def test_run(self): diff --git a/funtests/stress/stress/app.py b/funtests/stress/stress/app.py index 7c14a15cf..cd170bff8 100644 --- a/funtests/stress/stress/app.py +++ b/funtests/stress/stress/app.py @@ -121,6 +121,17 @@ def segfault(): assert False, 'should not get here' +@app.task(bind=True) +def chord_adds(self, x): + self.add_to_chord(add.s(x, x)) + return 42 + + +@app.task(bind=True) +def chord_replace(self, x): + return self.replace_in_chord(add.s(x, x)) + + @app.task def raising(exc=KeyError()): raise exc diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index bbcfd1469..0920319d1 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -91,7 +91,7 @@ class redis(default): @template() class redistore(default): - CELERY_RESULT_BACKEND = 'redis://' + CELERY_RESULT_BACKEND = 'redis://?new_join=1' @template() From 05e2567ede7c71f523cc5783ccb47ad8eac951d8 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 15 Sep 2014 17:10:53 +0100 Subject: [PATCH 0322/1103] Docs and cosmetics --- celery/app/task.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index a689cf8d6..eb990fcbd 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -590,17 +590,6 @@ def retry(self, args=None, kwargs=None, exc=None, throw=True, raise ret return ret - def replace(self, sig): - request = self.request - sig.set_immutable(True) - chord_id, request.chord = request.chord, None - group_id, request.group = request.group, None - callbacks, request.callbacks = request.callbacks, [sig] - if group_id or chord_id: - sig.set(group=group_id, chord=chord_id) - sig |= callbacks[0] - return sig - def apply(self, args=None, kwargs=None, link=None, link_error=None, **options): """Execute this task locally, by blocking until the task returns. @@ -697,7 +686,28 @@ def send_event(self, type_, **fields): with self.app.events.default_dispatcher(hostname=req.hostname) as d: return d.send(type_, uuid=req.id, **fields) + def replace(self, sig): + request = self.request + sig.set_immutable(True) + chord_id, request.chord = request.chord, None + group_id, request.group = request.group, None + callbacks, request.callbacks = request.callbacks, [sig] + if group_id or chord_id: + sig.set(group=group_id, chord=chord_id) + sig |= callbacks[0] + return sig + def replace_in_chord(self, sig): + """Replace the current task (which must be a member of a chord) + with a new task. + + Note that this will raise :exc:`~@Ignore`, so the best practice + is to always use ``return self.replace_in_chord(...)`` to convey + to the reader that the task will not continue after being replaced. + + :param: Signature of new task. + + """ sig.freeze(self.request.id, group_id=self.request.group, chord=self.request.chord, From 32a5e8acb5447093d2d7fcf3a9c829e9c40e30aa Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 15 Sep 2014 17:12:15 +0100 Subject: [PATCH 0323/1103] flakes --- celery/tests/backends/test_mongodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index 46628b05d..801da3c1b 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -11,7 +11,7 @@ from celery.exceptions import ImproperlyConfigured from celery.tests.case import ( AppCase, MagicMock, Mock, SkipTest, ANY, - call, depends_on_current_app, patch, sentinel, + depends_on_current_app, patch, sentinel, ) COLLECTION = 'taskmeta_celery' From ebb0b7826f6def4242553c56eaefbf7433218665 Mon Sep 17 00:00:00 2001 From: Nathaniel Varona Date: Wed, 17 Sep 2014 03:03:46 +0800 Subject: [PATCH 0324/1103] Added rabbitmq-server users tag instruction. Fix for unable to connect from remote connections --- docs/getting-started/brokers/rabbitmq.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/getting-started/brokers/rabbitmq.rst b/docs/getting-started/brokers/rabbitmq.rst index f599e756b..2b55670ce 100644 --- a/docs/getting-started/brokers/rabbitmq.rst +++ b/docs/getting-started/brokers/rabbitmq.rst @@ -54,6 +54,10 @@ allow that user access to that virtual host: $ sudo rabbitmqctl add_vhost myvhost +.. code-block:: bash + + $ sudo rabbitmqctl set_user_tags myuser mytag + .. code-block:: bash $ sudo rabbitmqctl set_permissions -p myvhost myuser ".*" ".*" ".*" From 4294293600d9ff0443b747861553eb63147b01b0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 18 Sep 2014 20:46:02 +0100 Subject: [PATCH 0325/1103] autodiscover: Adds support for Django 1.7 class names in INSTALLED_APPS. Closes #2248 --- celery/loaders/base.py | 7 +++++++ examples/django/proj/settings.py | 2 ++ 2 files changed, 9 insertions(+) diff --git a/celery/loaders/base.py b/celery/loaders/base.py index cf89ba2b5..c2a0823a5 100644 --- a/celery/loaders/base.py +++ b/celery/loaders/base.py @@ -278,6 +278,13 @@ def find_related_module(package, related_name): """Given a package name and a module name, tries to find that module.""" + # Django 1.7 allows for speciying a class name in INSTALLED_APPS. + # (Issue #2248). + try: + importlib.import_module(package) + except ImportError: + package, _, _ = package.rpartition('.') + try: pkg_path = importlib.import_module(package).__path__ except AttributeError: diff --git a/examples/django/proj/settings.py b/examples/django/proj/settings.py index c64ec949c..aa7fb38d3 100644 --- a/examples/django/proj/settings.py +++ b/examples/django/proj/settings.py @@ -131,6 +131,8 @@ 'django.contrib.sites', 'django.contrib.messages', 'django.contrib.staticfiles', + 'django.contrib.admin', + 'kombu.transport.django.KombuAppConfig', 'demoapp', # Uncomment the next line to enable the admin: # 'django.contrib.admin', From 373aa5c64013b013474e53547ea0fe17a9886104 Mon Sep 17 00:00:00 2001 From: roderick Date: Fri, 19 Sep 2014 22:47:53 +0200 Subject: [PATCH 0326/1103] typo / finished sentence in comment --- celery/loaders/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/loaders/base.py b/celery/loaders/base.py index c2a0823a5..d8e99736c 100644 --- a/celery/loaders/base.py +++ b/celery/loaders/base.py @@ -153,7 +153,7 @@ def _smart_import(self, path, imp=None): return symbol_by_name(path, imp=imp) # Not sure if path is just a module name or if it includes an - # attribute name (e.g. ``os.path``, vs, ``os.path.abspath`` + # attribute name (e.g. ``os.path``, vs, ``os.path.abspath``). try: return imp(path) except ImportError: From 1632b4336060eba18e9ea495ed73be632a93015e Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Tue, 23 Sep 2014 11:15:18 +0300 Subject: [PATCH 0327/1103] don't clobber other event types in enable_events. Fixes: #2272 --- celery/worker/control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/worker/control.py b/celery/worker/control.py index da4a52dc3..3b2953da5 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -110,7 +110,7 @@ def report(state): @Panel.register def enable_events(state): dispatcher = state.consumer.event_dispatcher - if 'task' not in dispatcher.groups: + if dispatcher.groups and 'task' not in dispatcher.groups: dispatcher.groups.add('task') logger.info('Events of group {task} enabled by remote.') return {'ok': 'task events enabled'} From f315ee26e32251d3b038c0a76e98ea21d3627198 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Wed, 24 Sep 2014 12:34:34 +0300 Subject: [PATCH 0328/1103] add Konstantinos Koukopoulos to CONTRIBUTORS.txt Also setting new record for longest contributor name. --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index de656117d..c4f830520 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -168,3 +168,4 @@ Tadej Janež, 2014/08/08 Akexander Koshelev, 2014/08/19 Davide Quarta, 2014/08/19 John Whitlock, 2014/08/19 +Konstantinos Koukopoulos, 2014/08/24 From 2493576c753eb493c808401dafa49e9b4af76ef9 Mon Sep 17 00:00:00 2001 From: Andrea Rabbaglietti Date: Sat, 27 Sep 2014 18:58:51 +0200 Subject: [PATCH 0329/1103] Fixed bug on 'raise exc' At line 572: raise exc() has to be raise exc since exc is an instance not a class --- celery/app/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/task.py b/celery/app/task.py index eb990fcbd..921d9ae7e 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -569,7 +569,7 @@ def retry(self, args=None, kwargs=None, exc=None, throw=True, # first try to reraise the original exception maybe_reraise() # or if not in an except block then raise the custom exc. - raise exc() + raise exc raise self.MaxRetriesExceededError( "Can't retry {0}[{1}] args:{2} kwargs:{3}".format( self.name, request.id, S.args, S.kwargs)) From 21e3726213d33a9352e0e1fcb49aad479e7ebce6 Mon Sep 17 00:00:00 2001 From: Michael Floering Date: Tue, 30 Sep 2014 15:32:45 -0400 Subject: [PATCH 0330/1103] In Monitoring User Guide, link to Flower docs Addresses issue #2292 --- docs/userguide/monitoring.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index 6b85ae328..6bf7a01e2 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -255,6 +255,11 @@ Then, you can visit flower in your web browser : $ open http://localhost:5555 +Flower has many more features than are detailed here, including +authorization options. Check out the `official documentation`_ for more +information. + +.. _official documentation: http://flower.readthedocs.org/en/latest/ .. _monitoring-celeryev: From 46e4ebfd4ef7c91dd5276595beffb94877dd11fc Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 2 Oct 2014 12:47:54 +0100 Subject: [PATCH 0331/1103] Tests passing --- celery/tests/app/test_loaders.py | 6 +++++- celery/tests/worker/test_control.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/celery/tests/app/test_loaders.py b/celery/tests/app/test_loaders.py index ab69e501d..d4bc4d42c 100644 --- a/celery/tests/app/test_loaders.py +++ b/celery/tests/app/test_loaders.py @@ -263,7 +263,11 @@ def test_find_related_module(self): imp.return_value.__path__ = 'foo' base.find_related_module(base, 'tasks') - imp.side_effect = AttributeError() + + def se1(val): + imp.side_effect = AttributeError() + + imp.side_effect = se1 base.find_related_module(base, 'tasks') imp.side_effect = None diff --git a/celery/tests/worker/test_control.py b/celery/tests/worker/test_control.py index 4aa7531a0..4388280bf 100644 --- a/celery/tests/worker/test_control.py +++ b/celery/tests/worker/test_control.py @@ -140,6 +140,9 @@ def test_enable_events(self): evd = consumer.event_dispatcher evd.groups = set() panel.handle('enable_events') + self.assertFalse(evd.groups) + evd.groups = set(['worker']) + panel.handle('enable_events') self.assertIn('task', evd.groups) evd.groups = {'task'} self.assertIn('already enabled', panel.handle('enable_events')['ok']) From c67de364262def9045cde5318a811d05adc169e5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 3 Oct 2014 17:12:40 +0100 Subject: [PATCH 0332/1103] Sphinx: Use getfullargspec for Python 3. Closes #2302 --- celery/contrib/sphinx.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/celery/contrib/sphinx.py b/celery/contrib/sphinx.py index d22d82f5f..2e5743123 100644 --- a/celery/contrib/sphinx.py +++ b/celery/contrib/sphinx.py @@ -32,7 +32,10 @@ """ from __future__ import absolute_import -from inspect import formatargspec, getargspec +try: + from inspect import formatargspec, getfullargspec as getargspec +except ImportError: # Py2 + from inspect import formatargspec, getargspec # noqa from sphinx.domains.python import PyModulelevel from sphinx.ext.autodoc import FunctionDocumenter From baf0687110d7a9830b145359583792fb1eecde92 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 3 Oct 2014 17:13:22 +0100 Subject: [PATCH 0333/1103] Chord: Never call callback more than once --- celery/backends/base.py | 6 +++++- celery/backends/redis.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index e75aaa7e0..ead8cddff 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -549,7 +549,11 @@ def on_chord_part_return(self, task, state, result, propagate=None): ChordError('GroupResult {0} no longer exists'.format(gid)), ) val = self.incr(key) - if val >= len(deps): + size = len(deps) + if val > size: + logger.warning('Chord counter incremented too many times for %r', + gid) + elif val == size: callback = maybe_signature(task.request.chord, app=app) j = deps.join_native if deps.supports_native_join else deps.join try: diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 6b8e4f12d..4fc7efc75 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -219,7 +219,7 @@ def _new_chord_return(self, task, state, result, propagate=None, try: callback = maybe_signature(request.chord, app=app) total = callback['chord_size'] + totaldiff - if readycount >= total: + if readycount == total: decode, unpack = self.decode, self._unpack_chord_result resl, _, _ = client.pipeline() \ .lrange(jkey, 0, total) \ From 3639f21c7f9f2fee37e63ad7f547a7f9cd8b442e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 6 Oct 2014 22:48:13 +0100 Subject: [PATCH 0334/1103] Removes celery.utils.log.ensure_process_aware_logger --- celery/app/log.py | 4 +--- celery/tests/app/test_log.py | 9 --------- celery/tests/bin/test_worker.py | 3 --- celery/utils/log.py | 32 +------------------------------- 4 files changed, 2 insertions(+), 46 deletions(-) diff --git a/celery/app/log.py b/celery/app/log.py index 2ccf7284b..372bc1ed6 100644 --- a/celery/app/log.py +++ b/celery/app/log.py @@ -27,8 +27,7 @@ from celery.utils import isatty, node_format from celery.utils.log import ( get_logger, mlevel, - ColorFormatter, ensure_process_aware_logger, - LoggingProxy, get_multiprocessing_logger, + ColorFormatter, LoggingProxy, get_multiprocessing_logger, reset_multiprocessing_logger, ) from celery.utils.term import colored @@ -98,7 +97,6 @@ def setup_logging_subsystem(self, loglevel=None, logfile=None, format=None, format = format or self.format colorize = self.supports_color(colorize, logfile) reset_multiprocessing_logger() - ensure_process_aware_logger() receivers = signals.setup_logging.send( sender=None, loglevel=loglevel, logfile=logfile, format=format, colorize=colorize, diff --git a/celery/tests/app/test_log.py b/celery/tests/app/test_log.py index 588e39bee..1cc43a526 100644 --- a/celery/tests/app/test_log.py +++ b/celery/tests/app/test_log.py @@ -19,7 +19,6 @@ task_logger, in_sighandler, logger_isa, - ensure_process_aware_logger, ) from celery.tests.case import ( AppCase, Mock, SkipTest, @@ -363,14 +362,6 @@ def get_logger(self, *args, **kwargs): return get_task_logger('test_task_logger') -class test_patch_logger_cls(AppCase): - - def test_patches(self): - ensure_process_aware_logger() - with in_sighandler(): - logging.getLoggerClass().log(get_logger('test')) - - class MockLogger(logging.Logger): _records = None diff --git a/celery/tests/bin/test_worker.py b/celery/tests/bin/test_worker.py index c6d6033af..dea82e5c5 100644 --- a/celery/tests/bin/test_worker.py +++ b/celery/tests/bin/test_worker.py @@ -18,7 +18,6 @@ ImproperlyConfigured, WorkerShutdown, WorkerTerminate, ) from celery.platforms import EX_FAILURE, EX_OK -from celery.utils.log import ensure_process_aware_logger from celery.worker import state from celery.tests.case import ( @@ -31,8 +30,6 @@ skip_if_jython, ) -ensure_process_aware_logger() - class WorkerAppCase(AppCase): diff --git a/celery/utils/log.py b/celery/utils/log.py index b9226e1d8..66feef659 100644 --- a/celery/utils/log.py +++ b/celery/utils/log.py @@ -27,7 +27,7 @@ __all__ = ['ColorFormatter', 'LoggingProxy', 'base_logger', 'set_in_sighandler', 'in_sighandler', 'get_logger', - 'get_task_logger', 'mlevel', 'ensure_process_aware_logger', + 'get_task_logger', 'mlevel', 'get_multiprocessing_logger', 'reset_multiprocessing_logger'] _process_aware = False @@ -252,35 +252,6 @@ def isatty(self): return False -def ensure_process_aware_logger(force=False): - """Make sure process name is recorded when loggers are used.""" - global _process_aware - if force or not _process_aware: - logging._acquireLock() - try: - _process_aware = True - Logger = logging.getLoggerClass() - if getattr(Logger, '_process_aware', False): # pragma: no cover - return - - class ProcessAwareLogger(Logger): - _signal_safe = True - _process_aware = True - - def makeRecord(self, *args, **kwds): - record = Logger.makeRecord(self, *args, **kwds) - record.processName = current_process()._name - return record - - def log(self, *args, **kwargs): - if _in_sighandler: - return - return Logger.log(self, *args, **kwargs) - logging.setLoggerClass(ProcessAwareLogger) - finally: - logging._releaseLock() - - def get_multiprocessing_logger(): return mputil.get_logger() if mputil else None @@ -294,4 +265,3 @@ def current_process_index(base=1): if current_process: index = getattr(current_process(), 'index', None) return index + base if index is not None else index -ensure_process_aware_logger() From 9d1c411610f756afa62c73646ce49b6ce1001e95 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 6 Oct 2014 23:18:32 +0100 Subject: [PATCH 0335/1103] Chords broken in master. Closes #2297 --- celery/backends/base.py | 3 ++- celery/backends/redis.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index ead8cddff..86041842f 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -350,7 +350,8 @@ def fallback_chord_unlock(self, group_id, body, result=None, def apply_chord(self, header, partial_args, group_id, body, options={}, **kwargs): - result = header(*partial_args, task_id=group_id, **options or {}) + options['task_id'] = group_id + result = header(*partial_args, **options or {}) self.fallback_chord_unlock(group_id, body, **kwargs) return result diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 4fc7efc75..2df9e8e0a 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -190,7 +190,8 @@ def _unpack_chord_result(self, tup, decode, def _new_chord_apply(self, header, partial_args, group_id, body, result=None, options={}, **kwargs): # avoids saving the group in the redis db. - return header(*partial_args, task_id=group_id, **options or {}) + options['task_id'] = group_id + return header(*partial_args, **options or {}) def _new_chord_return(self, task, state, result, propagate=None, PROPAGATE_STATES=states.PROPAGATE_STATES): From fcf3ff1b4fa70659bc54ae5abd6a13a6415359e9 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 7 Oct 2014 13:58:26 +0100 Subject: [PATCH 0336/1103] cosmetics --- celery/app/base.py | 4 ++-- celery/app/trace.py | 2 +- celery/backends/base.py | 2 +- celery/concurrency/asynpool.py | 2 +- celery/concurrency/prefork.py | 10 ++++------ celery/states.py | 12 ++++++------ celery/task/http.py | 2 +- celery/tests/app/test_loaders.py | 1 - celery/tests/worker/test_control.py | 2 +- extra/release/verify_config_reference.py | 4 ++-- 10 files changed, 19 insertions(+), 22 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index d5cc579da..70cbca501 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -52,9 +52,9 @@ __all__ = ['Celery'] _EXECV = os.environ.get('FORKED_BY_MULTIPROCESSING') -BUILTIN_FIXUPS = frozenset([ +BUILTIN_FIXUPS = { 'celery.fixups.django:fixup', -]) +} ERR_ENVVAR_NOT_SET = """\ The environment variable {0!r} is not set, diff --git a/celery/app/trace.py b/celery/app/trace.py index b34ca7e2b..e01543c65 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -104,7 +104,7 @@ RETRY = states.RETRY FAILURE = states.FAILURE EXCEPTION_STATES = states.EXCEPTION_STATES -IGNORE_STATES = frozenset([IGNORED, RETRY, REJECTED]) +IGNORE_STATES = frozenset({IGNORED, RETRY, REJECTED}) #: set by :func:`setup_worker_optimizations` _localized = [] diff --git a/celery/backends/base.py b/celery/backends/base.py index 86041842f..d22e633d5 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -43,7 +43,7 @@ __all__ = ['BaseBackend', 'KeyValueStoreBackend', 'DisabledBackend'] -EXCEPTION_ABLE_CODECS = frozenset(['pickle']) +EXCEPTION_ABLE_CODECS = frozenset({'pickle'}) PY3 = sys.version_info >= (3, 0) logger = get_logger(__name__) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 759861a21..650c812c4 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -76,7 +76,7 @@ def unpack_from(fmt, iobuf, unpack=struct.unpack): # noqa logger = get_logger(__name__) error, debug = logger.error, logger.debug -UNAVAIL = frozenset([errno.EAGAIN, errno.EINTR]) +UNAVAIL = frozenset({errno.EAGAIN, errno.EINTR}) #: Constant sent by child process when started (ready to accept work) WORKER_UP = 15 diff --git a/celery/concurrency/prefork.py b/celery/concurrency/prefork.py index ef4de4bed..c2f99caad 100644 --- a/celery/concurrency/prefork.py +++ b/celery/concurrency/prefork.py @@ -27,14 +27,12 @@ __all__ = ['TaskPool', 'process_initializer', 'process_destructor'] #: List of signals to reset when a child process starts. -WORKER_SIGRESET = frozenset(['SIGTERM', - 'SIGHUP', - 'SIGTTIN', - 'SIGTTOU', - 'SIGUSR1']) +WORKER_SIGRESET = { + 'SIGTERM', 'SIGHUP', 'SIGTTIN', 'SIGTTOU', 'SIGUSR1', +} #: List of signals to ignore when a child process starts. -WORKER_SIGIGNORE = frozenset(['SIGINT']) +WORKER_SIGIGNORE = {'SIGINT'} logger = get_logger(__name__) warning, debug = logger.warning, logger.debug diff --git a/celery/states.py b/celery/states.py index 665a57baf..ad8feebca 100644 --- a/celery/states.py +++ b/celery/states.py @@ -144,10 +144,10 @@ def __le__(self, other): IGNORED = 'IGNORED' REJECTED = 'REJECTED' -READY_STATES = frozenset([SUCCESS, FAILURE, REVOKED]) -UNREADY_STATES = frozenset([PENDING, RECEIVED, STARTED, RETRY]) -EXCEPTION_STATES = frozenset([RETRY, FAILURE, REVOKED]) -PROPAGATE_STATES = frozenset([FAILURE, REVOKED]) +READY_STATES = frozenset({SUCCESS, FAILURE, REVOKED}) +UNREADY_STATES = frozenset({PENDING, RECEIVED, STARTED, RETRY}) +EXCEPTION_STATES = frozenset({RETRY, FAILURE, REVOKED}) +PROPAGATE_STATES = frozenset({FAILURE, REVOKED}) -ALL_STATES = frozenset([PENDING, RECEIVED, STARTED, - SUCCESS, FAILURE, RETRY, REVOKED]) +ALL_STATES = frozenset({PENDING, RECEIVED, STARTED, + SUCCESS, FAILURE, RETRY, REVOKED}) diff --git a/celery/task/http.py b/celery/task/http.py index d3739b8ef..8d5a5e51d 100644 --- a/celery/task/http.py +++ b/celery/task/http.py @@ -25,7 +25,7 @@ __all__ = ['InvalidResponseError', 'RemoteExecuteError', 'UnknownStatusError', 'HttpDispatch', 'dispatch', 'URL'] -GET_METHODS = frozenset(['GET', 'HEAD']) +GET_METHODS = {'GET', 'HEAD'} logger = get_task_logger(__name__) diff --git a/celery/tests/app/test_loaders.py b/celery/tests/app/test_loaders.py index d4bc4d42c..bc39f6108 100644 --- a/celery/tests/app/test_loaders.py +++ b/celery/tests/app/test_loaders.py @@ -263,7 +263,6 @@ def test_find_related_module(self): imp.return_value.__path__ = 'foo' base.find_related_module(base, 'tasks') - def se1(val): imp.side_effect = AttributeError() diff --git a/celery/tests/worker/test_control.py b/celery/tests/worker/test_control.py index 4388280bf..340ade75b 100644 --- a/celery/tests/worker/test_control.py +++ b/celery/tests/worker/test_control.py @@ -141,7 +141,7 @@ def test_enable_events(self): evd.groups = set() panel.handle('enable_events') self.assertFalse(evd.groups) - evd.groups = set(['worker']) + evd.groups = {'worker'} panel.handle('enable_events') self.assertIn('task', evd.groups) evd.groups = {'task'} diff --git a/extra/release/verify_config_reference.py b/extra/release/verify_config_reference.py index 36183d8bb..7392a1c60 100644 --- a/extra/release/verify_config_reference.py +++ b/extra/release/verify_config_reference.py @@ -5,7 +5,7 @@ from celery.app.defaults import NAMESPACES, flatten -ignore = frozenset([ +ignore = { 'CELERYD_AGENT', 'CELERYD_POOL_PUTLOCKS', 'BROKER_HOST', @@ -18,7 +18,7 @@ 'CELERY_REDIS_PORT', 'CELERY_REDIS_DB', 'CELERY_REDIS_PASSWORD', -]) +} def is_ignored(setting, option): From 36a5aa938f57b7d0b66f58280010e860c28350b8 Mon Sep 17 00:00:00 2001 From: bee-keeper Date: Tue, 7 Oct 2014 16:49:59 +0100 Subject: [PATCH 0337/1103] apps typo in first-steps-with-django --- docs/django/first-steps-with-django.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/django/first-steps-with-django.rst b/docs/django/first-steps-with-django.rst index e25022e85..ac33d7da2 100644 --- a/docs/django/first-steps-with-django.rst +++ b/docs/django/first-steps-with-django.rst @@ -98,11 +98,11 @@ With the line above Celery will automatically discover tasks in reusable apps if you follow the ``tasks.py`` convention:: - app1/ - - app1/tasks.py - - app1/models.py + - tasks.py + - models.py - app2/ - - app2/tasks.py - - app2/models.py + - tasks.py + - models.py This way you do not have to manually add the individual modules to the :setting:`CELERY_IMPORTS` setting. The ``lambda`` so that the From 9d63a47fc3cf27098b177273ceff8e875d61e8c3 Mon Sep 17 00:00:00 2001 From: Kracekumar Ramaraju Date: Wed, 8 Oct 2014 12:10:28 +0530 Subject: [PATCH 0338/1103] Added missing backtick. --- celery/app/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/task.py b/celery/app/task.py index 921d9ae7e..e3a080515 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -432,7 +432,7 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, :keyword link_error: A single, or a list of tasks to apply if an error occurs while executing the task. - :keyword producer: :class:~@kombu.Producer` instance to use. + :keyword producer: :class:`~@kombu.Producer` instance to use. :keyword add_to_parent: If set to True (default) and the task is applied while executing another task, then the result will be appended to the parent tasks ``request.children`` From 1b70f9ef06f838395d5d7ddc9686fe88df82e0fb Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 8 Oct 2014 12:20:00 +0100 Subject: [PATCH 0339/1103] AsyncResult to raise ValueError if id is None. Closes #1996 --- celery/result.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/celery/result.py b/celery/result.py index 708fa69f4..7b1ab3971 100644 --- a/celery/result.py +++ b/celery/result.py @@ -81,6 +81,9 @@ class AsyncResult(ResultBase): def __init__(self, id, backend=None, task_name=None, app=None, parent=None): + if id is None: + raise ValueError( + 'AsyncResult requires valid id, not {0}'.format(type(id))) self.app = app_or_default(app or self.app) self.id = id self.backend = backend or self.app.backend From 8eae9d419bf2421c503e7f1f5b813223bf29dfa1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 8 Oct 2014 12:22:13 +0100 Subject: [PATCH 0340/1103] RPC backend: queue is now auto delete. Closes #2001 --- celery/backends/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/rpc.py b/celery/backends/rpc.py index 28d54263f..c78153622 100644 --- a/celery/backends/rpc.py +++ b/celery/backends/rpc.py @@ -57,7 +57,7 @@ def on_reply_declare(self, task_id): @property def binding(self): return self.Queue(self.oid, self.exchange, self.oid, - durable=False, auto_delete=False) + durable=False, auto_delete=True) @cached_property def oid(self): From eedb9ffecf4076f4b2a3738271c82a2e3ebdb7cc Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 8 Oct 2014 12:50:00 +0100 Subject: [PATCH 0341/1103] Tests passing --- celery/tests/backends/test_rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/tests/backends/test_rpc.py b/celery/tests/backends/test_rpc.py index 6fe594c19..60c3aaa5c 100644 --- a/celery/tests/backends/test_rpc.py +++ b/celery/tests/backends/test_rpc.py @@ -49,7 +49,7 @@ def test_binding(self): self.assertEqual(queue.exchange, self.b.exchange) self.assertEqual(queue.routing_key, self.b.oid) self.assertFalse(queue.durable) - self.assertFalse(queue.auto_delete) + self.assertTrue(queue.auto_delete) def test_many_bindings(self): self.assertListEqual( From cfa737d40c20eedc254dcdcdbe988090d71f762c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 8 Oct 2014 12:50:49 +0100 Subject: [PATCH 0342/1103] Travis: Disable pypy3 target, not working --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 11179dc00..867986b15 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,6 @@ env: - TOXENV=3.3 - TOXENV=3.4 - TOXENV=pypy - - TOXENV=pypy3 before_install: - | python --version From 94dae1b899aae6ae2ca333773fddbc6dd603213c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 10 Oct 2014 15:18:12 +0100 Subject: [PATCH 0343/1103] SQLAlchemy: Ignore all result engine options when using NullPool. Closes #1930 --- celery/backends/database/session.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/celery/backends/database/session.py b/celery/backends/database/session.py index 1575d7f32..022233aab 100644 --- a/celery/backends/database/session.py +++ b/celery/backends/database/session.py @@ -39,8 +39,7 @@ def get_engine(self, dburi, **kwargs): engine = self._engines[dburi] = create_engine(dburi, **kwargs) return engine else: - kwargs['poolclass'] = NullPool - return create_engine(dburi, **kwargs) + return create_engine(dburi, poolclass=NullPool) def create_session(self, dburi, short_lived_sessions=False, **kwargs): engine = self.get_engine(dburi, **kwargs) From 9ca65432ff797c487e70e25a046eec865afc3dd1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 10 Oct 2014 15:45:55 +0100 Subject: [PATCH 0344/1103] Fixes time_limit argument to request --- celery/worker/request.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/celery/worker/request.py b/celery/worker/request.py index 5058b79b6..4cc702c82 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -68,7 +68,7 @@ class Request(object): acknowledged = False time_start = None worker_pid = None - timeouts = (None, None) + time_limits = (None, None) _already_revoked = False _terminate_on_ack = None _apply_result = None @@ -107,8 +107,8 @@ def __init__(self, message, on_ack=noop, name = self.name = headers['task'] self.id = headers['id'] - if 'timeouts' in headers: - self.timeouts = headers['timeouts'] + if 'timelimit' in headers: + self.time_limits = headers['timelimit'] self.on_ack = on_ack self.on_reject = on_reject self.hostname = hostname or socket.gethostname() @@ -173,9 +173,9 @@ def execute_using_pool(self, pool, **kwargs): if self.revoked(): raise TaskRevokedError(task_id) - timeout, soft_timeout = self.timeouts - timeout = timeout or task.time_limit - soft_timeout = soft_timeout or task.soft_time_limit + time_limit, soft_time_limit = self.time_limits + time_limit = time_limit or task.time_limit + soft_time_limit = soft_time_limit or task.soft_time_limit result = pool.apply_async( trace_task_ret, args=(self.name, task_id, self.request_dict, self.body, @@ -184,8 +184,8 @@ def execute_using_pool(self, pool, **kwargs): timeout_callback=self.on_timeout, callback=self.on_success, error_callback=self.on_failure, - soft_timeout=soft_timeout or task.soft_time_limit, - timeout=timeout or task.time_limit, + soft_timeout=soft_time_limit, + timeout=time_limit, correlation_id=task_id, ) # cannot create weakref to None @@ -369,6 +369,7 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): def acknowledge(self): """Acknowledge task.""" if not self.acknowledged: + print('!!!!ACKING TASK!!!!') self.on_ack(logger, self.connection_errors) self.acknowledged = True @@ -455,9 +456,9 @@ def execute_using_pool(self, pool, **kwargs): if (self.expires or task_id in revoked_tasks) and self.revoked(): raise TaskRevokedError(task_id) - timeout, soft_timeout = self.timeouts - timeout = timeout or default_time_limit - soft_timeout = soft_timeout or default_soft_time_limit + time_limit, soft_time_limit = self.time_limits + time_limit = time_limit or default_time_limit + soft_time_limit = soft_time_limit or default_soft_time_limit result = apply_async( trace, args=(self.name, task_id, self.request_dict, self.body, @@ -466,8 +467,8 @@ def execute_using_pool(self, pool, **kwargs): timeout_callback=self.on_timeout, callback=self.on_success, error_callback=self.on_failure, - soft_timeout=soft_timeout, - timeout=timeout, + soft_timeout=soft_time_limit, + timeout=time_limit, correlation_id=task_id, ) # cannot create weakref to None From c01901db7462135cc3f4e6310b201eb68f9baf04 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 10 Oct 2014 15:46:14 +0100 Subject: [PATCH 0345/1103] Stresstest for task always timing out --- funtests/stress/stress/suite.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/funtests/stress/stress/suite.py b/funtests/stress/stress/suite.py index 2d8a1f67b..6e5e6a64a 100755 --- a/funtests/stress/stress/suite.py +++ b/funtests/stress/stress/suite.py @@ -19,7 +19,7 @@ from .app import ( marker, _marker, add, any_, exiting, kill, sleeping, - sleeping_ignore_limits, any_returning, print_unicode + sleeping_ignore_limits, any_returning ) from .data import BIG, SMALL from .fbi import FBI @@ -264,9 +264,18 @@ class Suite(BaseSuite): @testcase('all', 'green') def manyshort(self): - self.join(group(print_unicode.s(i, i) for i in range(1000))(), + self.join(group(add.s(i, i) for i in range(1000))(), timeout=10, propagate=True) + @testcase('all') + def always_timeout(self): + self.join( + group(sleeping.s(1).set(time_limit=0.1) + for _ in range(100) + )(), + timeout=10, propagate=True, + ) + @testcase('all') def termbysig(self): self._evil_groupmember(kill) From 4f43276c236bbef7239a49b93815f478aec1d9f6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 10 Oct 2014 16:13:58 +0100 Subject: [PATCH 0346/1103] Removes celery.contrib.methods. Too many bugs to be usable --- celery/contrib/methods.py | 126 ---------------------- celery/tests/contrib/test_methods.py | 34 ------ docs/reference/celery.contrib.methods.rst | 5 - docs/reference/index.rst | 1 - 4 files changed, 166 deletions(-) delete mode 100644 celery/contrib/methods.py delete mode 100644 celery/tests/contrib/test_methods.py delete mode 100644 docs/reference/celery.contrib.methods.rst diff --git a/celery/contrib/methods.py b/celery/contrib/methods.py deleted file mode 100644 index 56aa7f479..000000000 --- a/celery/contrib/methods.py +++ /dev/null @@ -1,126 +0,0 @@ -# -*- coding: utf-8 -*- -""" -celery.contrib.methods -====================== - -Task decorator that supports creating tasks out of methods. - -Examples --------- - -.. code-block:: python - - from celery.contrib.methods import task - - class X(object): - - @task() - def add(self, x, y): - return x + y - -or with any task decorator: - -.. code-block:: python - - from celery.contrib.methods import task_method - - class X(object): - - @app.task(filter=task_method) - def add(self, x, y): - return x + y - -.. note:: - - The task must use the new Task base class (:class:`celery.Task`), - and the old base class using classmethods (``celery.task.Task``, - ``celery.task.base.Task``). - - This means that you have to use the task decorator from a Celery app - instance, and not the old-API: - - .. code-block:: python - - - from celery import task # BAD - from celery.task import task # ALSO BAD - - # GOOD: - app = Celery(...) - - @app.task(filter=task_method) - def foo(self): pass - - # ALSO GOOD: - from celery import current_app - - @current_app.task(filter=task_method) - def foo(self): pass - - # ALSO GOOD: - from celery import shared_task - - @shared_task(filter=task_method) - def foo(self): pass - -Caveats -------- - -- Automatic naming won't be able to know what the class name is. - - The name will still be module_name + task_name, - so two methods with the same name in the same module will collide - so that only one task can run: - - .. code-block:: python - - class A(object): - - @task() - def add(self, x, y): - return x + y - - class B(object): - - @task() - def add(self, x, y): - return x + y - - would have to be written as: - - .. code-block:: python - - class A(object): - @task(name='A.add') - def add(self, x, y): - return x + y - - class B(object): - @task(name='B.add') - def add(self, x, y): - return x + y - -""" - -from __future__ import absolute_import - -from celery import current_app - -__all__ = ['task_method', 'task'] - - -class task_method(object): - - def __init__(self, task, *args, **kwargs): - self.task = task - - def __get__(self, obj, type=None): - if obj is None: - return self.task - task = self.task.__class__() - task.__self__ = obj - return task - - -def task(*args, **kwargs): - return current_app.task(*args, **dict(kwargs, filter=task_method)) diff --git a/celery/tests/contrib/test_methods.py b/celery/tests/contrib/test_methods.py deleted file mode 100644 index da74cc98b..000000000 --- a/celery/tests/contrib/test_methods.py +++ /dev/null @@ -1,34 +0,0 @@ -from __future__ import absolute_import - -from celery.contrib.methods import task_method, task - -from celery.tests.case import AppCase, patch - - -class test_task_method(AppCase): - - def test_task_method(self): - - class X(object): - - def __init__(self): - self.state = 0 - - @self.app.task(shared=False, filter=task_method) - def add(self, x): - self.state += x - - x = X() - x.add(2) - self.assertEqual(x.state, 2) - x.add(4) - self.assertEqual(x.state, 6) - - self.assertTrue(X.add) - self.assertIs(x.add.__self__, x) - - def test_task(self): - with patch('celery.contrib.methods.current_app') as curapp: - fun = object() - task(fun, x=1) - curapp.task.assert_called_with(fun, x=1, filter=task_method) diff --git a/docs/reference/celery.contrib.methods.rst b/docs/reference/celery.contrib.methods.rst deleted file mode 100644 index 539234e1f..000000000 --- a/docs/reference/celery.contrib.methods.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. currentmodule:: celery.contrib.methods - -.. automodule:: celery.contrib.methods - :members: - :undoc-members: diff --git a/docs/reference/index.rst b/docs/reference/index.rst index a8ae3ea02..118f220c4 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -39,7 +39,6 @@ celery.contrib.migrate celery.contrib.sphinx celery.contrib.rdb - celery.contrib.methods celery.events celery.events.state celery.beat From a97e66067aaf16391e25c6347616fd6fd28de736 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 10 Oct 2014 17:14:57 +0100 Subject: [PATCH 0347/1103] Chain partial args were not working properly in master --- celery/canvas.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 0be4e7a96..441a97bc2 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -376,6 +376,7 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, task_id=None, link=None, link_error=None, publisher=None, producer=None, root_id=None, app=None, **options): app = app or self.app + args = tuple(args) + tuple(self.args) tasks, results = self.prepare_steps( args, self.tasks, root_id, link_error, ) @@ -467,10 +468,6 @@ def apply(self, args=(), kwargs={}, **options): @classmethod def from_dict(self, d, app=None): - tasks = d['kwargs']['tasks'] - if d['args'] and tasks: - # partial args passed on to first task in chain (Issue #1057). - tasks[0]['args'] = tasks[0]._merge(d['args'])[0] return chain(*d['kwargs']['tasks'], app=app, **d['options']) @property From 4a138b9fe1cdbc62c7bbe7af9b6eaf71cd0d2806 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 10 Oct 2014 19:32:27 +0100 Subject: [PATCH 0348/1103] Removes print statement --- celery/worker/request.py | 1 - 1 file changed, 1 deletion(-) diff --git a/celery/worker/request.py b/celery/worker/request.py index 4cc702c82..0fb396f6a 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -369,7 +369,6 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): def acknowledge(self): """Acknowledge task.""" if not self.acknowledged: - print('!!!!ACKING TASK!!!!') self.on_ack(logger, self.connection_errors) self.acknowledged = True From 4f5c6dc8fc3409978f2ac6bdadd41d8bd0b36ec3 Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Mon, 13 Oct 2014 18:19:49 +0200 Subject: [PATCH 0349/1103] Fixed tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4a3a7cd6c..80cfd5c55 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = 2.7, 3.3, 3.4, - pypy + pypy, pypy3 [testenv] From 230b1ff7cb59b720464ffa9cc76fb19b4366d775 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 14 Oct 2014 00:34:01 +0100 Subject: [PATCH 0350/1103] Fixes deep dicts in chain --- celery/canvas.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index 441a97bc2..c8d4cfd4c 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -468,7 +468,11 @@ def apply(self, args=(), kwargs={}, **options): @classmethod def from_dict(self, d, app=None): - return chain(*d['kwargs']['tasks'], app=app, **d['options']) + tasks = d['kwargs']['tasks'] + if tasks: + # First task must be signature object to get app + tasks[0] = maybe_signature(tasks[0], app=app) + return chain(*tasks, app=app, **d['options']) @property def app(self): From 8c9cb819b1e112fe61eb76a9338069f244a7cb8a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 14 Oct 2014 00:34:27 +0100 Subject: [PATCH 0351/1103] ResultSet: Get app from first task --- celery/result.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/celery/result.py b/celery/result.py index 7b1ab3971..cdff02ff1 100644 --- a/celery/result.py +++ b/celery/result.py @@ -417,13 +417,13 @@ class ResultSet(ResultBase): :param results: List of result instances. """ - app = None + _app = None #: List of results in in the set. results = None def __init__(self, results, app=None, **kwargs): - self.app = app_or_default(app or self.app) + self.app = app self.results = results def add(self, result): @@ -733,6 +733,17 @@ def supports_native_join(self): except IndexError: pass + @property + def app(self): + if self._app is None: + self._app = (self.results[0].app if self.results else + current_app._get_current_object()) + return self._app + + @app.setter + def app(self, app): # noqa + self._app = app + @property def backend(self): return self.app.backend if self.app else self.results[0].backend From 728548d1be9e49b70fe553773f22b40d110bb41f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 14 Oct 2014 00:36:04 +0100 Subject: [PATCH 0352/1103] Task v1 requests missing group header. Closes #2297 --- celery/worker/strategy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index 801e58c3f..baf3070b5 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -41,6 +41,10 @@ def proto1_to_proto2(message, body): 'Task keyword arguments must be a mapping', ) body['headers'] = message.headers + try: + body['group'] = body['taskset'] + except KeyError: + pass return (args, kwargs), body, True, body.get('utc', True) From 8ff0371407ef33f16eeed2007b0f3a8069561aaf Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 14 Oct 2014 00:49:16 +0100 Subject: [PATCH 0353/1103] Fixes ResultSet --- celery/result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/result.py b/celery/result.py index cdff02ff1..902dea700 100644 --- a/celery/result.py +++ b/celery/result.py @@ -423,7 +423,7 @@ class ResultSet(ResultBase): results = None def __init__(self, results, app=None, **kwargs): - self.app = app + self._app = app self.results = results def add(self, result): From 17d9b2eed9c2c8a8901e0191fd1f227dbbf71b3e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 14 Oct 2014 00:51:53 +0100 Subject: [PATCH 0354/1103] Tests passing --- celery/canvas.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/celery/canvas.py b/celery/canvas.py index c8d4cfd4c..20c3fecf7 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -470,6 +470,8 @@ def apply(self, args=(), kwargs={}, **options): def from_dict(self, d, app=None): tasks = d['kwargs']['tasks'] if tasks: + if isinstance(tasks, tuple): # aaaargh + tasks = d['kwargs']['tasks'] = list(tasks) # First task must be signature object to get app tasks[0] = maybe_signature(tasks[0], app=app) return chain(*tasks, app=app, **d['options']) From 7015c3b1addc3b9b6acec5e2f4fe975bb3b5750b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 15 Oct 2014 19:17:33 +0100 Subject: [PATCH 0355/1103] [Py3] Fixes problem with ETA and local timezones. Closes #2306 --- celery/utils/timeutils.py | 49 +++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/celery/utils/timeutils.py b/celery/utils/timeutils.py index 494aa6f45..d1e324c08 100644 --- a/celery/utils/timeutils.py +++ b/celery/utils/timeutils.py @@ -10,6 +10,7 @@ import numbers import os +import sys import time as _time from calendar import monthrange @@ -17,7 +18,7 @@ from kombu.utils import cached_property, reprcall -from pytz import timezone as _timezone, AmbiguousTimeError +from pytz import timezone as _timezone, AmbiguousTimeError, FixedOffset from celery.five import string_t @@ -31,6 +32,9 @@ 'localize', 'to_utc', 'maybe_make_aware', 'ffwd', 'utcoffset', 'adjust_timestamp', 'maybe_s_to_ms'] +PY3 = sys.version_info[0] == 3 +PY33 = sys.version_info >= (3, 3) + C_REMDEBUG = os.environ.get('C_REMDEBUG', False) DAYNAMES = 'sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat' @@ -55,6 +59,7 @@ class LocalTimezone(tzinfo): Used only when UTC is not enabled. """ + _offset_cache = {} def __init__(self): # This code is moved in __init__ to execute it as late as possible @@ -68,23 +73,34 @@ def __init__(self): tzinfo.__init__(self) def __repr__(self): - return '' + return ''.format( + int(self.DSTOFFSET.total_seconds() / 3600), + ) def utcoffset(self, dt): - if self._isdst(dt): - return self.DSTOFFSET - else: - return self.STDOFFSET + return self.DSTOFFSET if self._isdst(dt) else self.STDOFFSET def dst(self, dt): - if self._isdst(dt): - return self.DSTDIFF - else: - return ZERO + return self.DSTDIFF if self._isdst(dt) else ZERO def tzname(self, dt): return _time.tzname[self._isdst(dt)] + if PY3: + + def fromutc(self, dt): + # The base tzinfo class no longer implements a DST + # offset aware .fromutc() in Python3 (Issue #2306). + + # I'd rather rely on pytz to do this, than port + # the C code from cpython's fromutc [asksol] + offset = int(self.utcoffset(dt).seconds / 60.0) + try: + tz = self._offset_cache[offset] + except KeyError: + tz = self._offset_cache[offset] = FixedOffset(offset) + return tz.fromutc(dt.replace(tzinfo=tz)) + def _isdst(self, dt): tt = (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, @@ -106,8 +122,17 @@ def to_local(self, dt, local=None, orig=None): dt = make_aware(dt, orig or self.utc) return localize(dt, self.tz_or_local(local)) - def to_system(self, dt): - return localize(dt, self.local) + if PY33: + + def to_system(self, dt): + # tz=None is a special case since Python 3.3, and will + # convert to the current local timezone (Issue #2306). + return dt.astimezone(tz=None) + + else: + + def to_system(self, dt): # noqa + return localize(dt, self.local) def to_local_fallback(self, dt): if is_naive(dt): From 07ecd08a8621affde3b8ed15d118164cb26e334d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 20 Oct 2014 15:38:26 +0100 Subject: [PATCH 0356/1103] Task.replace changed, removes Task.replace_in_chord. The two methods had almost the same functionality, but the old Task.replace would force the new task to inherit the callbacks/errbacks of the existing task. If you replace a node in a tree, then you would not expect the new node to inherit the children of the old node, so this seems like unexpected behavior. So self.replace(sig) now works for any task, in addition sig can now be a group. Groups are automatically converted to a chord, where the callback will "accumulate" the results of the group tasks. A new builtin task (`celery.accumulate` was added for this purpose) Closes #817 --- celery/app/builtins.py | 10 ++++++++++ celery/app/task.py | 33 +++++++++++++++------------------ 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/celery/app/builtins.py b/celery/app/builtins.py index b633aeb81..90cc9c9b7 100644 --- a/celery/app/builtins.py +++ b/celery/app/builtins.py @@ -33,6 +33,16 @@ def backend_cleanup(): return backend_cleanup +@connect_on_app_finalize +def add_accumulate_task(app): + """This task is used by Task.replace when replacing a task with + a group, to "collect" results.""" + @app.task(bind=True, name='celery.accumulate', shared=False, lazy=False) + def accumulate(self, *args, **kwargs): + index = kwargs.get('index') + return args[index] if index is not None else args + + @connect_on_app_finalize def add_unlock_chord_task(app): """This task is used by result backends without native chord support. diff --git a/celery/app/task.py b/celery/app/task.py index e3a080515..e8d1eafb0 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -12,7 +12,7 @@ from billiard.einfo import ExceptionInfo -from celery import current_app +from celery import current_app, group from celery import states from celery._state import _task_stack from celery.canvas import signature @@ -687,30 +687,27 @@ def send_event(self, type_, **fields): return d.send(type_, uuid=req.id, **fields) def replace(self, sig): - request = self.request - sig.set_immutable(True) - chord_id, request.chord = request.chord, None - group_id, request.group = request.group, None - callbacks, request.callbacks = request.callbacks, [sig] - if group_id or chord_id: - sig.set(group=group_id, chord=chord_id) - sig |= callbacks[0] - return sig - - def replace_in_chord(self, sig): - """Replace the current task (which must be a member of a chord) - with a new task. - - Note that this will raise :exc:`~@Ignore`, so the best practice - is to always use ``return self.replace_in_chord(...)`` to convey + """Replace the current task, with a new task inheriting the + same task id. + + :param sig: :class:`@signature` + + Note: This will raise :exc:`~@Ignore`, so the best practice + is to always use ``raise self.replace_in_chord(...)`` to convey to the reader that the task will not continue after being replaced. :param: Signature of new task. """ + chord = self.request.chord + if isinstance(sig, group): + sig |= self.app.tasks['celery.accumulate'].s(index=0).set( + chord=chord, + ) + chord = None sig.freeze(self.request.id, group_id=self.request.group, - chord=self.request.chord, + chord=chord, root_id=self.request.root_id) sig.delay() raise Ignore('Chord member replaced by new task') From 08752a708534f7b4918d00f9c59f48110c8e8a67 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 20 Oct 2014 19:02:05 +0100 Subject: [PATCH 0357/1103] celery logtool: Utility for filtering and parsing celery worker logfiles --- celery/bin/logtool.py | 171 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 celery/bin/logtool.py diff --git a/celery/bin/logtool.py b/celery/bin/logtool.py new file mode 100644 index 000000000..73cf9460f --- /dev/null +++ b/celery/bin/logtool.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +""" + +The :program:`celery logtool` command. + +.. program:: celery logtool + +""" + +from __future__ import absolute_import, unicode_literals + +import re + +from collections import Counter +from fileinput import FileInput + +from celery.datastructures import DependencyGraph, GraphFormatter +from celery.five import items + +from .base import Command + +__all__ = ['logtool'] + +RE_LOG_START = re.compile('^\[\d\d\d\d\-\d\d-\d\d ') +RE_TASK_RECEIVED = re.compile('.+?\] Received') +RE_TASK_READY = re.compile('.+?\] Task') +RE_TASK_INFO = re.compile('.+?([\w\.]+)\[(.+?)\].+') +RE_TASK_RESULT = re.compile('.+?[\w\.]+\[.+?\] (.+)') + +REPORT_FORMAT = """ +Report +====== + +Task total: {task[total]} +Task errors: {task[errors]} +Task success: {task[succeeded]} +Task completed: {task[completed]} + +Tasks +===== +{task[types].format} +""" + + +class _task_counts(list): + + @property + def format(self): + return '\n'.join('{0}: {1}'.format(*i) for i in self) + +def task_info(line): + m = RE_TASK_INFO.match(line) + return m.groups() + + +class Audit(object): + + def __init__(self, on_task_error=None, on_trace=None, on_debug=None): + self.ids = set() + self.names = {} + self.results = {} + self.ready = set() + self.task_types = Counter() + self.task_errors = 0 + self.on_task_error = on_task_error + self.on_trace = on_trace + self.on_debug = on_debug + self.prev_line = None + + def run(self, files): + for line in FileInput(files): + self.feed(line) + return self + + def task_received(self, line, task_name, task_id): + self.names[task_id] = task_name + self.ids.add(task_id) + self.task_types[task_name] += 1 + + def task_ready(self, line, task_name, task_id, result): + self.ready.add(task_id) + self.results[task_id] = result + if 'succeeded' not in result: + self.task_error(line, task_name, task_id, result) + + def task_error(self, line, task_name, task_id, result): + self.task_errors += 1 + if self.on_task_error: + self.on_task_error(line, task_name, task_id, result) + + def feed(self, line): + if RE_LOG_START.match(line): + if RE_TASK_RECEIVED.match(line): + task_name, task_id = task_info(line) + self.task_received(line, task_name, task_id) + elif RE_TASK_READY.match(line): + task_name, task_id = task_info(line) + result = RE_TASK_RESULT.match(line) + if result: + result, = result.groups() + self.task_ready(line, task_name, task_id, result) + else: + if self.on_debug: + self.on_debug(line) + self.prev_line = line + else: + if self.on_trace: + self.on_trace('\n'.join(filter(None, [self.prev_line, line]))) + self.prev_line = None + + def incomplete_tasks(self): + return self.ids ^ self.ready + + def report(self): + return { + 'task': { + 'types': _task_counts(self.task_types.most_common()), + 'total': len(self.ids), + 'errors': self.task_errors, + 'completed': len(self.ready), + 'succeeded': len(self.ready) - self.task_errors, + } + } + + +class logtool(Command): + args = """ [arguments] + ..... stats [file1|- [file2 [...]]] + ..... traces [file1|- [file2 [...]]] + ..... errors [file1|- [file2 [...]]] + ..... incomplete [file1|- [file2 [...]]] + ..... debug [file1|- [file2 [...]]] + """ + + def run(self, what=None, *files, **kwargs): + map = { + 'stats': self.stats, + 'traces': self.traces, + 'errors': self.errors, + 'incomplete': self.incomplete, + 'debug': self.debug, + } + if not what: + raise self.UsageError('missing action') + elif what not in map: + raise self.Error('no action {0} in {1}'.format(what, '|'.join(map))) + + return map[what](files) + + def stats(self, files): + self.out(REPORT_FORMAT.format( + **Audit().run(files).report() + )) + + def traces(self, files): + Audit(on_trace=self.out).run(files) + + def errors(self, files): + Audit(on_task_error=self.say1).run(files) + + def incomplete(self, files): + audit = Audit() + audit.run(files) + for task_id in audit.incomplete_tasks(): + self.error('Did not complete: %r' % (task_id, )) + + def debug(self, files): + Audit(on_debug=self.out).run(files) + + def say1(self, line, *_): + self.out(line) From 37befb163b33fc5d95053f9928474e34c7ef3ce3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 22 Oct 2014 13:55:23 +0100 Subject: [PATCH 0358/1103] flakes --- celery/bin/celery.py | 4 +++- celery/bin/logtool.py | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/celery/bin/celery.py b/celery/bin/celery.py index 191d28ade..d558dd8ac 100644 --- a/celery/bin/celery.py +++ b/celery/bin/celery.py @@ -31,6 +31,7 @@ from celery.bin.beat import beat from celery.bin.events import events from celery.bin.graph import graph +from celery.bin.logtool import logtool from celery.bin.worker import worker __all__ = ['CeleryCommand', 'main'] @@ -58,7 +59,7 @@ ] if DEBUG: # pragma: no cover command_classes.append( - ('Debug', ['graph'], 'red'), + ('Debug', ['graph', 'logtool'], 'red'), ) @@ -670,6 +671,7 @@ class CeleryCommand(Command): 'help': help, 'inspect': inspect, 'list': list_, + 'logtool': logtool, 'migrate': migrate, 'multi': multi, 'purge': purge, diff --git a/celery/bin/logtool.py b/celery/bin/logtool.py index 73cf9460f..872f64ec9 100644 --- a/celery/bin/logtool.py +++ b/celery/bin/logtool.py @@ -14,9 +14,6 @@ from collections import Counter from fileinput import FileInput -from celery.datastructures import DependencyGraph, GraphFormatter -from celery.five import items - from .base import Command __all__ = ['logtool'] @@ -48,6 +45,7 @@ class _task_counts(list): def format(self): return '\n'.join('{0}: {1}'.format(*i) for i in self) + def task_info(line): m = RE_TASK_INFO.match(line) return m.groups() @@ -143,7 +141,9 @@ def run(self, what=None, *files, **kwargs): if not what: raise self.UsageError('missing action') elif what not in map: - raise self.Error('no action {0} in {1}'.format(what, '|'.join(map))) + raise self.Error( + 'action {0} not in {1}'.format(what, '|'.join(map)), + ) return map[what](files) From fbb142e367fb271af21bd386ece2df55d262a2d3 Mon Sep 17 00:00:00 2001 From: Balthazar Rouberol Date: Wed, 22 Oct 2014 16:54:55 +0100 Subject: [PATCH 0359/1103] Specify the capabilities offered by Flower HTTP API Signed-off-by: Balthazar Rouberol --- docs/userguide/monitoring.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index 6b85ae328..55c31b929 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -206,6 +206,25 @@ Features - Revoke or terminate tasks - HTTP API + + - List workers + - Shut down a worker + - Restart worker’s pool + - Grow worker’s pool + - Shrink worker’s pool + - Autoscale worker pool + - Start consuming from a queue + - Stop consuming from a queue + - List tasks + - List (seen) task types + - Get a task info + - Execute a task + - Execute a task by name + - Get a task result + - Change soft and hard time limits for a task + - Change rate limit for a task + - Revoke a task + - OpenID authentication **Screenshots** From c998f50cc58809718d09780439fe94bb3fa0252b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 22 Oct 2014 17:28:21 +0100 Subject: [PATCH 0360/1103] Consistent decoding of exception result --- celery/backends/amqp.py | 6 ++---- celery/backends/base.py | 18 ++++++++++-------- celery/backends/redis.py | 2 +- celery/canvas.py | 3 ++- celery/result.py | 6 +----- docs/getting-started/next-steps.rst | 2 +- docs/userguide/canvas.rst | 2 +- 7 files changed, 18 insertions(+), 21 deletions(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index 96d8a3722..a19dd3362 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -256,14 +256,12 @@ def get_many(self, task_ids, timeout=None, no_ack=True, results = deque() push_result = results.append push_cache = self._cache.__setitem__ - to_exception = self.exception_to_python + decode_result = self.decode_result def on_message(message): - body = message.decode() + body = decode_result(message.body) state, uid = getfields(body) if state in READY_STATES: - if state in PROPAGATE_STATES: - body['result'] = to_exception(body['result']) push_result(body) \ if uid in task_ids else push_cache(uid, body) diff --git a/celery/backends/base.py b/celery/backends/base.py index d22e633d5..eee1d7591 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -181,6 +181,12 @@ def encode(self, data): _, _, payload = dumps(data, serializer=self.serializer) return payload + def decode_result(self, payload): + meta = self.decode(payload) + if meta['status'] in self.EXCEPTION_STATES: + meta['result'] = self.exception_to_python(meta['result']) + return meta + def decode(self, payload): payload = PY3 and payload or str(payload) return loads(payload, @@ -264,11 +270,7 @@ def get_traceback(self, task_id): def get_result(self, task_id): """Get the result of a task.""" - meta = self.get_task_meta(task_id) - if meta['status'] in self.EXCEPTION_STATES: - return self.exception_to_python(meta['result']) - else: - return meta['result'] + return self.get_task_meta(task_id).get('result') def get_children(self, task_id): """Get the list of subtasks sent by a task.""" @@ -436,13 +438,13 @@ def _mget_to_results(self, values, keys): if hasattr(values, 'items'): # client returns dict so mapping preserved. return { - self._strip_prefix(k): self.decode(v) + self._strip_prefix(k): self.decode_result(v) for k, v in items(values) if v is not None } else: # client returns list so need to recreate mapping. return { - bytes_to_str(keys[i]): self.decode(value) + bytes_to_str(keys[i]): self.decode_result(value) for i, value in enumerate(values) if value is not None } @@ -500,7 +502,7 @@ def _get_task_meta_for(self, task_id): meta = self.get(self.get_key_for_task(task_id)) if not meta: return {'status': states.PENDING, 'result': None} - return self.decode(meta) + return self.decode_result(meta) def _restore_group(self, group_id): """Get task metadata for a task by id.""" diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 2df9e8e0a..3e76513db 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -221,7 +221,7 @@ def _new_chord_return(self, task, state, result, propagate=None, callback = maybe_signature(request.chord, app=app) total = callback['chord_size'] + totaldiff if readycount == total: - decode, unpack = self.decode, self._unpack_chord_result + decode, unpack = self.decode_result, self._unpack_chord_result resl, _, _ = client.pipeline() \ .lrange(jkey, 0, total) \ .delete(jkey) \ diff --git a/celery/canvas.py b/celery/canvas.py index 20c3fecf7..c4e2f1b4b 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -618,7 +618,8 @@ def _prepared(self, tasks, partial_args, group_id, root_id, app, dict=dict, task.args = tuple(partial_args) + tuple(task.args) yield task, task.freeze(group_id=group_id, root_id=root_id) - def _apply_tasks(self, tasks, producer=None, app=None, **options): + def _apply_tasks(self, tasks, producer=None, app=None, + add_to_parent=None, **options): app = app or self.app with app.producer_or_acquire(producer) as producer: for sig, res in tasks: diff --git a/celery/result.py b/celery/result.py index 902dea700..ad5a8158d 100644 --- a/celery/result.py +++ b/celery/result.py @@ -174,9 +174,7 @@ def get(self, timeout=None, propagate=True, interval=0.5, self._maybe_set_cache(meta) status = meta['status'] if status in PROPAGATE_STATES and propagate: - raise self.backend.exception_to_python(meta['result']) - if status in EXCEPTION_STATES: - return self.backend.exception_to_python(meta['result']) + raise meta['result'] return meta['result'] wait = get # deprecated alias to :meth:`get`. @@ -345,8 +343,6 @@ def _get_task_meta(self): def _set_cache(self, d): state, children = d['status'], d.get('children') - if state in states.EXCEPTION_STATES: - d['result'] = self.backend.exception_to_python(d['result']) if children: d['children'] = [ result_from_tuple(child, self.app) for child in children diff --git a/docs/getting-started/next-steps.rst b/docs/getting-started/next-steps.rst index b2ea6132d..b6a49a72f 100644 --- a/docs/getting-started/next-steps.rst +++ b/docs/getting-started/next-steps.rst @@ -317,7 +317,7 @@ exception, in fact ``result.get()`` will propagate any errors by default:: File "/opt/devel/celery/celery/result.py", line 113, in get interval=interval) File "/opt/devel/celery/celery/backends/amqp.py", line 138, in wait_for - raise self.exception_to_python(meta['result']) + raise meta['result'] TypeError: add() takes exactly 2 arguments (1 given) If you don't wish for the errors to propagate then you can disable that diff --git a/docs/userguide/canvas.rst b/docs/userguide/canvas.rst index f285f6755..4ba43d842 100644 --- a/docs/userguide/canvas.rst +++ b/docs/userguide/canvas.rst @@ -735,7 +735,7 @@ to the :exc:`~@ChordError` exception: File "*/celery/result.py", line 120, in get interval=interval) File "*/celery/backends/amqp.py", line 150, in wait_for - raise self.exception_to_python(meta['result']) + raise meta['result'] celery.exceptions.ChordError: Dependency 97de6f3f-ea67-4517-a21c-d867c61fcb47 raised ValueError('something something',) From 36031b9679c54bb3cc1ef57999875e4f8d6460f0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 22 Oct 2014 17:35:40 +0100 Subject: [PATCH 0361/1103] Chord does not include self.args. Closes #2299 --- celery/canvas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index c4e2f1b4b..02d1ba60d 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -376,7 +376,7 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, task_id=None, link=None, link_error=None, publisher=None, producer=None, root_id=None, app=None, **options): app = app or self.app - args = tuple(args) + tuple(self.args) + args = tuple(args) + tuple(self.args) if args else self.args tasks, results = self.prepare_steps( args, self.tasks, root_id, link_error, ) @@ -778,6 +778,7 @@ def _get_app(self, body=None): def apply_async(self, args=(), kwargs={}, task_id=None, producer=None, publisher=None, connection=None, router=None, result_cls=None, **options): + args = tuple(args) + tuple(self.args) if args else self.args body = kwargs.get('body') or self.kwargs['body'] kwargs = dict(self.kwargs, **kwargs) body = body.clone(**options) From 4f3234e8894d1e3c515e6fe4f34b2a1f4cf40d14 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 22 Oct 2014 17:39:32 +0100 Subject: [PATCH 0362/1103] Immutable chain/chord --- celery/canvas.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 02d1ba60d..5b58004cd 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -376,7 +376,8 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, task_id=None, link=None, link_error=None, publisher=None, producer=None, root_id=None, app=None, **options): app = app or self.app - args = tuple(args) + tuple(self.args) if args else self.args + args = (tuple(args) + tuple(self.args) + if args and not self.immutable else self.args) tasks, results = self.prepare_steps( args, self.tasks, root_id, link_error, ) @@ -778,7 +779,8 @@ def _get_app(self, body=None): def apply_async(self, args=(), kwargs={}, task_id=None, producer=None, publisher=None, connection=None, router=None, result_cls=None, **options): - args = tuple(args) + tuple(self.args) if args else self.args + args = (tuple(args) + tuple(self.args) + if args and not self.immutable else self.args) body = kwargs.get('body') or self.kwargs['body'] kwargs = dict(self.kwargs, **kwargs) body = body.clone(**options) From ca3ee9ec058663a622d124ee622c82d60141c770 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 22 Oct 2014 17:47:03 +0100 Subject: [PATCH 0363/1103] Tests passing --- celery/backends/amqp.py | 4 ++-- celery/backends/base.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index a19dd3362..5587943e3 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -256,10 +256,10 @@ def get_many(self, task_ids, timeout=None, no_ack=True, results = deque() push_result = results.append push_cache = self._cache.__setitem__ - decode_result = self.decode_result + decode_result = self.meta_from_decoded def on_message(message): - body = decode_result(message.body) + body = decode_result(message.decode()) state, uid = getfields(body) if state in READY_STATES: push_result(body) \ diff --git a/celery/backends/base.py b/celery/backends/base.py index eee1d7591..cdb8fc217 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -181,12 +181,14 @@ def encode(self, data): _, _, payload = dumps(data, serializer=self.serializer) return payload - def decode_result(self, payload): - meta = self.decode(payload) + def meta_from_decoded(self, meta): if meta['status'] in self.EXCEPTION_STATES: meta['result'] = self.exception_to_python(meta['result']) return meta + def decode_result(self, payload): + return self.meta_from_decoded(self.decode(payload)) + def decode(self, payload): payload = PY3 and payload or str(payload) return loads(payload, From dec6cbdefc367fad995cc1b6fc987e1ba8206b17 Mon Sep 17 00:00:00 2001 From: llllllllll Date: Wed, 22 Oct 2014 22:56:45 -0400 Subject: [PATCH 0364/1103] BUG: The call to Pdb.set_trace occured in a conteext manager making rdb stop in contextlib, not in the frame given. The ignore_errno context manager there didn't suppress anything as Pdb.set_trace does not raise an ECONNRESET in any case. After the removal, the method was the same as Pdb.set_trace, so this just removes Rdb's override. The test was updated to reflect the fact that Pdb.set_trace will not raise a ECONNRESET. --- celery/contrib/rdb.py | 7 ------- celery/tests/contrib/test_rdb.py | 2 -- 2 files changed, 9 deletions(-) diff --git a/celery/contrib/rdb.py b/celery/contrib/rdb.py index 8435ec31b..99edb64d7 100644 --- a/celery/contrib/rdb.py +++ b/celery/contrib/rdb.py @@ -46,7 +46,6 @@ def add(x, y): from billiard import current_process from celery.five import range -from celery.platforms import ignore_errno __all__ = ['CELERY_RDB_HOST', 'CELERY_RDB_PORT', 'default_port', 'Rdb', 'debugger', 'set_trace'] @@ -153,12 +152,6 @@ def do_quit(self, arg): return 1 do_q = do_exit = do_quit - def set_trace(self, frame=None): - if frame is None: - frame = _frame().f_back - with ignore_errno(errno.ECONNRESET): - Pdb.set_trace(self, frame) - def set_quit(self): # this raises a BdbQuit exception that we are unable to catch. sys.settrace(None) diff --git a/celery/tests/contrib/test_rdb.py b/celery/tests/contrib/test_rdb.py index a933c6010..1fa398b81 100644 --- a/celery/tests/contrib/test_rdb.py +++ b/celery/tests/contrib/test_rdb.py @@ -52,8 +52,6 @@ def test_rdb(self, get_avail_port): rdb.set_trace() rdb.set_trace(Mock()) pset.side_effect = SockErr - pset.side_effect.errno = errno.ECONNRESET - rdb.set_trace() pset.side_effect.errno = errno.ENOENT with self.assertRaises(SockErr): rdb.set_trace() From 6df0f36f5b2d358cb3a3aa4fca07a8c275c454ce Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 23 Oct 2014 14:23:21 +0100 Subject: [PATCH 0365/1103] Fixes documentation build --- celery/app/task.py | 2 +- celery/task/base.py | 2 +- docs/_ext/celerydocs.py | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index e8d1eafb0..8499f5fab 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -432,7 +432,7 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, :keyword link_error: A single, or a list of tasks to apply if an error occurs while executing the task. - :keyword producer: :class:`~@kombu.Producer` instance to use. + :keyword producer: :class:`kombu.Producer` instance to use. :keyword add_to_parent: If set to True (default) and the task is applied while executing another task, then the result will be appended to the parent tasks ``request.children`` diff --git a/celery/task/base.py b/celery/task/base.py index 35f8877ad..27f72408b 100644 --- a/celery/task/base.py +++ b/celery/task/base.py @@ -201,7 +201,7 @@ def get_publisher(self, connection=None, exchange=None, exchange_type=None, **options): """Deprecated method to get the task publisher (now called producer). - Should be replaced with :class:`@kombu.Producer`: + Should be replaced with :class:`kombu.Producer`: .. code-block:: python diff --git a/docs/_ext/celerydocs.py b/docs/_ext/celerydocs.py index e89462aa6..a0d667800 100644 --- a/docs/_ext/celerydocs.py +++ b/docs/_ext/celerydocs.py @@ -65,14 +65,16 @@ def shorten(S, newtarget, src_dict): return S -def get_abbr(pre, rest, type): +def get_abbr(pre, rest, type, orig=None): if pre: for d in APPATTRS, ABBRS: try: return d[pre], rest, d except KeyError: pass - raise KeyError(pre) + raise KeyError('Unknown abbreviation: {0} ({1})'.format( + '.'.join([pre, rest]) if orig is None else orig, type, + )) else: for d in APPATTRS, ABBRS: try: @@ -83,6 +85,7 @@ def get_abbr(pre, rest, type): def resolve(S, type): + orig = S if S.startswith('@'): S = S.lstrip('@-') try: @@ -90,7 +93,7 @@ def resolve(S, type): except ValueError: pre, rest = '', S - target, rest, src = get_abbr(pre, rest, type) + target, rest, src = get_abbr(pre, rest, type, orig) return '.'.join([target, rest]) if rest else target, src return S, None From cc0063123aa13b34a79a5b4aa014ed89e41bdb64 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 23 Oct 2014 14:39:58 +0100 Subject: [PATCH 0366/1103] Template 3.2 news document --- docs/index.rst | 1 + docs/whatsnew-3.2.rst | 104 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 docs/whatsnew-3.2.rst diff --git a/docs/index.rst b/docs/index.rst index 86e47949b..7d2c32381 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -49,6 +49,7 @@ Contents tutorials/index faq changelog + whatsnew-3.2 whatsnew-3.1 whatsnew-3.0 whatsnew-2.5 diff --git a/docs/whatsnew-3.2.rst b/docs/whatsnew-3.2.rst new file mode 100644 index 000000000..ffe60f796 --- /dev/null +++ b/docs/whatsnew-3.2.rst @@ -0,0 +1,104 @@ +.. _whatsnew-3.1: + +=========================================== + What's new in Celery 3.2 (TBA) +=========================================== +:Author: Ask Solem (ask at celeryproject.org) + +.. sidebar:: Change history + + What's new documents describe the changes in major versions, + we also have a :ref:`changelog` that lists the changes in bugfix + releases (0.0.x), while older series are archived under the :ref:`history` + section. + +Celery is a simple, flexible and reliable distributed system to +process vast amounts of messages, while providing operations with +the tools required to maintain such a system. + +It's a task queue with focus on real-time processing, while also +supporting task scheduling. + +Celery has a large and diverse community of users and contributors, +you should come join us :ref:`on IRC ` +or :ref:`our mailing-list `. + +To read more about Celery you should go read the :ref:`introduction `. + +While this version is backward compatible with previous versions +it's important that you read the following section. + +This version is officially supported on CPython 2.6, 2.7 and 3.3, +and also supported on PyPy. + +.. _`website`: http://celeryproject.org/ + +.. topic:: Table of Contents + + Make sure you read the important notes before upgrading to this version. + +.. contents:: + :local: + :depth: 2 + +Preface +======= + + +.. _v320-important: + +Important Notes +=============== + +Dropped support for Python 2.6 +------------------------------ + +Celery now requires Python 2.7 or later. + +JSON is now the default serializer +---------------------------------- + + +.. _v320-news: + +News +==== + +Item 1 +------ + +Bla bla + +- blah blah + +In Other News +------------- + +- Now depends on :ref:`Kombu 3.1 `. + +- Now depends on :mod:`billiard` version 3.4. + + +.. _v320-removals: + +Scheduled Removals +================== + +.. _v320-deprecations: + +Deprecations +============ + +See the :ref:`deprecation-timeline`. + +.. _v320-fixes: + +Fixes +===== + +.. _v320-internal: + +Internal changes +================ + +- Module ``celery.worker.job`` has been renamed to :mod:`celery.worker.request`. From 7491cdc2dbce77cb2a9ea2ee6e04bbf39e03c911 Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Thu, 23 Oct 2014 09:55:37 -0400 Subject: [PATCH 0367/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index c4f830520..222284f77 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -169,3 +169,4 @@ Akexander Koshelev, 2014/08/19 Davide Quarta, 2014/08/19 John Whitlock, 2014/08/19 Konstantinos Koukopoulos, 2014/08/24 +Joe Jevnik, 2014/10/22 From bc18d0859c1570f5eb59f5a969d1d32c63af764b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 23 Oct 2014 18:59:15 +0100 Subject: [PATCH 0368/1103] Adds app.add_periodic_task --- celery/app/base.py | 83 ++++++++++++++++++++++--------- docs/_ext/celerydocs.py | 1 + docs/userguide/periodic-tasks.rst | 41 +++++++++++++-- funtests/stress/stress/app.py | 4 ++ 4 files changed, 100 insertions(+), 29 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 70cbca501..8d2950564 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -118,6 +118,7 @@ class Celery(object): registry_cls = TaskRegistry _fixups = None _pool = None + _conf = None builtin_fixups = BUILTIN_FIXUPS #: Signal sent when app is loading configuration. @@ -154,6 +155,7 @@ def __init__(self, main=None, loader=None, backend=None, self.configured = False self._config_source = config_source self._pending_defaults = deque() + self._pending_periodic_tasks = deque() self.finalized = False self._finalize_mutex = threading.Lock() @@ -311,13 +313,13 @@ def add_defaults(self, fun): if not callable(fun): d, fun = fun, lambda: d if self.configured: - return self.conf.add_defaults(fun()) + return self._conf.add_defaults(fun()) self._pending_defaults.append(fun) def config_from_object(self, obj, silent=False, force=False): self._config_source = obj if force or self.configured: - del(self.conf) + self._conf = None return self.loader.config_from_object(obj, silent=silent) def config_from_envvar(self, variable_name, silent=False, force=False): @@ -330,7 +332,9 @@ def config_from_envvar(self, variable_name, silent=False, force=False): return self.config_from_object(module_name, silent=silent, force=force) def config_from_cmdline(self, argv, namespace='celery'): - self.conf.update(self.loader.cmdline_config_parser(argv, namespace)) + (self._conf if self.configured else self.conf).update( + self.loader.cmdline_config_parser(argv, namespace) + ) def setup_security(self, allowed_serializers=None, key=None, cert=None, store=None, digest='sha1', serializer='json'): @@ -440,18 +444,19 @@ def now(self): return self.loader.now(utc=self.conf.CELERY_ENABLE_UTC) def mail_admins(self, subject, body, fail_silently=False): - if self.conf.ADMINS: - to = [admin_email for _, admin_email in self.conf.ADMINS] + conf = self.conf + if conf.ADMINS: + to = [admin_email for _, admin_email in conf.ADMINS] return self.loader.mail_admins( subject, body, fail_silently, to=to, - sender=self.conf.SERVER_EMAIL, - host=self.conf.EMAIL_HOST, - port=self.conf.EMAIL_PORT, - user=self.conf.EMAIL_HOST_USER, - password=self.conf.EMAIL_HOST_PASSWORD, - timeout=self.conf.EMAIL_TIMEOUT, - use_ssl=self.conf.EMAIL_USE_SSL, - use_tls=self.conf.EMAIL_USE_TLS, + sender=conf.SERVER_EMAIL, + host=conf.EMAIL_HOST, + port=conf.EMAIL_PORT, + user=conf.EMAIL_HOST_USER, + password=conf.EMAIL_HOST_PASSWORD, + timeout=conf.EMAIL_TIMEOUT, + use_ssl=conf.EMAIL_USE_SSL, + use_tls=conf.EMAIL_USE_TLS, ) def select_queues(self, queues=None): @@ -472,7 +477,7 @@ def _get_backend(self): self.loader) return backend(app=self, url=url) - def _get_config(self): + def _load_config(self): if isinstance(self.on_configure, Signal): self.on_configure.send(sender=self) else: @@ -482,12 +487,19 @@ def _get_config(self): self.loader.config_from_object(self._config_source) defaults = dict(deepcopy(DEFAULTS), **self._preconf) self.configured = True - s = Settings({}, [self.prepare_config(self.loader.conf), - defaults]) + s = self._conf = Settings( + {}, [self.prepare_config(self.loader.conf), defaults], + ) # load lazy config dict initializers. - pending = self._pending_defaults - while pending: - s.add_defaults(maybe_evaluate(pending.popleft()())) + pending_def = self._pending_defaults + while pending_def: + s.add_defaults(maybe_evaluate(pending_def.popleft()())) + + # load lazy periodic tasks + pending_beat = self._pending_periodic_tasks + while pending_beat: + pargs, pkwargs = pending_beat.popleft() + self._add_periodic_task(*pargs, **pkwargs) self.on_after_configure.send(sender=self, source=s) return s @@ -507,6 +519,27 @@ def signature(self, *args, **kwargs): kwargs['app'] = self return self.canvas.signature(*args, **kwargs) + def add_periodic_task(self, *args, **kwargs): + if not self.configured: + return self._pending_periodic_tasks.append((args, kwargs)) + return self._add_periodic_task(*args, **kwargs) + + def _add_periodic_task(self, schedule, sig, + args=(), kwargs={}, name=None, **opts): + from .task import Task + + sig = (self.signature(sig.name, args, kwargs) + if isinstance(sig, Task) else sig.clone(args, kwargs)) + + name = name or ':'.join([sig.name, ','.join(map(str, sig.args))]) + self._conf.CELERYBEAT_SCHEDULE[name] = { + 'schedule': schedule, + 'task': sig.name, + 'args': sig.args, + 'kwargs': sig.kwargs, + 'options': dict(sig.options, **opts), + } + def create_task_cls(self): """Creates a base task class using default configuration taken from this app.""" @@ -568,7 +601,7 @@ def __reduce_keys__(self): when unpickling.""" return { 'main': self.main, - 'changes': self.conf.changes, + 'changes': self._conf.changes if self._conf else {}, 'loader': self.loader_cls, 'backend': self.backend_cls, 'amqp': self.amqp_cls, @@ -582,7 +615,7 @@ def __reduce_keys__(self): def __reduce_args__(self): """Deprecated method, please use :meth:`__reduce_keys__` instead.""" - return (self.main, self.conf.changes, + return (self.main, self._conf.changes if self._conf else {}, self.loader_cls, self.backend_cls, self.amqp_cls, self.events_cls, self.log_cls, self.control_cls, False, self._config_source) @@ -653,9 +686,11 @@ def amqp(self): def backend(self): return self._get_backend() - @cached_property + @property def conf(self): - return self._get_config() + if self._conf is None: + self._load_config() + return self._conf @cached_property def control(self): @@ -691,5 +726,5 @@ def timezone(self): if not tz: return (timezone.get_timezone('UTC') if conf.CELERY_ENABLE_UTC else timezone.local) - return timezone.get_timezone(self.conf.CELERY_TIMEZONE) + return timezone.get_timezone(conf.CELERY_TIMEZONE) App = Celery # compat diff --git a/docs/_ext/celerydocs.py b/docs/_ext/celerydocs.py index a0d667800..d2c170c08 100644 --- a/docs/_ext/celerydocs.py +++ b/docs/_ext/celerydocs.py @@ -34,6 +34,7 @@ 'select_queues', 'either', 'bugreport', 'create_task_cls', 'subclass_with_self', 'annotations', 'current_task', 'oid', 'timezone', '__reduce_keys__', 'fixups', 'finalized', 'configured', + 'add_periodic_task', 'autofinalize', 'steps', 'user_options', 'main', 'clock', } diff --git a/docs/userguide/periodic-tasks.rst b/docs/userguide/periodic-tasks.rst index b539e1f71..7a8293a2a 100644 --- a/docs/userguide/periodic-tasks.rst +++ b/docs/userguide/periodic-tasks.rst @@ -74,19 +74,50 @@ schedule manually. Entries ======= -To schedule a task periodically you have to add an entry to the -:setting:`CELERYBEAT_SCHEDULE` setting. +To call a task periodically you have to add an entry to the +beat schedule list. + +.. code-block:: python + + from celery import Celery + from celery.schedules import crontab + + app = Celery() + + @app.on_after_configure.connect + def setup_periodic_tasks(sender, **kwargs): + # Calls test('hello') every 10 seconds. + sender.add_periodic_task(10.0, test.s('hello'), name='add every 10') + + # Calls test('world') every 30 seconds + sender.add_periodic_task(30.0, test.s('world'), expires=10) + + # Executes every Monday morning at 7:30 A.M + sender.add_periodic_task( + crontab(hour=7, minute=30, day_of_week=1), + test.s('Happy Mondays!'), + ) + + @app.task + def test(arg): + print(arg) + + +Setting these up from within the ``on_after_configure`` handler means +that we will not evaluate the app at module level when using ``test.s()``. + +The `@add_periodic_task` function will add the entry to the +:setting:`CELERYBEAT_SCHEDULE` setting behind the scenes, which also +can be used to set up periodic tasks manually: Example: Run the `tasks.add` task every 30 seconds. .. code-block:: python - from datetime import timedelta - CELERYBEAT_SCHEDULE = { 'add-every-30-seconds': { 'task': 'tasks.add', - 'schedule': timedelta(seconds=30), + 'schedule': 30.0, 'args': (16, 16) }, } diff --git a/funtests/stress/stress/app.py b/funtests/stress/stress/app.py index cd170bff8..c26481f65 100644 --- a/funtests/stress/stress/app.py +++ b/funtests/stress/stress/app.py @@ -149,3 +149,7 @@ def marker(s, sep='-'): return _marker.delay(s, sep) except Exception as exc: print("Retrying marker.delay(). It failed to start: %s" % exc) + +@app.on_after_configure.connect +def setup_periodic_tasks(sender, **kwargs): + sender.add_periodic_task(10, add.s(2, 2), expires=10) From 6e71620c59c730e2ba5f35f192df23756b608024 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 23 Oct 2014 20:51:21 +0100 Subject: [PATCH 0369/1103] Tests passing --- celery/app/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/celery/app/base.py b/celery/app/base.py index 8d2950564..8ff3ceb34 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -692,6 +692,10 @@ def conf(self): self._load_config() return self._conf + @conf.setter + def conf(self, d): # noqa + self._conf = d + @cached_property def control(self): return instantiate(self.control_cls, app=self) From 289ec8e003d3d271a4fae32cb3da1a4e793b143b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 24 Oct 2014 13:41:43 +0100 Subject: [PATCH 0370/1103] Revokes: Make sure heap does not contain duplicates. Closes #2336 --- celery/datastructures.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/celery/datastructures.py b/celery/datastructures.py index 1551ab861..84c393c9f 100644 --- a/celery/datastructures.py +++ b/celery/datastructures.py @@ -12,7 +12,7 @@ import time from collections import defaultdict, Mapping, MutableMapping, MutableSet -from heapq import heappush, heappop +from heapq import heapify, heappush, heappop from functools import partial from itertools import chain @@ -555,7 +555,7 @@ class LimitedSet(object): """Kind-of Set with limitations. Good for when you need to test for membership (`a in set`), - but the list might become too big. + but the set should not grow unbounded. :keyword maxlen: Maximum number of members before we start evicting expired members. @@ -564,24 +564,31 @@ class LimitedSet(object): """ def __init__(self, maxlen=None, expires=None, data=None, heap=None): + # heap is ignored self.maxlen = maxlen self.expires = expires self._data = {} if data is None else data - self._heap = [] if heap is None else heap + self._heap = [] + # make shortcuts self.__len__ = self._heap.__len__ - self.__iter__ = self._heap.__iter__ self.__contains__ = self._data.__contains__ - def add(self, value, now=time.time): + self._refresh_heap() + + def _refresh_heap(self): + self._heap[:] = [(t, key) for key, t in items(self._data)] + heapify(self._heap) + + def add(self, key, now=time.time, heappush=heappush): """Add a new member.""" # offset is there to modify the length of the list, # this way we can expire an item before inserting the value, - # and it will end up in correct order. + # and it will end up in the correct order. self.purge(1, offset=1) inserted = now() - self._data[value] = inserted - heappush(self._heap, (inserted, value)) + self._data[key] = inserted + heappush(self._heap, (inserted, key)) def clear(self): """Remove all members""" @@ -630,11 +637,10 @@ def purge(self, limit=None, offset=0, now=time.time): pass i += 1 - def update(self, other, heappush=heappush): + def update(self, other): if isinstance(other, LimitedSet): self._data.update(other._data) - self._heap.extend(other._heap) - self._heap.sort() + self._refresh_heap() else: for obj in other: self.add(obj) @@ -661,7 +667,5 @@ def __contains__(self, key): return key in self._data def __reduce__(self): - return self.__class__, ( - self.maxlen, self.expires, self._data, self._heap, - ) + return self.__class__, (self.maxlen, self.expires, self._data) MutableSet.register(LimitedSet) From e5ff4e5265be72d5abaf0a8e25d2e2de7eb9a45d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 24 Oct 2014 17:01:24 +0100 Subject: [PATCH 0371/1103] Rest of result backends must handle encoded exception --- celery/backends/cassandra.py | 4 ++-- celery/backends/mongodb.py | 4 ++-- funtests/stress/stress/templates.py | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index adb70afc7..fb0362d52 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -170,14 +170,14 @@ def _do_get(): meta['task_id'] = task_id else: obj = cf.get(task_id) - meta = { + meta = self.meta_from_decoded({ 'task_id': task_id, 'status': obj['status'], 'result': self.decode(obj['result']), 'date_done': obj['date_done'], 'traceback': self.decode(obj['traceback']), 'children': self.decode(obj['children']), - } + }) except (KeyError, pycassa.NotFoundException): meta = {'status': states.PENDING, 'result': None} return meta diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 27f8fa3cc..581b93dd7 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -174,14 +174,14 @@ def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" obj = self.collection.find_one({'_id': task_id}) if obj: - return { + return self.meta_from_decoded({ 'task_id': obj['_id'], 'status': obj['status'], 'result': self.decode(obj['result']), 'date_done': obj['date_done'], 'traceback': self.decode(obj['traceback']), 'children': self.decode(obj['children']), - } + }) return {'status': states.PENDING, 'result': None} def _save_group(self, group_id, result): diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index 0920319d1..96dd2aa90 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -50,6 +50,7 @@ def template_names(): @template() class default(object): + BROKER_HEARTBEAT=2 CELERY_ACCEPT_CONTENT = ['json'] CELERY_DEFAULT_QUEUE = CSTRESS_QUEUE CELERY_TASK_SERIALIZER = 'json' From 8f8652e7c4561d3a51f9a1d25ca02bb5971088ff Mon Sep 17 00:00:00 2001 From: Andrea Rabbaglietti Date: Thu, 2 Oct 2014 15:52:00 +0200 Subject: [PATCH 0372/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index c4f830520..5c7f7bbbe 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -169,3 +169,4 @@ Akexander Koshelev, 2014/08/19 Davide Quarta, 2014/08/19 John Whitlock, 2014/08/19 Konstantinos Koukopoulos, 2014/08/24 +Andrea Rabbaglietti, 2014/10/02 From 725c28ba892b806796bf2e9de727f02eb03fda6b Mon Sep 17 00:00:00 2001 From: Mitchel Humpherys Date: Mon, 29 Sep 2014 21:44:17 -0700 Subject: [PATCH 0373/1103] Remove reference to celeryd-django The `celeryd-django` supervisord configuration was removed in aad31dcec1d65fbd8. Remove a dangling reference to the now non-existent file. --- extra/supervisord/supervisord.conf | 6 ------ 1 file changed, 6 deletions(-) diff --git a/extra/supervisord/supervisord.conf b/extra/supervisord/supervisord.conf index 26e5fcbff..1bde65a78 100644 --- a/extra/supervisord/supervisord.conf +++ b/extra/supervisord/supervisord.conf @@ -26,9 +26,3 @@ serverurl=unix:///tmp/supervisor.sock ; use unix:// schem for a unix sockets. # Uncomment this line for celeryd for Python ;files=celeryd.conf -# Uncomment this line for celeryd for Django. -;files=django/celeryd.conf - - - - From a399bd77134633f9304d7092cd65a1973608301b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 28 Oct 2014 18:15:03 +0000 Subject: [PATCH 0374/1103] Cosmetics for Issue #2253 --- celery/backends/amqp.py | 4 -- celery/backends/base.py | 7 ++- celery/backends/cassandra.py | 17 +++--- celery/backends/couchbase.py | 4 -- celery/backends/couchdb.py | 85 ++++++++++----------------- celery/backends/database/__init__.py | 8 +-- celery/backends/mongodb.py | 7 +-- celery/backends/redis.py | 5 +- celery/backends/riak.py | 4 -- celery/tests/backends/test_couchdb.py | 52 ++++------------ docs/configuration.rst | 19 ++---- 11 files changed, 68 insertions(+), 144 deletions(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index 5587943e3..5111d5936 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -80,10 +80,6 @@ def __init__(self, app, connection=None, exchange=None, exchange_type=None, ) self.serializer = serializer or conf.CELERY_RESULT_SERIALIZER self.auto_delete = auto_delete - - self.expires = None - if 'expires' not in kwargs or kwargs['expires'] is not None: - self.expires = self.prepare_expires(kwargs.get('expires')) self.queue_arguments = dictfilter({ 'x-expires': maybe_s_to_ms(self.expires), }) diff --git a/celery/backends/base.py b/celery/backends/base.py index cdb8fc217..a802bb1cf 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -91,8 +91,9 @@ class BaseBackend(object): 'interval_max': 1, } - def __init__(self, app, serializer=None, - max_cached_results=None, accept=None, **kwargs): + def __init__(self, app, + serializer=None, max_cached_results=None, accept=None, + expires=None, expires_type=None, **kwargs): self.app = app conf = self.app.conf self.serializer = serializer or conf.CELERY_RESULT_SERIALIZER @@ -101,6 +102,8 @@ def __init__(self, app, serializer=None, self.encoder) = serializer_registry._encoders[self.serializer] cmax = max_cached_results or conf.CELERY_MAX_CACHED_RESULTS self._cache = _nulldict() if cmax == -1 else LRUCache(limit=cmax) + + self.expires = self.prepare_expires(expires, expires_type) self.accept = prepare_accept_content( conf.CELERY_ACCEPT_CONTENT if accept is None else accept, ) diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index fb0362d52..8d1b80181 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -22,7 +22,6 @@ from celery.exceptions import ImproperlyConfigured from celery.five import monotonic from celery.utils.log import get_logger -from celery.utils.timeutils import maybe_timedelta from .base import BaseBackend @@ -60,9 +59,6 @@ def __init__(self, servers=None, keyspace=None, column_family=None, """ super(CassandraBackend, self).__init__(**kwargs) - self.expires = kwargs.get('expires') or maybe_timedelta( - self.app.conf.CELERY_TASK_RESULT_EXPIRES) - if not pycassa: raise ImproperlyConfigured( 'You need to install the pycassa library to use the ' @@ -140,21 +136,22 @@ def _store_result(self, task_id, result, status, """Store return value and status of an executed task.""" def _do_store(): + detailed = self.detailed_mode cf = self._get_column_family() date_done = self.app.now() meta = {'status': status, 'date_done': date_done.strftime('%Y-%m-%dT%H:%M:%SZ'), 'traceback': self.encode(traceback), + 'result': result if detailed else self.encode(result), 'children': self.encode( self.current_task_children(request), )} - ttl = self.expires and max(self.expires.total_seconds(), 0) - if self.detailed_mode: - meta['result'] = result - cf.insert(task_id, {date_done: self.encode(meta)}, ttl=ttl) + if detailed: + cf.insert( + task_id, {date_done: self.encode(meta)}, ttl=self.expires, + ) else: - meta['result'] = self.encode(result) - cf.insert(task_id, meta, ttl=ttl) + cf.insert(task_id, meta, ttl=self.expires) return self._retry_on_error(_do_store) diff --git a/celery/backends/couchbase.py b/celery/backends/couchbase.py index 2d51b8001..9381fcfc6 100644 --- a/celery/backends/couchbase.py +++ b/celery/backends/couchbase.py @@ -20,7 +20,6 @@ from kombu.utils.url import _parse_url from celery.exceptions import ImproperlyConfigured -from celery.utils.timeutils import maybe_timedelta from .base import KeyValueStoreBackend @@ -49,9 +48,6 @@ def __init__(self, url=None, *args, **kwargs): """ super(CouchBaseBackend, self).__init__(*args, **kwargs) - self.expires = kwargs.get('expires') or maybe_timedelta( - self.app.conf.CELERY_TASK_RESULT_EXPIRES) - if Couchbase is None: raise ImproperlyConfigured( 'You need to install the couchbase library to use the ' diff --git a/celery/backends/couchdb.py b/celery/backends/couchdb.py index 0d2a68d4b..f1a3ebde5 100644 --- a/celery/backends/couchdb.py +++ b/celery/backends/couchdb.py @@ -8,8 +8,6 @@ """ from __future__ import absolute_import -import logging - try: import pycouchdb except ImportError: @@ -18,26 +16,23 @@ from kombu.utils.url import _parse_url from celery.exceptions import ImproperlyConfigured -from celery.utils.timeutils import maybe_timedelta from .base import KeyValueStoreBackend -__all__ = ['CouchDBBackend'] +__all__ = ['CouchBackend'] + +ERR_LIB_MISSING = """\ +You need to install the pycouchdb library to use the CouchDB result backend\ +""" -class CouchDBBackend(KeyValueStoreBackend): +class CouchBackend(KeyValueStoreBackend): container = 'default' scheme = 'http' host = 'localhost' port = 5984 username = None password = None - quiet = False - conncache = None - unlock_gil = True - timeout = 2.5 - transcoder = None - # supports_autoexpire = False def __init__(self, url=None, *args, **kwargs): """Initialize CouchDB backend instance. @@ -46,63 +41,47 @@ def __init__(self, url=None, *args, **kwargs): module :mod:`pycouchdb` is not available. """ - super(CouchDBBackend, self).__init__(*args, **kwargs) - - self.expires = kwargs.get('expires') or maybe_timedelta( - self.app.conf.CELERY_TASK_RESULT_EXPIRES) + super(CouchBackend, self).__init__(*args, **kwargs) if pycouchdb is None: - raise ImproperlyConfigured( - 'You need to install the pycouchdb library to use the ' - 'CouchDB backend.', - ) + raise ImproperlyConfigured(ERR_LIB_MISSING) uscheme = uhost = uport = uname = upass = ucontainer = None if url: - _, uhost, uport, uname, upass, ucontainer , _ = _parse_url(url) # noqa + _, uhost, uport, uname, upass, ucontainer, _ = _parse_url(url) # noqa ucontainer = ucontainer.strip('/') if ucontainer else None - config = self.app.conf.get('CELERY_COUCHDB_BACKEND_SETTINGS', None) - if config is not None: - if not isinstance(config, dict): - raise ImproperlyConfigured( - 'CouchDB backend settings should be grouped in a dict', - ) - else: - config = {} - - self.scheme = uscheme or config.get('scheme', self.scheme) - self.host = uhost or config.get('host', self.host) - self.port = int(uport or config.get('port', self.port)) - self.container = ucontainer or config.get('container', self.container) - self.username = uname or config.get('username', self.username) - self.password = upass or config.get('password', self.password) + self.scheme = uscheme or self.scheme + self.host = uhost or self.host + self.port = int(uport or self.port) + self.container = ucontainer or self.container + self.username = uname or self.username + self.password = upass or self.password self._connection = None def _get_connection(self): """Connect to the CouchDB server.""" - if self._connection is None: - if self.username and self.password: - conn_string = '%s://%s:%s@%s:%s' % ( - self.scheme, self.username, self.password, - self.host, str(self.port)) - server = pycouchdb.Server(conn_string, authmethod='basic') - else: - conn_string = '%s://%s:%s' % ( - self.scheme, self.host, str(self.port)) - server = pycouchdb.Server(conn_string) - - logging.debug('couchdb conn string: %s', conn_string) - try: - self._connection = server.database(self.container) - except pycouchdb.exceptions.NotFound: - self._connection = server.create(self.container) - return self._connection + if self.username and self.password: + conn_string = '%s://%s:%s@%s:%s' % ( + self.scheme, self.username, self.password, + self.host, str(self.port)) + server = pycouchdb.Server(conn_string, authmethod='basic') + else: + conn_string = '%s://%s:%s' % ( + self.scheme, self.host, str(self.port)) + server = pycouchdb.Server(conn_string) + + try: + return server.database(self.container) + except pycouchdb.exceptions.NotFound: + return server.create(self.container) @property def connection(self): - return self._get_connection() + if self._connection is None: + self._connection = self._get_connection() + return self._connection def get(self, key): try: diff --git a/celery/backends/database/__init__.py b/celery/backends/database/__init__.py index c52e75879..96dbb0a0d 100644 --- a/celery/backends/database/__init__.py +++ b/celery/backends/database/__init__.py @@ -79,13 +79,13 @@ class DatabaseBackend(BaseBackend): # to not bombard the database with queries. subpolling_interval = 0.5 - def __init__(self, dburi=None, expires=None, - engine_options=None, url=None, **kwargs): + def __init__(self, dburi=None, engine_options=None, url=None, **kwargs): # The `url` argument was added later and is used by # the app to set backend by url (celery.backends.get_backend_by_url) - super(DatabaseBackend, self).__init__(**kwargs) + super(DatabaseBackend, self).__init__( + expires_type=maybe_timedelta, **kwargs + ) conf = self.app.conf - self.expires = maybe_timedelta(self.prepare_expires(expires)) self.dburi = url or dburi or conf.CELERY_RESULT_DBURI self.engine_options = dict( engine_options or {}, diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 581b93dd7..0e455497f 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -31,7 +31,6 @@ from celery import states from celery.exceptions import ImproperlyConfigured from celery.five import string_t -from celery.utils.timeutils import maybe_timedelta from .base import BaseBackend @@ -60,7 +59,7 @@ class MongoBackend(BaseBackend): _connection = None - def __init__(self, *args, **kwargs): + def __init__(self, **kwargs): """Initialize MongoDB backend instance. :raises celery.exceptions.ImproperlyConfigured: if @@ -69,9 +68,7 @@ def __init__(self, *args, **kwargs): """ self.options = {} - super(MongoBackend, self).__init__(*args, **kwargs) - self.expires = kwargs.get('expires') or maybe_timedelta( - self.app.conf.CELERY_TASK_RESULT_EXPIRES) + super(MongoBackend, self).__init__(**kwargs) if not pymongo: raise ImproperlyConfigured( diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 3e76513db..2b3d51df6 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -57,9 +57,9 @@ class RedisBackend(KeyValueStoreBackend): implements_incr = True def __init__(self, host=None, port=None, db=None, password=None, - expires=None, max_connections=None, url=None, + max_connections=None, url=None, connection_pool=None, new_join=False, **kwargs): - super(RedisBackend, self).__init__(**kwargs) + super(RedisBackend, self).__init__(expires_type=int, **kwargs) conf = self.app.conf if self.redis is None: raise ImproperlyConfigured(REDIS_MISSING) @@ -90,7 +90,6 @@ def _get(key): if url: self.connparams = self._params_from_url(url, self.connparams) self.url = url - self.expires = self.prepare_expires(expires, type=int) try: new_join = strtobool(self.connparams.pop('new_join')) diff --git a/celery/backends/riak.py b/celery/backends/riak.py index f25ae4f3e..f9bc8cf3a 100644 --- a/celery/backends/riak.py +++ b/celery/backends/riak.py @@ -18,7 +18,6 @@ from kombu.utils.url import _parse_url from celery.exceptions import ImproperlyConfigured -from celery.utils.timeutils import maybe_timedelta from .base import KeyValueStoreBackend @@ -60,9 +59,6 @@ def __init__(self, host=None, port=None, bucket_name=None, protocol=None, """ super(RiakBackend, self).__init__(*args, **kwargs) - self.expires = kwargs.get('expires') or maybe_timedelta( - self.app.conf.CELERY_TASK_RESULT_EXPIRES) - if not riak: raise ImproperlyConfigured( 'You need to install the riak library to use the ' diff --git a/celery/tests/backends/test_couchdb.py b/celery/tests/backends/test_couchdb.py index 1df8eec46..2a81f54d6 100644 --- a/celery/tests/backends/test_couchdb.py +++ b/celery/tests/backends/test_couchdb.py @@ -1,7 +1,7 @@ from __future__ import absolute_import from celery.backends import couchdb as module -from celery.backends.couchdb import CouchDBBackend +from celery.backends.couchdb import CouchBackend from celery.exceptions import ImproperlyConfigured from celery import backends from celery.tests.case import ( @@ -16,33 +16,22 @@ COUCHDB_CONTAINER = 'celery_container' -class test_CouchDBBackend(AppCase): +class test_CouchBackend(AppCase): def setup(self): if pycouchdb is None: raise SkipTest('pycouchdb is not installed.') - self.backend = CouchDBBackend(app=self.app) + self.backend = CouchBackend(app=self.app) def test_init_no_pycouchdb(self): """test init no pycouchdb raises""" prev, module.pycouchdb = module.pycouchdb, None try: with self.assertRaises(ImproperlyConfigured): - CouchDBBackend(app=self.app) + CouchBackend(app=self.app) finally: module.pycouchdb = prev - def test_init_no_settings(self): - """test init no settings""" - self.app.conf.CELERY_COUCHDB_BACKEND_SETTINGS = [] - with self.assertRaises(ImproperlyConfigured): - CouchDBBackend(app=self.app) - - def test_init_settings_is_None(self): - """Test init settings is None""" - self.app.conf.CELERY_COUCHDB_BACKEND_SETTINGS = None - CouchDBBackend(app=self.app) - def test_get_container_exists(self): with patch('pycouchdb.client.Database') as mock_Connection: self.backend._connection = sentinel._connection @@ -55,13 +44,12 @@ def test_get_container_exists(self): def test_get(self): """test_get - CouchDBBackend.get should return and take two params + CouchBackend.get should return and take two params db conn to couchdb is mocked. TODO Should test on key not exists """ - self.app.conf.CELERY_COUCHDB_BACKEND_SETTINGS = {} - x = CouchDBBackend(app=self.app) + x = CouchBackend(app=self.app) x._connection = Mock() mocked_get = x._connection.get = Mock() mocked_get.return_value = sentinel.retval @@ -72,13 +60,12 @@ def test_get(self): def test_delete(self): """test_delete - CouchDBBackend.delete should return and take two params + CouchBackend.delete should return and take two params db conn to pycouchdb is mocked. TODO Should test on key not exists """ - self.app.conf.CELERY_COUCHDB_BACKEND_SETTINGS = {} - x = CouchDBBackend(app=self.app) + x = CouchBackend(app=self.app) x._connection = Mock() mocked_delete = x._connection.delete = Mock() mocked_delete.return_value = None @@ -86,29 +73,10 @@ def test_delete(self): self.assertIsNone(x.delete('1f3fab')) x._connection.delete.assert_called_once_with('1f3fab') - def test_config_params(self): - """test_config_params - - celery.conf.CELERY_COUCHDB_BACKEND_SETTINGS is properly set - """ - self.app.conf.CELERY_COUCHDB_BACKEND_SETTINGS = { - 'container': 'mycoolcontainer', - 'host': ['here.host.com', 'there.host.com'], - 'username': 'johndoe', - 'password': 'mysecret', - 'port': '1234', - } - x = CouchDBBackend(app=self.app) - self.assertEqual(x.container, 'mycoolcontainer') - self.assertEqual(x.host, ['here.host.com', 'there.host.com'],) - self.assertEqual(x.username, 'johndoe',) - self.assertEqual(x.password, 'mysecret') - self.assertEqual(x.port, 1234) - def test_backend_by_url(self, url='couchdb://myhost/mycoolcontainer'): - from celery.backends.couchdb import CouchDBBackend + from celery.backends.couchdb import CouchBackend backend, url_ = backends.get_backend_by_url(url, self.app.loader) - self.assertIs(backend, CouchDBBackend) + self.assertIs(backend, CouchBackend) self.assertEqual(url_, url) def test_backend_params_by_url(self): diff --git a/docs/configuration.rst b/docs/configuration.rst index e97f49c07..5ad2277ec 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -794,20 +794,18 @@ CouchDB backend settings $ pip install pycouchdb This backend can be configured via the :setting:`CELERY_RESULT_BACKEND` -set to a couchbase URL:: +set to a couchdb URL:: CELERY_RESULT_BACKEND = 'couchdb://username:password@host:port/container' -.. setting:: CELERY_COUCHDB_BACKEND_SETTINGS - -CELERY_COUCHDB_BACKEND_SETTINGS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The URL is formed out of the following parts: -This is a dict supporting the following keys: +* username + User name to authenticate to the CouchDB server as (optional). -* scheme - http or https. Defaults to ``http``. +* password + Password to authenticate to the CouchDB server (optional). * host Host name of the CouchDB server. Defaults to ``localhost``. @@ -819,11 +817,6 @@ This is a dict supporting the following keys: The default container the CouchDB server is writing to. Defaults to ``default``. -* username - User name to authenticate to the CouchDB server as (optional). - -* password - Password to authenticate to the CouchDB server (optional). .. _conf-messaging: From b677d73848763f3be00db13945e00e83b0248d59 Mon Sep 17 00:00:00 2001 From: Nathan Van Gheem Date: Tue, 28 Oct 2014 13:17:57 -0500 Subject: [PATCH 0375/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 3b4409231..44f8e24bd 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -171,3 +171,4 @@ John Whitlock, 2014/08/19 Konstantinos Koukopoulos, 2014/08/24 Andrea Rabbaglietti, 2014/10/02 Joe Jevnik, 2014/10/22 +Nathan Van Gheem, 2014/10/28 From 7a14d6f1a1158839e0beae75a4e634b2e84e654c Mon Sep 17 00:00:00 2001 From: Gino Ledesma Date: Thu, 28 Aug 2014 17:43:57 -0700 Subject: [PATCH 0376/1103] Fix results for cassandra backend in detailed mode When using the Cassandra backend in detailed mode, attempting to retrieve the results results in an error because of improperly decoded metadata (introduced by 6d64d7a24f15311a0bd13baf7f6a9879176d2d08). The metadata for `children` and `traceback` are encoded before being grouped into the dictionary, which itself is encoded later prior to being stored in Cassandra. During retrieval in detailed mode, these are not decoded again, causing them to be interpreted as encoded unicode literals instead of their native types. CHANGES: - the `results` metadata is encoded regardless of whether Cassandra is used in detailed or non-detailed mode. This is being consistent with the other backends where the results are always encoded. It may be unnecessary (results in double encoding), however. - The logic for encoding/decoding metadata is now consistent for both detailed and non-detailed modes. REFERENCE: - 21cdf82fc4310320c21687cf80a3f98802565bd5 - cassandra detailed mode - 6d64d7a24f15311a0bd13baf7f6a9879176d2d08 - backends support children Conflicts: celery/backends/cassandra.py --- celery/backends/cassandra.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index 8d1b80181..aa8e688cc 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -136,17 +136,16 @@ def _store_result(self, task_id, result, status, """Store return value and status of an executed task.""" def _do_store(): - detailed = self.detailed_mode cf = self._get_column_family() date_done = self.app.now() meta = {'status': status, 'date_done': date_done.strftime('%Y-%m-%dT%H:%M:%SZ'), 'traceback': self.encode(traceback), - 'result': result if detailed else self.encode(result), + 'result': self.encode(result), 'children': self.encode( self.current_task_children(request), )} - if detailed: + if self.detailed_mode: cf.insert( task_id, {date_done: self.encode(meta)}, ttl=self.expires, ) @@ -163,11 +162,10 @@ def _do_get(): try: if self.detailed_mode: row = cf.get(task_id, column_reversed=True, column_count=1) - meta = self.decode(list(row.values())[0]) - meta['task_id'] = task_id + return self.decode(list(row.values())[0]) else: obj = cf.get(task_id) - meta = self.meta_from_decoded({ + return self.meta_from_decoded({ 'task_id': task_id, 'status': obj['status'], 'result': self.decode(obj['result']), @@ -176,8 +174,7 @@ def _do_get(): 'children': self.decode(obj['children']), }) except (KeyError, pycassa.NotFoundException): - meta = {'status': states.PENDING, 'result': None} - return meta + return {'status': states.PENDING, 'result': None} return self._retry_on_error(_do_get) From 4c9e884e7fc3e90ca8098c872124efa1ed6619af Mon Sep 17 00:00:00 2001 From: Albert Wang Date: Fri, 29 Aug 2014 17:42:34 -0700 Subject: [PATCH 0377/1103] Made sure a time zone is provided for request expiration checking --- celery/worker/request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/worker/request.py b/celery/worker/request.py index 0fb396f6a..3a28def05 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -220,7 +220,7 @@ def execute(self, loglevel=None, logfile=None): def maybe_expire(self): """If expired, mark the task as revoked.""" if self.expires: - now = datetime.now(tz_or_local(self.tzlocal) if self.utc else None) + now = datetime.now(self.expires.tzinfo) if now > self.expires: revoked_tasks.add(self.id) return True From a32d0c88e733da6d98c86655d10d43ebd1dbbd9b Mon Sep 17 00:00:00 2001 From: Albert Wang Date: Tue, 28 Oct 2014 18:31:22 -0700 Subject: [PATCH 0378/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 44f8e24bd..23c102ef9 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -169,6 +169,7 @@ Akexander Koshelev, 2014/08/19 Davide Quarta, 2014/08/19 John Whitlock, 2014/08/19 Konstantinos Koukopoulos, 2014/08/24 +Albert Yee Wang, 2014/08/29 Andrea Rabbaglietti, 2014/10/02 Joe Jevnik, 2014/10/22 Nathan Van Gheem, 2014/10/28 From a46d337c2cce82c7742f13c2c1349b6a8b5321af Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 29 Oct 2014 17:28:56 +0000 Subject: [PATCH 0379/1103] Clone must keep chord_size for all signature types. Closes #2339 --- celery/backends/redis.py | 5 ++++- celery/canvas.py | 31 +++++++++++++++++++++++-------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 2b3d51df6..a207e05b7 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -180,8 +180,11 @@ def add_to_chord(self, group_id, result): self.client.incr(self.get_key_for_group(group_id, '.t'), 1) def _unpack_chord_result(self, tup, decode, + EXCEPTION_STATES=states.EXCEPTION_STATES, PROPAGATE_STATES=states.PROPAGATE_STATES): _, tid, state, retval = decode(tup) + if state in EXCEPTION_STATES: + retval = self.exception_to_python(retval) if state in PROPAGATE_STATES: raise ChordError('Dependency {0} raised {1!r}'.format(tid, retval)) return retval @@ -220,7 +223,7 @@ def _new_chord_return(self, task, state, result, propagate=None, callback = maybe_signature(request.chord, app=app) total = callback['chord_size'] + totaldiff if readycount == total: - decode, unpack = self.decode_result, self._unpack_chord_result + decode, unpack = self.decode, self._unpack_chord_result resl, _, _ = client.pipeline() \ .lrange(jkey, 0, total) \ .delete(jkey) \ diff --git a/celery/canvas.py b/celery/canvas.py index 5b58004cd..8c9ad2ad1 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -103,6 +103,12 @@ def task_name_from(task): return getattr(task, 'name', task) +def _upgrade(fields, sig): + """Used by custom signatures in .from_dict, to keep common fields.""" + sig.update(chord_size=fields.get('chord_size')) + return sig + + class Signature(dict): """Class that wraps the arguments and execution options for a single task invocation. @@ -162,7 +168,8 @@ def __init__(self, task=None, args=None, kwargs=None, options=None, kwargs=kwargs or {}, options=dict(options or {}, **ex), subtask_type=subtask_type, - immutable=immutable) + immutable=immutable, + chord_size=None) def __call__(self, *partial_args, **partial_kwargs): args, kwargs, _ = self._merge(partial_args, partial_kwargs, None) @@ -194,6 +201,7 @@ def clone(self, args=(), kwargs={}, **opts): s = Signature.from_dict({'task': self.task, 'args': tuple(args), 'kwargs': kwargs, 'options': deepcopy(opts), 'subtask_type': self.subtask_type, + 'chord_size': self.chord_size, 'immutable': self.immutable}, app=self._app) s._type = self._type return s @@ -345,6 +353,7 @@ def _apply_async(self): kwargs = _getitem_property('kwargs') options = _getitem_property('options') subtask_type = _getitem_property('subtask_type') + chord_size = _getitem_property('chord_size') immutable = _getitem_property('immutable') @@ -475,7 +484,7 @@ def from_dict(self, d, app=None): tasks = d['kwargs']['tasks'] = list(tasks) # First task must be signature object to get app tasks[0] = maybe_signature(tasks[0], app=app) - return chain(*tasks, app=app, **d['options']) + return _upgrade(d, chain(*tasks, app=app, **d['options'])) @property def app(self): @@ -511,7 +520,9 @@ def apply_async(self, args=(), kwargs={}, **opts): @classmethod def from_dict(cls, d, app=None): - return cls(*cls._unpack_args(d['kwargs']), app=app, **d['options']) + return _upgrade( + d, cls(*cls._unpack_args(d['kwargs']), app=app, **d['options']), + ) @Signature.register_type @@ -547,7 +558,10 @@ def __init__(self, task, it, n, **options): @classmethod def from_dict(self, d, app=None): - return chunks(*self._unpack_args(d['kwargs']), app=app, **d['options']) + return _upgrade( + d, chunks(*self._unpack_args( + d['kwargs']), app=app, **d['options']), + ) def apply_async(self, args=(), kwargs={}, **opts): return self.group().apply_async( @@ -594,7 +608,9 @@ def __init__(self, *tasks, **options): @classmethod def from_dict(self, d, app=None): - return group(d['kwargs']['tasks'], app=app, **d['options']) + return _upgrade( + d, group(d['kwargs']['tasks'], app=app, **d['options']), + ) def _prepared(self, tasks, partial_args, group_id, root_id, app, dict=dict, Signature=Signature, from_dict=Signature.from_dict): @@ -756,7 +772,7 @@ def freeze(self, *args, **kwargs): @classmethod def from_dict(self, d, app=None): args, d['kwargs'] = self._unpack_args(**d['kwargs']) - return self(*args, app=app, **d) + return _upgrade(d, self(*args, app=app, **d)) @staticmethod def _unpack_args(header=None, body=None, **kwargs): @@ -820,8 +836,7 @@ def run(self, header, body, partial_args, app=None, interval=None, if propagate is None else propagate) group_id = uuid() root_id = body.options.get('root_id') - if 'chord_size' not in body: - body['chord_size'] = self.__length_hint__() + body.chord_size = self.__length_hint__() options = dict(self.options, **options) if options else self.options if options: body.options.update(options) From 5c9ee7eb72f31fca789485d5bc3a8a4f3ee7b7a7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 29 Oct 2014 17:39:31 +0000 Subject: [PATCH 0380/1103] flakes --- celery/result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/result.py b/celery/result.py index ad5a8158d..3784547f0 100644 --- a/celery/result.py +++ b/celery/result.py @@ -342,7 +342,7 @@ def _get_task_meta(self): return self._cache def _set_cache(self, d): - state, children = d['status'], d.get('children') + children = d.get('children') if children: d['children'] = [ result_from_tuple(child, self.app) for child in children From 6de60a9ea48205df1160e154a930b610ce28ebb9 Mon Sep 17 00:00:00 2001 From: Gino Ledesma Date: Thu, 30 Oct 2014 14:54:58 -0700 Subject: [PATCH 0381/1103] Updated CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 23c102ef9..e240a043f 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -173,3 +173,4 @@ Albert Yee Wang, 2014/08/29 Andrea Rabbaglietti, 2014/10/02 Joe Jevnik, 2014/10/22 Nathan Van Gheem, 2014/10/28 +Gino Ledesma, 2014/10/28 From 9efe2d4b3e1ffc28ee9078a5006b236e1f454930 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 3 Nov 2014 21:17:25 +0000 Subject: [PATCH 0382/1103] Task: Multiple callbacks not applied correctly (Closes #2350) --- celery/app/trace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/trace.py b/celery/app/trace.py index e01543c65..fa75c4a6e 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -392,7 +392,7 @@ def trace_task(uuid, args, kwargs, request=None): for group_ in groups: group.apply_async((retval, )) if sigs: - group(sigs).apply_async(retval, ) + group(sigs).apply_async((retval, )) else: signature(callbacks[0], app=app).delay(retval) if publish_result: From 77d7eb06e3aea636e0f2e1388c491181b955358f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 3 Nov 2014 21:48:22 +0000 Subject: [PATCH 0383/1103] Do not rely on billiard.util being available (Issue #2345) --- celery/app/__init__.py | 7 +++++-- celery/app/base.py | 8 ++++++-- celery/apps/worker.py | 2 +- celery/backends/database/session.py | 8 ++++++-- celery/beat.py | 3 ++- celery/contrib/rdb.py | 2 +- celery/platforms.py | 9 ++++++--- celery/tests/app/test_log.py | 7 ++----- celery/tests/bin/test_worker.py | 2 +- celery/utils/log.py | 31 ++++++++++++++++++++++------- celery/worker/state.py | 2 +- 11 files changed, 55 insertions(+), 26 deletions(-) diff --git a/celery/app/__init__.py b/celery/app/__init__.py index 3053a59b9..8e8d9a79c 100644 --- a/celery/app/__init__.py +++ b/celery/app/__init__.py @@ -63,13 +63,16 @@ def _app_or_default(app=None): def _app_or_default_trace(app=None): # pragma: no cover from traceback import print_stack - from billiard import current_process + try: + from billiard.process import current_process + except ImportError: + current_process = None if app is None: if getattr(_state._tls, 'current_app', None): print('-- RETURNING TO CURRENT APP --') # noqa+ print_stack() return _state._tls.current_app - if current_process()._name == 'MainProcess': + if not current_process or current_process()._name == 'MainProcess': raise Exception('DEFAULT APP') print('-- RETURNING TO DEFAULT APP --') # noqa+ print_stack() diff --git a/celery/app/base.py b/celery/app/base.py index 8ff3ceb34..7449f591d 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -17,7 +17,10 @@ from operator import attrgetter from amqp import promise -from billiard.util import register_after_fork +try: + from billiard.util import register_after_fork +except ImportError: + register_after_fork = None from kombu.clocks import LamportClock from kombu.common import oid_from from kombu.utils import cached_property, uuid @@ -98,7 +101,8 @@ def _global_after_fork(obj): def _ensure_after_fork(): global _after_fork_registered _after_fork_registered = True - register_after_fork(_global_after_fork, _global_after_fork) + if register_after_fork is not None: + register_after_fork(_global_after_fork, _global_after_fork) class Celery(object): diff --git a/celery/apps/worker.py b/celery/apps/worker.py index d3c220853..887471a39 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -20,7 +20,7 @@ from functools import partial -from billiard import current_process +from billiard.process import current_process from kombu.utils.encoding import safe_str from kombu.utils.url import maybe_sanitize_url diff --git a/celery/backends/database/session.py b/celery/backends/database/session.py index 022233aab..036b84300 100644 --- a/celery/backends/database/session.py +++ b/celery/backends/database/session.py @@ -8,7 +8,10 @@ """ from __future__ import absolute_import -from billiard.util import register_after_fork +try: + from billiard.util import register_after_fork +except ImportError: + register_after_fork = None from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base @@ -26,7 +29,8 @@ def __init__(self): self._sessions = {} self.forked = False self.prepared = False - register_after_fork(self, self._after_fork) + if register_after_fork is not None: + register_after_fork(self, self._after_fork) def _after_fork(self,): self.forked = True diff --git a/celery/beat.py b/celery/beat.py index aaa4df74a..c88f521c3 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -19,7 +19,8 @@ from collections import namedtuple from threading import Event, Thread -from billiard import Process, ensure_multiprocessing +from billiard import ensure_multiprocessing +from billiard.process import Process from billiard.common import reset_signals from kombu.utils import cached_property, reprcall from kombu.utils.functional import maybe_evaluate diff --git a/celery/contrib/rdb.py b/celery/contrib/rdb.py index 99edb64d7..bab9c8029 100644 --- a/celery/contrib/rdb.py +++ b/celery/contrib/rdb.py @@ -43,7 +43,7 @@ def add(x, y): from pdb import Pdb -from billiard import current_process +from billiard.process import current_process from celery.five import range diff --git a/celery/platforms.py b/celery/platforms.py index b9d39177c..6b0900916 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -21,7 +21,10 @@ from collections import namedtuple -from billiard import current_process +try: + from billiard.process import current_process +except ImportError: + current_process = None # fileno used to be in this module from kombu.utils import maybe_fileno from kombu.utils.encoding import safe_str @@ -706,8 +709,8 @@ def set_mp_process_title(progname, info=None, hostname=None): # noqa """ if hostname: progname = '{0}: {1}'.format(progname, hostname) - return set_process_title( - '{0}:{1}'.format(progname, current_process().name), info=info) + name = current_process().name if current_process else 'MainProcess' + return set_process_title('{0}:{1}'.format(progname, name), info=info) def get_errno_name(n): diff --git a/celery/tests/app/test_log.py b/celery/tests/app/test_log.py index 1cc43a526..ce905686b 100644 --- a/celery/tests/app/test_log.py +++ b/celery/tests/app/test_log.py @@ -21,7 +21,7 @@ logger_isa, ) from celery.tests.case import ( - AppCase, Mock, SkipTest, + AppCase, Mock, SkipTest, mask_modules, get_handlers, override_stdouts, patch, wrap_logger, restore_logging, ) @@ -209,11 +209,8 @@ def test_setup_logging_subsystem_colorize(self): def test_setup_logging_subsystem_no_mputil(self): from celery.utils import log as logtools with restore_logging(): - mputil, logtools.mputil = logtools.mputil, None - try: + with mask_modules('billiard.util'): self.app.log.setup_logging_subsystem() - finally: - logtools.mputil = mputil def _assertLog(self, logger, logmsg, loglevel=logging.ERROR): diff --git a/celery/tests/bin/test_worker.py b/celery/tests/bin/test_worker.py index dea82e5c5..ea60da462 100644 --- a/celery/tests/bin/test_worker.py +++ b/celery/tests/bin/test_worker.py @@ -6,7 +6,7 @@ from functools import wraps -from billiard import current_process +from billiard.process import current_process from kombu import Exchange, Queue from celery import platforms diff --git a/celery/utils/log.py b/celery/utils/log.py index 66feef659..ccb715a6d 100644 --- a/celery/utils/log.py +++ b/celery/utils/log.py @@ -16,7 +16,6 @@ import traceback from contextlib import contextmanager -from billiard import current_process, util as mputil from kombu.five import values from kombu.log import get_logger as _get_logger, LOG_LEVELS from kombu.utils.encoding import safe_str @@ -253,15 +252,33 @@ def isatty(self): def get_multiprocessing_logger(): - return mputil.get_logger() if mputil else None + try: + from billiard import util + except ImportError: + pass + else: + return util.get_logger() def reset_multiprocessing_logger(): - if mputil and hasattr(mputil, '_logger'): - mputil._logger = None + try: + from billiard import util + except ImportError: + pass + else: + if hasattr(util, '_logger'): + util._logger = None + + +def current_process(): + try: + from billiard import process + except ImportError: + pass + else: + return process.current_process() def current_process_index(base=1): - if current_process: - index = getattr(current_process(), 'index', None) - return index + base if index is not None else index + index = getattr(current_process(), 'index', None) + return index + base if index is not None else index diff --git a/celery/worker/state.py b/celery/worker/state.py index 2e2773e56..9a3ff49c1 100644 --- a/celery/worker/state.py +++ b/celery/worker/state.py @@ -90,7 +90,7 @@ def task_ready(request): if C_BENCH: # pragma: no cover import atexit - from billiard import current_process + from billiard.process import current_process from celery.five import monotonic from celery.utils.debug import memdump, sample_mem From 15134f4e2462450377ac63128bcacdfce21d2636 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 5 Nov 2014 11:37:00 +0000 Subject: [PATCH 0384/1103] App: Be sure to pickle _preconf for Windows if the app is not configured when pickling. Closes #897 --- celery/app/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/base.py b/celery/app/base.py index 7449f591d..ff2ca46c9 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -605,7 +605,7 @@ def __reduce_keys__(self): when unpickling.""" return { 'main': self.main, - 'changes': self._conf.changes if self._conf else {}, + 'changes': self._conf.changes if self._conf else self._preconf, 'loader': self.loader_cls, 'backend': self.backend_cls, 'amqp': self.amqp_cls, From 00404a6e642828e76c90ce2c55c89891d3ff14a0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 5 Nov 2014 13:52:24 +0000 Subject: [PATCH 0385/1103] Clarify worker kill signal (Issue #1628) --- docs/faq.rst | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index f46477490..7bfc544f1 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -827,9 +827,23 @@ executing jobs and shut down as soon as possible. No tasks should be lost. You should never stop :mod:`~celery.bin.worker` with the :sig:`KILL` signal (:option:`-9`), unless you've tried :sig:`TERM` a few times and waited a few -minutes to let it get a chance to shut down. As if you do tasks may be -terminated mid-execution, and they will not be re-run unless you have the -`acks_late` option set (`Task.acks_late` / :setting:`CELERY_ACKS_LATE`). +minutes to let it get a chance to shut down. + +Also make sure you kill the main worker process, not its child processes. +You can direct a kill signal to a specific child process if you know the +process is currently executing a task the worker shutdown is depending on, +but this also means that a ``WorkerLostError`` state will be set for the +task so the task will not run again. + +Identifying the type of process is easier if you have installed the +``setproctitle`` module: + +.. code-block:: bash + + pip install setproctitle + +With this library installed you will be able to see the type of process in ps +listings, but the worker must be restarted for this to take effect. .. seealso:: From 898affcf96a4645aa170e46a0831ceb36a3a45df Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 5 Nov 2014 14:57:37 +0000 Subject: [PATCH 0386/1103] Canvas: group | group is now a chord --- celery/canvas.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 8c9ad2ad1..fcf20d76c 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -436,8 +436,7 @@ def prepare_steps(self, args, tasks, # splice the chain steps.extendleft(reversed(task.tasks)) continue - elif isinstance(task, group) and steps and \ - not isinstance(steps[0], group): + elif isinstance(task, group) and steps: # automatically upgrade group(...) | s to chord(group, s) try: next_step = steps.popleft() From 7024c2c23a0cbc0b1c05b0796ca8c35924667c42 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 5 Nov 2014 15:45:18 +0000 Subject: [PATCH 0387/1103] task | group now returns a GroupResult (Issue #2354) --- celery/canvas.py | 58 +++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index fcf20d76c..25158d8b0 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -388,27 +388,19 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, args = (tuple(args) + tuple(self.args) if args and not self.immutable else self.args) tasks, results = self.prepare_steps( - args, self.tasks, root_id, link_error, + args, self.tasks, root_id, link_error, app, + task_id, group_id, chord, ) - if not results: - return - result = results[-1] - last_task = tasks[-1] - if group_id: - last_task.set(group_id=group_id) - if chord: - last_task.set(chord=chord) - if task_id: - last_task.set(task_id=task_id) - result = last_task.type.AsyncResult(task_id) - # make sure we can do a link() and link_error() on a chain object. - if link: - tasks[-1].set(link=link) - tasks[0].apply_async(**options) - return result + if results: + # make sure we can do a link() and link_error() on a chain object. + if link: + tasks[-1].set(link=link) + tasks[0].apply_async(**options) + return results[-1] def prepare_steps(self, args, tasks, root_id=None, link_error=None, app=None, + last_task_id=None, group_id=None, chord_body=None, from_dict=Signature.from_dict): app = app or self.app steps = deque(tasks) @@ -417,21 +409,15 @@ def prepare_steps(self, args, tasks, i = 0 while steps: task = steps.popleft() + if not isinstance(task, Signature): task = from_dict(task, app=app) - if not i: # first task - # first task gets partial args from chain - task = task.clone(args) - res = task.freeze(root_id=root_id) - root_id = res.id if root_id is None else root_id - else: - task = task.clone() - res = task.freeze(root_id=root_id) - i += 1 - if isinstance(task, group): task = maybe_unroll_group(task) + # first task gets partial args from chain + task = task.clone(args) if not i else task.clone() + if isinstance(task, chain): # splice the chain steps.extendleft(reversed(task.tasks)) @@ -442,7 +428,7 @@ def prepare_steps(self, args, tasks, next_step = steps.popleft() # for chords we freeze by pretending it's a normal # signature instead of a group. - res = Signature.freeze(next_step) + res = Signature.freeze(next_step, root_id=root_id) task = chord( task, body=next_step, task_id=res.task_id, root_id=root_id, @@ -450,6 +436,22 @@ def prepare_steps(self, args, tasks, except IndexError: pass # no callback, so keep as group. + if steps: + res = task.freeze(root_id=root_id) + else: + # chain(task_id=id) means task id is set for the last task + # in the chain. If the chord is part of a chord/group + # then that chord/group must synchronize based on the + # last task in the chain, so we only set the group_id and + # chord callback for the last task. + res = task.freeze( + last_task_id, + root_id=root_id, group_id=group_id, chord=chord_body, + ) + root_id = res.id if root_id is None else root_id + i += 1 + + if prev_task: # link previous task to this task. prev_task.link(task) From d0b18cdfb303835de7f662014198d691b645fadc Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 6 Nov 2014 18:49:07 +0000 Subject: [PATCH 0388/1103] Task: Document that retry(max_retries=None) means "use the default". Closes #1875 --- celery/app/task.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/celery/app/task.py b/celery/app/task.py index 8499f5fab..031d52d4c 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -506,6 +506,9 @@ def retry(self, args=None, kwargs=None, exc=None, throw=True, :keyword eta: Explicit time and date to run the retry at (must be a :class:`~datetime.datetime` instance). :keyword max_retries: If set, overrides the default retry limit. + A value of :const:`None`, means "use the default", so if you want infinite + retries you would have to set the :attr:`max_retries` attribute of the + task to :const:`None` first. :keyword time_limit: If set, overrides the default time limit. :keyword soft_time_limit: If set, overrides the default soft time limit. From 638148cdf7d6c94311e53e2a7ae87c019cebff2d Mon Sep 17 00:00:00 2001 From: Michael Permana Date: Thu, 6 Nov 2014 17:30:39 -0800 Subject: [PATCH 0389/1103] https://github.com/celery/celery/issues/2357 --- celery/worker/strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index baf3070b5..49012476a 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -61,7 +61,6 @@ def default(task, app, consumer, call_at = consumer.timer.call_at apply_eta_task = consumer.apply_eta_task rate_limits_enabled = not consumer.disable_rate_limits - bucket = consumer.task_buckets[task.name] handle = consumer.on_task_request limit_task = consumer._limit_task body_can_be_buffer = consumer.pool.body_can_be_buffer @@ -116,6 +115,7 @@ def task_message_handler(message, body, ack, reject, callbacks, call_at(eta, apply_eta_task, (req, ), priority=6) else: if rate_limits_enabled: + bucket = consumer.task_buckets[task.name] if bucket: return limit_task(req, bucket, 1) task_reserved(req) From e3d90ce84741472f3f80d577207e5e231ae7bd7b Mon Sep 17 00:00:00 2001 From: Thomas French Date: Fri, 7 Nov 2014 16:44:16 +0000 Subject: [PATCH 0390/1103] redis backed: missing self reference --- celery/backends/redis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index a207e05b7..236ac3871 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -85,7 +85,7 @@ def _get(key): 'port': _get('PORT') or 6379, 'db': _get('DB') or 0, 'password': _get('PASSWORD'), - 'max_connections': max_connections, + 'max_connections': self.max_connections, } if url: self.connparams = self._params_from_url(url, self.connparams) From 885f67e7bf15c026eb57a43127296b2a3cb0d780 Mon Sep 17 00:00:00 2001 From: Thomas French Date: Fri, 7 Nov 2014 16:44:16 +0000 Subject: [PATCH 0391/1103] redis backend: missing self reference --- celery/backends/redis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index a207e05b7..236ac3871 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -85,7 +85,7 @@ def _get(key): 'port': _get('PORT') or 6379, 'db': _get('DB') or 0, 'password': _get('PASSWORD'), - 'max_connections': max_connections, + 'max_connections': self.max_connections, } if url: self.connparams = self._params_from_url(url, self.connparams) From 6f6269331b941c61516f4b71c98941c7ac8f1b94 Mon Sep 17 00:00:00 2001 From: Thomas French Date: Mon, 10 Nov 2014 16:13:39 +0000 Subject: [PATCH 0392/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index e240a043f..53c62af09 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -174,3 +174,4 @@ Andrea Rabbaglietti, 2014/10/02 Joe Jevnik, 2014/10/22 Nathan Van Gheem, 2014/10/28 Gino Ledesma, 2014/10/28 +Thomas French, 2014/11/10 From 3f076aa6ef851be00973f9f337e18e6def2e6f02 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 10 Nov 2014 20:40:41 +0000 Subject: [PATCH 0393/1103] update_state examples check direct calls. Closes #1977 --- docs/userguide/tasks.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 6342790bb..be36a43ac 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -877,8 +877,9 @@ Use :meth:`~@Task.update_state` to update a task's state:: @app.task(bind=True) def upload_files(self, filenames): for i, file in enumerate(filenames): - self.update_state(state='PROGRESS', - meta={'current': i, 'total': len(filenames)}) + if not self.request.called_directly: + self.update_state(state='PROGRESS', + meta={'current': i, 'total': len(filenames)}) Here I created the state `"PROGRESS"`, which tells any application @@ -986,7 +987,8 @@ Example that stores results manually: @app.task(bind=True) def get_tweets(self, user): timeline = twitter.get_timeline(user) - self.update_state(state=states.SUCCESS, meta=timeline) + if not self.request.called_directly: + self.update_state(state=states.SUCCESS, meta=timeline) raise Ignore() .. _task-semipred-reject: From 3e91891c7b3abc8f68a65167c73e4bd895d2c497 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 11 Nov 2014 02:20:34 +0000 Subject: [PATCH 0394/1103] Now checks task arguments when calling a task (.delay, .apply_async) --- celery/app/base.py | 3 ++- celery/app/task.py | 7 ++++++ celery/utils/functional.py | 49 +++++++++++++++++++++++++++++++++++--- 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index ff2ca46c9..de58622c4 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -38,7 +38,7 @@ from celery.local import PromiseProxy, maybe_evaluate from celery.utils import gen_task_name from celery.utils.dispatch import Signal -from celery.utils.functional import first, maybe_list +from celery.utils.functional import first, maybe_list, head_from_fun from celery.utils.imports import instantiate, symbol_by_name from celery.utils.objects import FallbackContext, mro_lookup @@ -286,6 +286,7 @@ def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): '_decorated': True, '__doc__': fun.__doc__, '__module__': fun.__module__, + '__header__': staticmethod(head_from_fun(fun)), '__wrapped__': fun}, **options))() self._tasks[task.name] = task task.bind(self) # connects task to this app diff --git a/celery/app/task.py b/celery/app/task.py index 031d52d4c..91ad2ba7a 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -448,6 +448,13 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, be replaced by a local :func:`apply` call instead. """ + try: + check_arguments = self.__header__ + except AttributeError: + pass + else: + check_arguments(*args or (), **kwargs or {}) + app = self._get_app() if app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, task_id=task_id or uuid(), diff --git a/celery/utils/functional.py b/celery/utils/functional.py index c696a17b1..e78d28089 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -6,13 +6,14 @@ Utilities for functions. """ -from __future__ import absolute_import +from __future__ import absolute_import, print_function import sys import threading from collections import OrderedDict -from functools import wraps +from functools import partial, wraps +from inspect import getargspec, isfunction, ismethod from itertools import islice from kombu.utils import cached_property @@ -22,10 +23,15 @@ __all__ = ['LRUCache', 'is_list', 'maybe_list', 'memoize', 'mlazy', 'noop', 'first', 'firstmethod', 'chunks', 'padlist', 'mattrgetter', 'uniq', - 'regen', 'dictfilter', 'lazy', 'maybe_evaluate'] + 'regen', 'dictfilter', 'lazy', 'maybe_evaluate', 'head_from_fun'] KEYWORD_MARK = object() +FUNHEAD_TEMPLATE = """ +def {fun_name}({fun_args}): + return {fun_value} +""" + class LRUCache(UserDict): """LRU Cache implementation using a doubly linked list to track access. @@ -302,3 +308,40 @@ def dictfilter(d=None, **kw): """Remove all keys from dict ``d`` whose value is :const:`None`""" d = kw if d is None else (dict(d, **kw) if kw else d) return {k: v for k, v in items(d) if v is not None} + + +def _argsfromspec(spec, replace_defaults=True): + if spec.defaults: + split = len(spec.defaults) + defaults = (list(range(len(spec.defaults))) if replace_defaults + else spec.defaults) + positional = spec.args[:-split] + optional = list(zip(spec.args[-split:], defaults)) + else: + positional, optional = spec.args, [] + return ', '.join(filter(None, [ + ', '.join(positional), + ', '.join('{0}={1}'.format(k, v) for k, v in optional), + '*{0}'.format(spec.varargs) if spec.varargs else None, + '**{0}'.format(spec.keywords) if spec.keywords else None, + ])) + + +def head_from_fun(fun, debug=True): + if not isfunction(fun) and hasattr(fun, '__call__'): + name, fun = fun.__class__.__name__, fun.__call__ + else: + name = fun.__name__ + spec = getargspec(fun) + definition = FUNHEAD_TEMPLATE.format( + fun_name=name, + fun_args=_argsfromspec(getargspec(fun)), + fun_value=1, + ) + if debug: + print(definition, file=sys.stderr) + namespace = {'__name__': 'headof_{0}'.format(name)} + exec(definition, namespace) + result = namespace[name] + result._source = definition + return result From faf8795a27634f5998d5f91b5bb7c13cf4aaa0c6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 11 Nov 2014 16:33:23 +0000 Subject: [PATCH 0395/1103] Removes out of date documentation for CELERYD_FORCE_EXECV (Issue #2020) --- docs/configuration.rst | 18 ------------------ extra/release/verify_config_reference.py | 1 + 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 5ad2277ec..ce8a629c4 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1361,24 +1361,6 @@ to have different import categories. The modules in this setting are imported after the modules in :setting:`CELERY_IMPORTS`. -.. setting:: CELERYD_FORCE_EXECV - -CELERYD_FORCE_EXECV -~~~~~~~~~~~~~~~~~~~ - -On Unix the prefork pool will fork, so that child processes -start with the same memory as the parent process. - -This can cause problems as there is a known deadlock condition -with pthread locking primitives when `fork()` is combined with threads. - -You should enable this setting if you are experiencing hangs (deadlocks), -especially in combination with time limits or having a max tasks per child limit. - -This option will be enabled by default in a later version. - -This is not a problem on Windows, as it does not have `fork()`. - .. setting:: CELERYD_WORKER_LOST_WAIT CELERYD_WORKER_LOST_WAIT diff --git a/extra/release/verify_config_reference.py b/extra/release/verify_config_reference.py index 7392a1c60..b4d37c893 100644 --- a/extra/release/verify_config_reference.py +++ b/extra/release/verify_config_reference.py @@ -18,6 +18,7 @@ 'CELERY_REDIS_PORT', 'CELERY_REDIS_DB', 'CELERY_REDIS_PASSWORD', + 'CELERYD_FORCE_EXECV', } From 9bea083ce76f93e6b0351755bc62bdc6a6b3ee76 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 11 Nov 2014 16:37:17 +0000 Subject: [PATCH 0396/1103] make configcheck passes --- celery/app/defaults.py | 1 + docs/configuration.rst | 2 ++ 2 files changed, 3 insertions(+) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 1fa5cb337..bdbc52c56 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -141,6 +141,7 @@ def __repr__(self): 'RESULT_EXCHANGE_TYPE': Option('direct'), 'RESULT_SERIALIZER': Option('json'), 'RESULT_PERSISTENT': Option(None, type='bool'), + 'RIAK_BACKEND_SETTINGS': Option(type='dict'), 'ROUTES': Option(type='any'), 'SEND_EVENTS': Option(False, type='bool'), 'SEND_TASK_ERROR_EMAILS': Option(False, type='bool'), diff --git a/docs/configuration.rst b/docs/configuration.rst index ce8a629c4..aab186901 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1256,6 +1256,8 @@ This is the total number of results to cache before older results are evicted. The default is 5000. 0 or None means no limit, and a value of :const:`-1` will disable the cache. +.. setting:: CELERY_TRACK_STARTED + CELERY_TRACK_STARTED ~~~~~~~~~~~~~~~~~~~~ From 7245458cac9b13249cfafc2bcdf85ab3bc742f5d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 13 Nov 2014 11:42:02 +0000 Subject: [PATCH 0397/1103] Prefork: Use poll() to avoid limitations of select() (Issue #2373) --- celery/concurrency/asynpool.py | 41 +++++++-- celery/tests/concurrency/test_prefork.py | 109 +++++++++++++---------- 2 files changed, 95 insertions(+), 55 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 650c812c4..2a68d6dbc 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -28,6 +28,7 @@ from collections import deque, namedtuple from io import BytesIO +from numbers import Integral from pickle import HIGHEST_PROTOCOL from time import sleep from weakref import WeakValueDictionary, ref @@ -109,8 +110,11 @@ def _get_job_writer(job): return writer() # is a weakref -def _select(readers=None, writers=None, err=None, timeout=0): - """Simple wrapper to :class:`~select.select`. +def _select(readers=None, writers=None, err=None, timeout=0, + poll=select.poll, POLLIN=select.POLLIN, + POLLOUT=select.POLLOUT, POLLERR=select.POLLERR): + """Simple wrapper to :class:`~select.select`, using :`~select.poll` + as the implementation. :param readers: Set of reader fds to test if readable. :param writers: Set of writer fds to test if writable. @@ -131,25 +135,44 @@ def _select(readers=None, writers=None, err=None, timeout=0): readers = set() if readers is None else readers writers = set() if writers is None else writers err = set() if err is None else err + poller = poll() + register = poller.register + + if readers: + [register(fd, POLLIN) for fd in readers] + if writers: + [register(fd, POLLOUT) for fd in writers] + if err: + [register(fd, POLLERR) for fd in err] + + R, W = set(), set() + timeout = 0 if timeout and timeout < 0 else round(timeout * 1e3) try: - r, w, e = select.select(readers, writers, err, timeout) - if e: - r = list(set(r) | set(e)) - return r, w, 0 + events = poller.poll(timeout) + for fd, event in events: + if not isinstance(fd, Integral): + fd = fd.fileno() + if event & POLLIN: + R.add(fd) + if event & POLLOUT: + W.add(fd) + if event & POLLERR: + R.add(fd) + return R, W, 0 except (select.error, socket.error) as exc: if exc.errno == errno.EINTR: - return [], [], 1 + return set(), set(), 1 elif exc.errno in SELECT_BAD_FD: for fd in readers | writers | err: try: select.select([fd], [], [], 0) except (select.error, socket.error) as exc: - if exc.errno not in SELECT_BAD_FD: + if getattr(exc, 'errno', None) not in SELECT_BAD_FD: raise readers.discard(fd) writers.discard(fd) err.discard(fd) - return [], [], 1 + return set(), set(), 1 else: raise diff --git a/celery/tests/concurrency/test_prefork.py b/celery/tests/concurrency/test_prefork.py index 8216531ba..00fec85d9 100644 --- a/celery/tests/concurrency/test_prefork.py +++ b/celery/tests/concurrency/test_prefork.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import errno +import select import socket import time @@ -8,7 +9,7 @@ from celery.five import items, range from celery.utils.functional import noop -from celery.tests.case import AppCase, Mock, SkipTest, call, patch +from celery.tests.case import AppCase, Mock, SkipTest, patch try: from celery.concurrency import prefork as mp from celery.concurrency import asynpool @@ -147,67 +148,83 @@ def gen(): list(g) self.assertFalse(asynpool.gen_not_started(g)) - def test_select(self): + @patch('select.select', create=True) + def test_select(self, __select): ebadf = socket.error() ebadf.errno = errno.EBADF - with patch('select.select') as select: - select.return_value = ([3], [], []) + with patch('select.poll', create=True) as poller: + poll = poller.return_value = Mock(name='poll.poll') + poll.poll.return_value = [(3, select.POLLIN)] self.assertEqual( - asynpool._select({3}), - ([3], [], 0), + asynpool._select({3}, poll=poller), + ({3}, set(), 0), ) - select.return_value = ([], [], [3]) + poll.poll.return_value = [(3, select.POLLERR)] self.assertEqual( - asynpool._select({3}, None, {3}), - ([3], [], 0), + asynpool._select({3}, None, {3}, poll=poller), + ({3}, set(), 0), ) eintr = socket.error() eintr.errno = errno.EINTR - select.side_effect = eintr + poll.poll.side_effect = eintr readers = {3} - self.assertEqual(asynpool._select(readers), ([], [], 1)) + self.assertEqual( + asynpool._select(readers, poll=poller), + (set(), set(), 1), + ) self.assertIn(3, readers) - with patch('select.select') as select: - select.side_effect = ebadf - readers = {3} - self.assertEqual(asynpool._select(readers), ([], [], 1)) - select.assert_has_calls([call([3], [], [], 0)]) - self.assertNotIn(3, readers) - - with patch('select.select') as select: - select.side_effect = MemoryError() - with self.assertRaises(MemoryError): - asynpool._select({1}) - - with patch('select.select') as select: - - def se(*args): - select.side_effect = MemoryError() - raise ebadf - select.side_effect = se + with patch('select.poll') as poller: + poll = poller.return_value = Mock(name='poll.poll') + poll.poll.side_effect = ebadf + with patch('select.select') as selcheck: + selcheck.side_effect = ebadf + readers = {3} + self.assertEqual( + asynpool._select(readers, poll=poller), + (set(), set(), 1), + ) + self.assertNotIn(3, readers) + + with patch('select.poll') as poller: + poll = poller.return_value = Mock(name='poll.poll') + poll.poll.side_effect = MemoryError() with self.assertRaises(MemoryError): - asynpool._select({3}) - - with patch('select.select') as select: - - def se2(*args): - select.side_effect = socket.error() - select.side_effect.errno = 1321 - raise ebadf - select.side_effect = se2 - with self.assertRaises(socket.error): - asynpool._select({3}) - - with patch('select.select') as select: - - select.side_effect = socket.error() - select.side_effect.errno = 34134 + asynpool._select({1}, poll=poller) + + with patch('select.poll') as poller: + poll = poller.return_value = Mock(name='poll.poll') + with patch('select.select') as selcheck: + + def se(*args): + selcheck.side_effect = MemoryError() + raise ebadf + poll.poll.side_effect = se + with self.assertRaises(MemoryError): + asynpool._select({3}, poll=poller) + + with patch('select.poll') as poller: + poll = poller.return_value = Mock(name='poll.poll') + with patch('select.select') as selcheck: + + def se2(*args): + selcheck.side_effect = socket.error() + selcheck.side_effect.errno = 1321 + raise ebadf + poll.poll.side_effect = se2 + with self.assertRaises(socket.error): + asynpool._select({3}, poll=poller) + + with patch('select.poll') as poller: + poll = poller.return_value = Mock(name='poll.poll') + + poll.poll.side_effect = socket.error() + poll.poll.side_effect.errno = 34134 with self.assertRaises(socket.error): - asynpool._select({3}) + asynpool._select({3}, poll=poller) def test_promise(self): fun = Mock() From b0cfa0d818743262a032c541cce2fa8c43fabad4 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 13 Nov 2014 11:48:35 +0000 Subject: [PATCH 0398/1103] Tests passing --- celery/app/amqp.py | 4 ++-- celery/app/base.py | 2 +- celery/app/task.py | 6 +++--- celery/canvas.py | 1 - celery/tests/app/test_app.py | 11 ++++++++--- celery/tests/app/test_builtins.py | 2 +- celery/tests/app/test_log.py | 1 - celery/tests/tasks/test_tasks.py | 4 ++-- celery/utils/functional.py | 7 ++++--- 9 files changed, 21 insertions(+), 17 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 108e707ac..85d3f5bea 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -275,9 +275,9 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, kwargs = kwargs or {} utc = self.utc if not isinstance(args, (list, tuple)): - raise ValueError('task args must be a list or tuple') + raise TypeError('task args must be a list or tuple') if not isinstance(kwargs, Mapping): - raise ValueError('task keyword arguments must be a mapping') + raise TypeError('task keyword arguments must be a mapping') if countdown: # convert countdown to ETA now = now or self.app.now() timezone = timezone or self.app.timezone diff --git a/celery/app/base.py b/celery/app/base.py index de58622c4..d4b305036 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -286,7 +286,7 @@ def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): '_decorated': True, '__doc__': fun.__doc__, '__module__': fun.__module__, - '__header__': staticmethod(head_from_fun(fun)), + '__header__': staticmethod(head_from_fun(fun, bound=bind)), '__wrapped__': fun}, **options))() self._tasks[task.name] = task task.bind(self) # connects task to this app diff --git a/celery/app/task.py b/celery/app/task.py index 91ad2ba7a..8e1d791de 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -513,9 +513,9 @@ def retry(self, args=None, kwargs=None, exc=None, throw=True, :keyword eta: Explicit time and date to run the retry at (must be a :class:`~datetime.datetime` instance). :keyword max_retries: If set, overrides the default retry limit. - A value of :const:`None`, means "use the default", so if you want infinite - retries you would have to set the :attr:`max_retries` attribute of the - task to :const:`None` first. + A value of :const:`None`, means "use the default", so if you want + infinite retries you would have to set the :attr:`max_retries` + attribute of the task to :const:`None` first. :keyword time_limit: If set, overrides the default time limit. :keyword soft_time_limit: If set, overrides the default soft time limit. diff --git a/celery/canvas.py b/celery/canvas.py index 25158d8b0..36e985c08 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -451,7 +451,6 @@ def prepare_steps(self, args, tasks, root_id = res.id if root_id is None else root_id i += 1 - if prev_task: # link previous task to this task. prev_task.link(task) diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index a58c27fb7..413d71857 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -332,9 +332,14 @@ def _inner(*args, **kwargs): def test_apply_async_has__self__(self): @self.app.task(__self__='hello', shared=False) - def aawsX(): + def aawsX(x, y): pass + with self.assertRaises(TypeError): + aawsX.apply_async(()) + with self.assertRaises(TypeError): + aawsX.apply_async((2, )) + with patch('celery.app.amqp.AMQP.create_task_message') as create: with patch('celery.app.amqp.AMQP.send_task_message') as send: create.return_value = Mock(), Mock(), Mock(), Mock() @@ -346,11 +351,11 @@ def aawsX(): def test_apply_async_adds_children(self): from celery._state import _task_stack - @self.app.task(shared=False) + @self.app.task(bind=True, shared=False) def a3cX1(self): pass - @self.app.task(shared=False) + @self.app.task(bind=True, shared=False) def a3cX2(self): pass diff --git a/celery/tests/app/test_builtins.py b/celery/tests/app/test_builtins.py index 5642cbcd0..bb70a8e1f 100644 --- a/celery/tests/app/test_builtins.py +++ b/celery/tests/app/test_builtins.py @@ -84,7 +84,7 @@ def setup(self): def test_apply_async_eager(self): self.task.apply = Mock() self.app.conf.CELERY_ALWAYS_EAGER = True - self.task.apply_async() + self.task.apply_async((1, 2, 3, 4, 5)) self.assertTrue(self.task.apply.called) def test_apply(self): diff --git a/celery/tests/app/test_log.py b/celery/tests/app/test_log.py index ce905686b..fffffa7b2 100644 --- a/celery/tests/app/test_log.py +++ b/celery/tests/app/test_log.py @@ -207,7 +207,6 @@ def test_setup_logging_subsystem_colorize(self): self.app.log.setup_logging_subsystem(colorize=True) def test_setup_logging_subsystem_no_mputil(self): - from celery.utils import log as logtools with restore_logging(): with mask_modules('billiard.util'): self.app.log.setup_logging_subsystem() diff --git a/celery/tests/tasks/test_tasks.py b/celery/tests/tasks/test_tasks.py index 8d9da1f46..dca6d2cf1 100644 --- a/celery/tests/tasks/test_tasks.py +++ b/celery/tests/tasks/test_tasks.py @@ -261,12 +261,12 @@ class IncompleteTask(Task): IncompleteTask().run() def test_task_kwargs_must_be_dictionary(self): - with self.assertRaises(ValueError): + with self.assertRaises(TypeError): self.increment_counter.apply_async([], 'str') def test_task_args_must_be_list(self): with self.assertRaises(ValueError): - self.increment_counter.apply_async('str', {}) + self.increment_counter.apply_async('s', {}) def test_regular_task(self): self.assertIsInstance(self.mytask, Task) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index e78d28089..83b5ba29c 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -13,7 +13,7 @@ from collections import OrderedDict from functools import partial, wraps -from inspect import getargspec, isfunction, ismethod +from inspect import getargspec, isfunction from itertools import islice from kombu.utils import cached_property @@ -327,12 +327,11 @@ def _argsfromspec(spec, replace_defaults=True): ])) -def head_from_fun(fun, debug=True): +def head_from_fun(fun, bound=False, debug=False): if not isfunction(fun) and hasattr(fun, '__call__'): name, fun = fun.__class__.__name__, fun.__call__ else: name = fun.__name__ - spec = getargspec(fun) definition = FUNHEAD_TEMPLATE.format( fun_name=name, fun_args=_argsfromspec(getargspec(fun)), @@ -344,4 +343,6 @@ def head_from_fun(fun, debug=True): exec(definition, namespace) result = namespace[name] result._source = definition + if bound: + return partial(result, object()) return result From 0eb5f70a0e4b43b84dda4ad760593094415560be Mon Sep 17 00:00:00 2001 From: Marin Atanasov Nikolov Date: Thu, 13 Nov 2014 17:00:37 +0200 Subject: [PATCH 0399/1103] Typo fix --- celery/app/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/base.py b/celery/app/base.py index d4b305036..bc1eda601 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -168,7 +168,7 @@ def __init__(self, main=None, loader=None, backend=None, if not isinstance(self._tasks, TaskRegistry): self._tasks = TaskRegistry(self._tasks or {}) - # If the class defins a custom __reduce_args__ we need to use + # If the class defines a custom __reduce_args__ we need to use # the old way of pickling apps, which is pickling a list of # args instead of the new way that pickles a dict of keywords. self._using_v1_reduce = app_has_custom(self, '__reduce_args__') From b0c5cf055abe3139294b3fa0c28c580934f8f9de Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 18 Nov 2014 14:25:40 +0000 Subject: [PATCH 0400/1103] Rollover changelog --- Changelog | 1086 +-------------------------- docs/history/changelog-3.1.rst | 1260 ++++++++++++++++++++++++++++++++ docs/history/index.rst | 1 + 3 files changed, 1269 insertions(+), 1078 deletions(-) create mode 100644 docs/history/changelog-3.1.rst diff --git a/Changelog b/Changelog index 4fd02f286..11eb699e6 100644 --- a/Changelog +++ b/Changelog @@ -4,1085 +4,15 @@ Change history ================ -This document contains change notes for bugfix releases in the 3.1.x series -(Cipater), please see :ref:`whatsnew-3.1` for an overview of what's -new in Celery 3.1. +This document contains change notes for bugfix releases in the 3.2.x series +(Cipater), please see :ref:`whatsnew-3.2` for an overview of what's +new in Celery 3.2. -.. _version-3.1.13: +.. _version-3.2.0: -3.1.13 -====== - -Security Fixes --------------- - -* [Security: `CELERYSA-0002`_] Insecure default umask. - - The built-in utility used to daemonize the Celery worker service sets - an insecure umask by default (umask 0). - - This means that any files or directories created by the worker will - end up having world-writable permissions. - - Special thanks to Red Hat for originally discovering and reporting the - issue! - - This version will no longer set a default umask by default, so if unset - the umask of the parent process will be used. - -.. _`CELERYSA-0002`: - http://github.com/celery/celery/tree/master/docs/sec/CELERYSA-0002.txt - -News ----- - -- **Requirements** - - - Now depends on :ref:`Kombu 3.0.21 `. - - - Now depends on :mod:`billiard` 3.3.0.18. - - -- **App**: ``backend`` argument now also sets the :setting:`CELERY_RESULT_BACKEND` - setting. - -- **Task**: ``signature_from_request`` now propagates ``reply_to`` so that - the RPC backend works with retried tasks (Issue #2113). - -- **Task**: ``retry`` will no longer attempt to requeue the task if sending - the retry message fails. - - Unrelated exceptions being raised could cause a message loop, so it was - better to remove this behavior. - -- **Beat**: Accounts for standard 1ms drift by always waking up 0.010s - earlier. - - This will adjust the latency so that the periodic tasks will not move - 1ms after every invocation. - -- Documentation fixes - - Contributed by Yuval Greenfield, Lucas Wiman, nicholsonjf - -- **Worker**: Removed an outdated assert statement that could lead to errors - being masked (Issue #2086). - - - -.. _version-3.1.12: - -3.1.12 -====== -:release-date: 2014-06-09 10:12 P.M UTC -:release-by: Ask Solem - -- **Requirements** - - Now depends on :ref:`Kombu 3.0.19 `. - -- **App**: Connections were not being closed after fork due to an error in the - after fork handler (Issue #2055). - - This could manifest itself by causing framing errors when using RabbitMQ. - (``Unexpected frame``). - -- **Django**: ``django.setup()`` was being called too late when - using Django 1.7 (Issue #1802). - -- **Django**: Fixed problems with event timezones when using Django - (``Substantial drift``). - - Celery did not take into account that Django modifies the - ``time.timeone`` attributes and friends. - -- **Canvas**: ``Signature.link`` now works when the link option is a scalar - value (Issue #2019). - -- **Prefork pool**: Fixed race conditions for when file descriptors are - removed from the event loop. - - Fix contributed by Roger Hu. - -- **Prefork pool**: Improved solution for dividing tasks between child - processes. - - This change should improve performance when there are many child - processes, and also decrease the chance that two subsequent tasks are - written to the same child process. - -- **Worker**: Now ignores unknown event types, instead of crashing. - - Fix contributed by Illes Solt. - -- **Programs**: :program:`celery worker --detach` no longer closes open file - descriptors when :envvar:`C_FAKEFORK` is used so that the workers output - can be seen. - -- **Programs**: The default working directory for :program:`celery worker - --detach` is now the current working directory, not ``/``. - -- **Canvas**: ``signature(s, app=app)`` did not upgrade serialized signatures - to their original class (``subtask_type``) when the ``app`` keyword argument - was used. - -- **Control**: The ``duplicate nodename`` warning emitted by control commands - now shows the duplicate node name. - -- **Tasks**: Can now call ``ResultSet.get()`` on a result set without members. - - Fix contributed by Alexey Kotlyarov. - -- **App**: Fixed strange traceback mangling issue for - ``app.connection_or_acquire``. - -- **Programs**: The :program:`celery multi stopwait` command is now documented - in usage. - -- **Other**: Fixed cleanup problem with ``PromiseProxy`` when an error is - raised while trying to evaluate the promise. - -- **Other**: The utility used to censor configuration values now handles - non-string keys. - - Fix contributed by Luke Pomfrey. - -- **Other**: The ``inspect conf`` command did not handle non-string keys well. - - Fix contributed by Jay Farrimond. - -- **Programs**: Fixed argument handling problem in - :program:`celery worker --detach`. - - Fix contributed by Dmitry Malinovsky. - -- **Programs**: :program:`celery worker --detach` did not forward working - directory option (Issue #2003). - -- **Programs**: :program:`celery inspect registered` no longer includes - the list of built-in tasks. - -- **Worker**: The ``requires`` attribute for boot steps were not being handled - correctly (Issue #2002). - -- **Eventlet**: The eventlet pool now supports the ``pool_grow`` and - ``pool_shrink`` remote control commands. - - Contributed by Mher Movsisyan. - -- **Eventlet**: The eventlet pool now implements statistics for - :program:``celery inspect stats``. - - Contributed by Mher Movsisyan. - -- **Documentation**: Clarified ``Task.rate_limit`` behavior. - - Contributed by Jonas Haag. - -- **Documentation**: ``AbortableTask`` examples now updated to use the new - API (Issue #1993). - -- **Documentation**: The security documentation examples used an out of date - import. - - Fix contributed by Ian Dees. - -- **Init scripts**: The CentOS init scripts did not quote - :envvar:`CELERY_CHDIR`. - - Fix contributed by ffeast. - -.. _version-3.1.11: - -3.1.11 -====== -:release-date: 2014-04-16 11:00 P.M UTC -:release-by: Ask Solem - -- **Now compatible with RabbitMQ 3.3.0** - - You need to run Celery 3.1.11 or later when using RabbitMQ 3.3, - and if you use the ``librabbitmq`` module you also have to upgrade - to librabbitmq 1.5.0: - - .. code-block:: bash - - $ pip install -U librabbitmq - -- **Requirements**: - - - Now depends on :ref:`Kombu 3.0.15 `. - - - Now depends on `billiard 3.3.0.17`_. - - - Bundle ``celery[librabbitmq]`` now depends on :mod:`librabbitmq` 1.5.0. - -.. _`billiard 3.3.0.17`: - https://github.com/celery/billiard/blob/master/CHANGES.txt - -- **Tasks**: The :setting:`CELERY_DEFAULT_DELIVERY_MODE` setting was being - ignored (Issue #1953). - -- **Worker**: New :option:`--heartbeat-interval` can be used to change the - time (in seconds) between sending event heartbeats. - - Contributed by Matthew Duggan and Craig Northway. - -- **App**: Fixed memory leaks occurring when creating lots of temporary - app instances (Issue #1949). - -- **MongoDB**: SSL configuration with non-MongoDB transport breaks MongoDB - results backend (Issue #1973). - - Fix contributed by Brian Bouterse. - -- **Logging**: The color formatter accidentally modified ``record.msg`` - (Issue #1939). - -- **Results**: Fixed problem with task trails being stored multiple times, - causing ``result.collect()`` to hang (Issue #1936, Issue #1943). - -- **Results**: ``ResultSet`` now implements a ``.backend`` attribute for - compatibility with ``AsyncResult``. - -- **Results**: ``.forget()`` now also clears the local cache. - -- **Results**: Fixed problem with multiple calls to ``result._set_cache`` - (Issue #1940). - -- **Results**: ``join_native`` populated result cache even if disabled. - -- **Results**: The YAML result serializer should now be able to handle storing - exceptions. - -- **Worker**: No longer sends task error emails for expected errors (in - ``@task(throws=(..., )))``. - -- **Canvas**: Fixed problem with exception deserialization when using - the JSON serializer (Issue #1987). - -- **Eventlet**: Fixes crash when ``celery.contrib.batches`` attempted to - cancel a non-existing timer (Issue #1984). - -- Can now import ``celery.version_info_t``, and ``celery.five`` (Issue #1968). - - -.. _version-3.1.10: - -3.1.10 -====== -:release-date: 2014-03-22 09:40 P.M UTC -:release-by: Ask Solem - -- **Requirements**: - - - Now depends on :ref:`Kombu 3.0.14 `. - -- **Redis:** Important note about events (Issue #1882). - - There is a new transport option for Redis that enables monitors - to filter out unwanted events. Enabling this option in the workers - will increase performance considerably: - - .. code-block:: python - - BROKER_TRANSPORT_OPTIONS = {'fanout_patterns': True} - - Enabling this option means that your workers will not be able to see - workers with the option disabled (or is running an older version of - Celery), so if you do enable it then make sure you do so on all - nodes. - - See :ref:`redis-caveat-fanout-patterns`. - - This will be the default in Celery 3.2. - -- **Results**: The :class:`@AsyncResult` object now keeps a local cache - of the final state of the task. - - This means that the global result cache can finally be disabled, - and you can do so by setting :setting:`CELERY_MAX_CACHED_RESULTS` to - :const:`-1`. The lifetime of the cache will then be bound to the - lifetime of the result object, which will be the default behavior - in Celery 3.2. - -- **Events**: The "Substantial drift" warning message is now logged once - per node name only (Issue #1802). - -- **Worker**: Ability to use one log file per child process when using the - prefork pool. - - This can be enabled by using the new ``%i`` and ``%I`` format specifiers - for the log file name. See :ref:`worker-files-process-index`. - -- **Redis**: New experimental chord join implementation. - - This is an optimization for chords when using the Redis result backend, - where the join operation is now considerably faster and using less - resources than the previous strategy. - - The new option can be set in the result backend URL: - - CELERY_RESULT_BACKEND = 'redis://localhost?new_join=1' - - This must be enabled manually as it's incompatible - with workers and clients not using it, so be sure to enable - the option in all clients and workers if you decide to use it. - -- **Multi**: With ``-opt:index`` (e.g. :option:`-c:1`) the index now always refers - to the position of a node in the argument list. - - This means that referring to a number will work when specifying a list - of node names and not just for a number range: - - .. code-block:: bash - - celery multi start A B C D -c:1 4 -c:2-4 8 - - In this example ``1`` refers to node A (as it's the first node in the - list). - -- **Signals**: The sender argument to ``Signal.connect`` can now be a proxy - object, which means that it can be used with the task decorator - (Issue #1873). - -- **Task**: A regression caused the ``queue`` argument to ``Task.retry`` to be - ignored (Issue #1892). - -- **App**: Fixed error message for :meth:`~@Celery.config_from_envvar`. - - Fix contributed by Dmitry Malinovsky. - -- **Canvas**: Chords can now contain a group of other chords (Issue #1921). - -- **Canvas**: Chords can now be combined when using the amqp result backend - (a chord where the callback is also a chord). - -- **Canvas**: Calling ``result.get()`` for a chain task will now complete - even if one of the tasks in the chain is ``ignore_result=True`` - (Issue #1905). - -- **Canvas**: Worker now also logs chord errors. - -- **Canvas**: A chord task raising an exception will now result in - any errbacks (``link_error``) to the chord callback to also be called. - -- **Results**: Reliability improvements to the SQLAlchemy database backend - (Issue #1786). - - Previously the connection from the ``MainProcess`` was improperly - inherited by child processes. - - Fix contributed by Ionel Cristian Mărieș. - -- **Task**: Task callbacks and errbacks are now called using the group - primitive. - -- **Task**: ``Task.apply`` now properly sets ``request.headers`` - (Issue #1874). - -- **Worker**: Fixed ``UnicodeEncodeError`` occuring when worker is started - by `supervisord`. - - Fix contributed by Codeb Fan. - -- **Beat**: No longer attempts to upgrade a newly created database file - (Issue #1923). - -- **Beat**: New setting :setting:``CELERYBEAT_SYNC_EVERY`` can be be used - to control file sync by specifying the number of tasks to send between - each sync. - - Contributed by Chris Clark. - -- **Commands**: :program:`celery inspect memdump` no longer crashes - if the :mod:`psutil` module is not installed (Issue #1914). - -- **Worker**: Remote control commands now always accepts json serialized - messages (Issue #1870). - -- **Worker**: Gossip will now drop any task related events it receives - by mistake (Issue #1882). - - -.. _version-3.1.9: - -3.1.9 -===== -:release-date: 2014-02-10 06:43 P.M UTC -:release-by: Ask Solem - -- **Requirements**: - - - Now depends on :ref:`Kombu 3.0.12 `. - -- **Prefork pool**: Better handling of exiting child processes. - - Fix contributed by Ionel Cristian Mărieș. - -- **Prefork pool**: Now makes sure all file descriptors are removed - from the hub when a process is cleaned up. - - Fix contributed by Ionel Cristian Mărieș. - -- **New Sphinx extension**: for autodoc documentation of tasks: - :mod:`celery.contrib.spinx` (Issue #1833). - -- **Django**: Now works with Django 1.7a1. - -- **Task**: Task.backend is now a property that forwards to ``app.backend`` - if no custom backend has been specified for the task (Issue #1821). - -- **Generic init scripts**: Fixed bug in stop command. - - Fix contributed by Rinat Shigapov. - -- **Generic init scripts**: Fixed compatibility with GNU :manpage:`stat`. - - Fix contributed by Paul Kilgo. - -- **Generic init scripts**: Fixed compatibility with the minimal - :program:`dash` shell (Issue #1815). - -- **Commands**: The :program:`celery amqp basic.publish` command was not - working properly. - - Fix contributed by Andrey Voronov. - -- **Commands**: Did no longer emit an error message if the pidfile exists - and the process is still alive (Issue #1855). - -- **Commands**: Better error message for missing arguments to preload - options (Issue #1860). - -- **Commands**: :program:`celery -h` did not work because of a bug in the - argument parser (Issue #1849). - -- **Worker**: Improved error message for message decoding errors. - -- **Time**: Now properly parses the `Z` timezone specifier in ISO 8601 date - strings. - - Fix contributed by Martin Davidsson. - -- **Worker**: Now uses the *negotiated* heartbeat value to calculate - how often to run the heartbeat checks. - -- **Beat**: Fixed problem with beat hanging after the first schedule - iteration (Issue #1822). - - Fix contributed by Roger Hu. - -- **Signals**: The header argument to :signal:`before_task_publish` is now - always a dictionary instance so that signal handlers can add headers. - -- **Worker**: A list of message headers is now included in message related - errors. - -.. _version-3.1.8: - -3.1.8 -===== -:release-date: 2014-01-17 10:45 P.M UTC -:release-by: Ask Solem - -- **Requirements**: - - - Now depends on :ref:`Kombu 3.0.10 `. - - - Now depends on `billiard 3.3.0.14`_. - -.. _`billiard 3.3.0.14`: - https://github.com/celery/billiard/blob/master/CHANGES.txt - -- **Worker**: The event loop was not properly reinitialized at consumer restart - which would force the worker to continue with a closed ``epoll`` instance on - Linux, resulting in a crash. - -- **Events:** Fixed issue with both heartbeats and task events that could - result in the data not being kept in sorted order. - - As a result this would force the worker to log "heartbeat missed" - events even though the remote node was sending heartbeats in a timely manner. - -- **Results:** The pickle serializer no longer converts group results to tuples, - and will keep the original type (*Issue #1750*). - -- **Results:** ``ResultSet.iterate`` is now pending deprecation. - - The method will be deprecated in version 3.2 and removed in version 3.3. - - Use ``result.get(callback=)`` (or ``result.iter_native()`` where available) - instead. - -- **Worker**\|eventlet/gevent: A regression caused ``Ctrl+C`` to be ineffective - for shutdown. - -- **Redis result backend:** Now using a pipeline to store state changes - for improved performance. - - Contributed by Pepijn de Vos. - -- **Redis result backend:** Will now retry storing the result if disconnected. - -- **Worker**\|gossip: Fixed attribute error occurring when another node leaves. - - Fix contributed by Brodie Rao. - -- **Generic init scripts:** Now runs a check at startup to verify - that any configuration scripts are owned by root and that they - are not world/group writeable. - - The init script configuration is a shell script executed by root, - so this is a preventive measure to ensure that users do not - leave this file vulnerable to changes by unprivileged users. - - .. note:: - - Note that upgrading celery will not update the init scripts, - instead you need to manually copy the improved versions from the - source distribution: - https://github.com/celery/celery/tree/3.1/extra/generic-init.d - -- **Commands**: The :program:`celery purge` command now warns that the operation - will delete all tasks and prompts the user for confirmation. - - A new :option:`-f` was added that can be used to disable - interactive mode. - -- **Task**: ``.retry()`` did not raise the value provided in the ``exc`` argument - when called outside of an error context (*Issue #1755*). - -- **Commands:** The :program:`celery multi` command did not forward command - line configuration to the target workers. - - The change means that multi will forward the special ``--`` argument and - configuration content at the end of the arguments line to the specified - workers. - - Example using command-line configuration to set a broker heartbeat - from :program:`celery multi`: - - .. code-block:: bash - - $ celery multi start 1 -c3 -- broker.heartbeat=30 - - Fix contributed by Antoine Legrand. - -- **Canvas:** ``chain.apply_async()`` now properly forwards execution options. - - Fix contributed by Konstantin Podshumok. - -- **Redis result backend:** Now takes ``connection_pool`` argument that can be - used to change the connection pool class/constructor. - -- **Worker:** Now truncates very long arguments and keyword arguments logged by - the pool at debug severity. - -- **Worker:** The worker now closes all open files on :sig:`SIGHUP` (regression) - (*Issue #1768*). - - Fix contributed by Brodie Rao - -- **Worker:** Will no longer accept remote control commands while the - worker startup phase is incomplete (*Issue #1741*). - -- **Commands:** The output of the event dump utility - (:program:`celery events -d`) can now be piped into other commands. - -- **Documentation:** The RabbitMQ installation instructions for OS X was - updated to use modern homebrew practices. - - Contributed by Jon Chen. - -- **Commands:** The :program:`celery inspect conf` utility now works. - -- **Commands:** The :option:`-no-color` argument was not respected by - all commands (*Issue #1799*). - -- **App:** Fixed rare bug with ``autodiscover_tasks()`` (*Issue #1797*). - -- **Distribution:** The sphinx docs will now always add the parent directory - to path so that the current celery source code is used as a basis for - API documentation (*Issue #1782*). - -- **Documentation:** Supervisord examples contained an extraneous '-' in a - `--logfile` argument example. - - Fix contributed by Mohammad Almeer. - -.. _version-3.1.7: - -3.1.7 -===== -:release-date: 2013-12-17 06:00 P.M UTC -:release-by: Ask Solem - -.. _v317-important: - -Important Notes ---------------- - -Init script security improvements ---------------------------------- - -Where the generic init scripts (for ``celeryd``, and ``celerybeat``) before -delegated the responsibility of dropping privileges to the target application, -it will now use ``su`` instead, so that the Python program is not trusted -with superuser privileges. - -This is not in reaction to any known exploit, but it will -limit the possibility of a privilege escalation bug being abused in the -future. - -You have to upgrade the init scripts manually from this directory: -https://github.com/celery/celery/tree/3.1/extra/generic-init.d - -AMQP result backend -~~~~~~~~~~~~~~~~~~~ - -The 3.1 release accidentally left the amqp backend configured to be -non-persistent by default. - -Upgrading from 3.0 would give a "not equivalent" error when attempting to -set or retrieve results for a task. That is unless you manually set the -persistence setting:: - - CELERY_RESULT_PERSISTENT = True - -This version restores the previous value so if you already forced -the upgrade by removing the existing exchange you must either -keep the configuration by setting ``CELERY_RESULT_PERSISTENT = False`` -or delete the ``celeryresults`` exchange again. - -Synchronous subtasks -~~~~~~~~~~~~~~~~~~~~ - -Tasks waiting for the result of a subtask will now emit -a :exc:`RuntimeWarning` warning when using the prefork pool, -and in 3.2 this will result in an exception being raised. - -It's not legal for tasks to block by waiting for subtasks -as this is likely to lead to resource starvation and eventually -deadlock when using the prefork pool (see also :ref:`task-synchronous-subtasks`). - -If you really know what you are doing you can avoid the warning (and -the future exception being raised) by moving the operation in a whitelist -block: - -.. code-block:: python - - from celery.result import allow_join_result - - @app.task - def misbehaving(): - result = other_task.delay() - with allow_join_result(): - result.get() - -Note also that if you wait for the result of a subtask in any form -when using the prefork pool you must also disable the pool prefetching -behavior with the worker :ref:`-Ofair option `. - -.. _v317-fixes: - -Fixes ------ - -- Now depends on :ref:`Kombu 3.0.8 `. - -- Now depends on :mod:`billiard` 3.3.0.13 - -- Events: Fixed compatibility with non-standard json libraries - that sends float as :class:`decimal.Decimal` (Issue #1731) - -- Events: State worker objects now always defines attributes: - ``active``, ``processed``, ``loadavg``, ``sw_ident``, ``sw_ver`` - and ``sw_sys``. - -- Worker: Now keeps count of the total number of tasks processed, - not just by type (``all_active_count``). - -- Init scripts: Fixed problem with reading configuration file - when the init script is symlinked to a runlevel (e.g. ``S02celeryd``). - (Issue #1740). - - This also removed a rarely used feature where you can symlink the script - to provide alternative configurations. You instead copy the script - and give it a new name, but perhaps a better solution is to provide - arguments to ``CELERYD_OPTS`` to separate them: - - .. code-block:: bash - - CELERYD_NODES="X1 X2 Y1 Y2" - CELERYD_OPTS="-A:X1 x -A:X2 x -A:Y1 y -A:Y2 y" - -- Fallback chord unlock task is now always called after the chord header - (Issue #1700). - - This means that the unlock task will not be started if there's - an error sending the header. - -- Celery command: Fixed problem with arguments for some control commands. - - Fix contributed by Konstantin Podshumok. - -- Fixed bug in ``utcoffset`` where the offset when in DST would be - completely wrong (Issue #1743). - -- Worker: Errors occurring while attempting to serialize the result of a - task will now cause the task to be marked with failure and a - :class:`kombu.exceptions.EncodingError` error. - - Fix contributed by Ionel Cristian Mărieș. - -- Worker with ``-B`` argument did not properly shut down the beat instance. - -- Worker: The ``%n`` and ``%h`` formats are now also supported by the - :option:`--logfile`, :option:`--pidfile` and :option:`--statedb` arguments. - - Example: - - .. code-block:: bash - - $ celery -A proj worker -n foo@%h --logfile=%n.log --statedb=%n.db - -- Redis/Cache result backends: Will now timeout if keys evicted while trying - to join a chord. - -- The fallbock unlock chord task now raises :exc:`Retry` so that the - retry even is properly logged by the worker. - -- Multi: Will no longer apply Eventlet/gevent monkey patches (Issue #1717). - -- Redis result backend: Now supports UNIX sockets. - - Like the Redis broker transport the result backend now also supports - using ``redis+socket:///tmp/redis.sock`` URLs. - - Contributed by Alcides Viamontes Esquivel. - -- Events: Events sent by clients was mistaken for worker related events - (Issue #1714). - - For ``events.State`` the tasks now have a ``Task.client`` attribute - that is set when a ``task-sent`` event is being received. - - Also, a clients logical clock is not in sync with the cluster so - they live in a "time bubble". So for this reason monitors will no - longer attempt to merge with the clock of an event sent by a client, - instead it will fake the value by using the current clock with - a skew of -1. - -- Prefork pool: The method used to find terminated processes was flawed - in that it did not also take into account missing popen objects. - -- Canvas: ``group`` and ``chord`` now works with anon signatures as long - as the group/chord object is associated with an app instance (Issue #1744). - - You can pass the app by using ``group(..., app=app)``. - -.. _version-3.1.6: - -3.1.6 -===== -:release-date: 2013-12-02 06:00 P.M UTC -:release-by: Ask Solem - -- Now depends on :mod:`billiard` 3.3.0.10. - -- Now depends on :ref:`Kombu 3.0.7 `. - -- Fixed problem where Mingle caused the worker to hang at startup - (Issue #1686). - -- Beat: Would attempt to drop privileges twice (Issue #1708). - -- Windows: Fixed error with ``geteuid`` not being available (Issue #1676). - -- Tasks can now provide a list of expected error classes (Issue #1682). - - The list should only include errors that the task is expected to raise - during normal operation:: - - @task(throws=(KeyError, HttpNotFound)) - - What happens when an exceptions is raised depends on the type of error: - - - Expected errors (included in ``Task.throws``) - - Will be logged using severity ``INFO``, and traceback is excluded. - - - Unexpected errors - - Will be logged using severity ``ERROR``, with traceback included. - -- Cache result backend now compatible with Python 3 (Issue #1697). - -- CentOS init script: Now compatible with sys-v style init symlinks. - - Fix contributed by Jonathan Jordan. - -- Events: Fixed problem when task name is not defined (Issue #1710). - - Fix contributed by Mher Movsisyan. - -- Task: Fixed unbound local errors (Issue #1684). - - Fix contributed by Markus Ullmann. - -- Canvas: Now unrolls groups with only one task (optimization) (Issue #1656). - -- Task: Fixed problem with eta and timezones. - - Fix contributed by Alexander Koval. - -- Django: Worker now performs model validation (Issue #1681). - -- Task decorator now emits less confusing errors when used with - incorrect arguments (Issue #1692). - -- Task: New method ``Task.send_event`` can be used to send custom events - to Flower and other monitors. - -- Fixed a compatibility issue with non-abstract task classes - -- Events from clients now uses new node name format (``gen@``). - -- Fixed rare bug with Callable not being defined at interpreter shutdown - (Issue #1678). - - Fix contributed by Nick Johnson. - -- Fixed Python 2.6 compatibility (Issue #1679). - -.. _version-3.1.5: - -3.1.5 -===== -:release-date: 2013-11-21 06:20 P.M UTC -:release-by: Ask Solem - -- Now depends on :ref:`Kombu 3.0.6 `. - -- Now depends on :mod:`billiard` 3.3.0.8 - -- App: ``config_from_object`` is now lazy (Issue #1665). - -- App: ``autodiscover_tasks`` is now lazy. - - Django users should now wrap access to the settings object - in a lambda:: - - app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) - - this ensures that the settings object is not prepared - prematurely. - -- Fixed regression for ``--app`` argument experienced by - some users (Issue #1653). - -- Worker: Now respects the ``--uid`` and ``--gid`` arguments - even if ``--detach`` is not enabled. - -- Beat: Now respects the ``--uid`` and ``--gid`` arguments - even if ``--detach`` is not enabled. - -- Python 3: Fixed unorderable error occuring with the worker ``-B`` - argument enabled. - -- ``celery.VERSION`` is now a named tuple. - -- ``maybe_signature(list)`` is now applied recursively (Issue #1645). - -- ``celery shell`` command: Fixed ``IPython.frontend`` deprecation warning. - -- The default app no longer includes the builtin fixups. - - This fixes a bug where ``celery multi`` would attempt - to load the Django settings module before entering - the target working directory. - -- The Django daemonization tutorial was changed. - - Users no longer have to explicitly export ``DJANGO_SETTINGS_MODULE`` - in :file:`/etc/default/celeryd` when the new project layout is used. - -- Redis result backend: expiry value can now be 0 (Issue #1661). - -- Censoring settings now accounts for non-string keys (Issue #1663). - -- App: New ``autofinalize`` option. - - Apps are automatically finalized when the task registry is accessed. - You can now disable this behavior so that an exception is raised - instead. - - Example: - - .. code-block:: python - - app = Celery(autofinalize=False) - - # raises RuntimeError - tasks = app.tasks - - @app.task - def add(x, y): - return x + y - - # raises RuntimeError - add.delay(2, 2) - - app.finalize() - # no longer raises: - tasks = app.tasks - add.delay(2, 2) - -- The worker did not send monitoring events during shutdown. - -- Worker: Mingle and gossip is now automatically disabled when - used with an unsupported transport (Issue #1664). - -- ``celery`` command: Preload options now supports - the rare ``--opt value`` format (Issue #1668). - -- ``celery`` command: Accidentally removed options - appearing before the subcommand, these are now moved to the end - instead. - -- Worker now properly responds to ``inspect stats`` commands - even if received before startup is complete (Issue #1659). - -- :signal:`task_postrun` is now sent within a finally block, to make - sure the signal is always sent. - -- Beat: Fixed syntax error in string formatting. - - Contributed by nadad. - -- Fixed typos in the documentation. - - Fixes contributed by Loic Bistuer, sunfinite. - -- Nested chains now works properly when constructed using the - ``chain`` type instead of the ``|`` operator (Issue #1656). - -.. _version-3.1.4: - -3.1.4 -===== -:release-date: 2013-11-15 11:40 P.M UTC -:release-by: Ask Solem - -- Now depends on :ref:`Kombu 3.0.5 `. - -- Now depends on :mod:`billiard` 3.3.0.7 - -- Worker accidentally set a default socket timeout of 5 seconds. - -- Django: Fixup now sets the default app so that threads will use - the same app instance (e.g. for manage.py runserver). - -- Worker: Fixed Unicode error crash at startup experienced by some users. - -- Calling ``.apply_async`` on an empty chain now works again (Issue #1650). - -- The ``celery multi show`` command now generates the same arguments - as the start command does. - -- The ``--app`` argument could end up using a module object instead - of an app instance (with a resulting crash). - -- Fixed a syntax error problem in the celerybeat init script. - - Fix contributed by Vsevolod. - -- Tests now passing on PyPy 2.1 and 2.2. - -.. _version-3.1.3: - -3.1.3 -===== -:release-date: 2013-11-13 00:55 A.M UTC -:release-by: Ask Solem - -- Fixed compatibility problem with Python 2.7.0 - 2.7.5 (Issue #1637) - - ``unpack_from`` started supporting ``memoryview`` arguments - in Python 2.7.6. - -- Worker: :option:`-B` argument accidentally closed files used - for logging. - -- Task decorated tasks now keep their docstring (Issue #1636) - -.. _version-3.1.2: - -3.1.2 -===== -:release-date: 2013-11-12 08:00 P.M UTC -:release-by: Ask Solem - -- Now depends on :mod:`billiard` 3.3.0.6 - -- No longer needs the billiard C extension to be installed. - -- The worker silently ignored task errors. - -- Django: Fixed ``ImproperlyConfigured`` error raised - when no database backend specified. - - Fix contributed by j0hnsmith - -- Prefork pool: Now using ``_multiprocessing.read`` with ``memoryview`` - if available. - -- ``close_open_fds`` now uses ``os.closerange`` if available. - -- ``get_fdmax`` now takes value from ``sysconfig`` if possible. - -.. _version-3.1.1: - -3.1.1 -===== -:release-date: 2013-11-11 06:30 P.M UTC -:release-by: Ask Solem - -- Now depends on :mod:`billiard` 3.3.0.4. - -- Python 3: Fixed compatibility issues. - -- Windows: Accidentally showed warning that the billiard C extension - was not installed (Issue #1630). - -- Django: Tutorial updated with a solution that sets a default - :envvar:`DJANGO_SETTINGS_MODULE` so that it doesn't have to be typed - in with the :program:`celery` command. - - Also fixed typos in the tutorial, and added the settings - required to use the Django database backend. - - Thanks to Chris Ward, orarbel. - -- Django: Fixed a problem when using the Django settings in Django 1.6. - -- Django: Fixup should not be applied if the django loader is active. - -- Worker: Fixed attribute error for ``human_write_stats`` when using the - compatibility prefork pool implementation. - -- Worker: Fixed compatibility with billiard without C extension. - -- Inspect.conf: Now supports a ``with_defaults`` argument. - -- Group.restore: The backend argument was not respected. - -.. _version-3.1.0: - -3.1.0 +3.2.0 ======= -:release-date: 2013-11-09 11:00 P.M UTC -:release-by: Ask Solem +:release-date: TBA +:release-by: -See :ref:`whatsnew-3.1`. +See :ref:`whatsnew-3.2`. diff --git a/docs/history/changelog-3.1.rst b/docs/history/changelog-3.1.rst new file mode 100644 index 000000000..f128e79a1 --- /dev/null +++ b/docs/history/changelog-3.1.rst @@ -0,0 +1,1260 @@ +.. _changelog-3.1: + +=============================== + Change history for Celery 3.1 +=============================== + +This document contains change notes for bugfix releases in the 3.1.x series +(Cipater), please see :ref:`whatsnew-3.1` for an overview of what's +new in Celery 3.1. + +If you're looking for versions prior to 3.1.x you should go to :ref:`history`. + +.. _version-3.1.17: + +3.1.17 +====== + +.. admonition:: CELERYD_FORCE_EXECV should not be used. + + Please disable this option if you're using the RabbitMQ or Redis + transports. + + Keeping this option enabled in 3.1 means the async based worker will + be disabled, so using is more likely to lead to trouble than doing + anything good. + +- **Requirements** + + - Now depends on :ref:`Kombu 3.0.24 `. + + - Now depends on :mod:`billiard` 3.3.0.19. + +- **Task**: The timing for ETA/countdown tasks were off after the example ``LocalTimezone`` + implementation in the Python documentation no longer works in Python 3.4. + (Issue #2306). + +- **Task**: Raising :exc:`~celery.exceptions.Ignore` no longer sends + ``task-failed`` event (Issue #2365). + +- **Redis result backend**: Fixed errors about unbound local ``self``. + + Fix contributed by Thomas French. + +- **Task**: Callbacks was not called properly if ``link`` was a list of + signatures (Issuse #2350). + +- **Canvas**: chain and group now handles json serialized signatures + (Issue #2076). + +- **Canvas**: The ``chord_size`` attribute is now set for all canvas primitives, + making sure more combinations will work with the ``new_join`` optimization + for Redis (Issue #2339). + +- **Task**: Fixed problem with app not being properly propagated to + ``trace_task`` in all cases. + + Fix contributed by kristaps. + +- **Worker**: Expires from task message now associated with a timezone. + + Fix contributed by Albert Wang. + +- **Cassandra result backend**: Fixed problems when using detailed mode. + + When using the Cassandra backend in detailed mode, a regression + caused errors when attempting to retrieve results. + + Fix contributed by Gino Ledesma. + +- **Task**: Exception info was not properly set for tasks raising + :exc:`~celery.exceptions.Reject` (Issue #2043). + +- **Worker**: The set of revokes tasks are now deduplicated when loading from + the worker state database (Issue #2336). + +- **celery.contrib.rdb**: Fixed problems with ``rdb.set_trace`` calling stop + from the wrong frame. + + Fix contributed by llllllllll. + +- **Canvas**: ``chain`` and ``chord`` can now be immutable. + +- **Canvas**: ``chord.apply_async`` will now keep partial args set in + ``self.args`` (Issue #2299). + +- **Results**: Small refactoring so that results are decoded the same way in + all result backends. + +- **Logging**: The ``processName`` format was introduced in Py2.6.2 so for + compatibility this format is now excluded when using earlier versions + (Issue #1644). + +.. _version-3.1.16: + +3.1.16 +====== +:release-date: 2014-10-03 06:00 P.M UTC +:release-by: Ask Solem + +- **Worker**: 3.1.15 broke ``-Ofair`` behavior (Issue #2286). + + This regression could result in all tasks executing + in a single child process if ``-Ofair`` was enabled. + +- **Canvas**: ``celery.signature`` now properly forwards app argument + in all cases. + +- **Task**: ``.retry()`` did not raise the exception correctly + when called without a current exception. + + Fix contributed by Andrea Rabbaglietti. + +- **Worker**: The ``enable_events`` remote control command + disabled worker-related events by mistake (Issue #2272). + + Fix contributed by Konstantinos Koukopoulos. + +- **Django**: Adds support for Django 1.7 class names in INSTALLED_APPS + when using ``app.autodiscover_tasks()`` (Issue #2248). + +- **Sphinx**: ``celery.contrib.sphinx`` now uses ``getfullargspec`` + on Python 3 (Issue #2302). + +- **Redis/Cache Backends**: Chords will now run at most once if one or more tasks + in the chord are executed multiple times for some reason. + +.. _version-3.1.15: + +3.1.15 +====== +:release-date: 2014-09-14 11:00 P.M UTC +:release-by: Ask Solem + +- **Django**: Now makes sure ``django.setup()`` is called + before importing any task modules (Django 1.7 compatibility, Issue #2227) + +- **Results**: ``result.get()`` was misbehaving by calling + ``backend.get_task_meta`` in a finally call leading to + AMQP result backend queues not being properly cleaned up (Issue #2245). + +.. _version-3.1.14: + +3.1.14 +====== +:release-date: 2014-09-08 03:00 P.M UTC +:release-by: Ask Solem + +- **Requirements** + + - Now depends on :ref:`Kombu 3.0.22 `. + +- **Init scripts**: The generic worker init scripts ``status`` command + now gets an accurate pidfile list (Issue #1942). + +- **Init scripts**: The generic beat script now implements the ``status`` + command. + + Contributed by John Whitlock. + +- **Commands**: Multi now writes informational output to stdout instead of stderr. + +- **Worker**: Now ignores not implemented error for ``pool.restart`` + (Issue #2153). + +- **Task**: Retry no longer raises retry exception when executed in eager + mode (Issue #2164). + +- **AMQP Result backend**: Now ensured ``on_interval`` is called at least + every second for blocking calls to properly propagate parent errors. + +- **Django**: Compatibility with Django 1.7 on Windows (Issue #2126). + +- **Programs**: `--umask` argument can be now specified in both octal (if starting + with 0) or decimal. + + +.. _version-3.1.13: + +3.1.13 +====== + +Security Fixes +-------------- + +* [Security: `CELERYSA-0002`_] Insecure default umask. + + The built-in utility used to daemonize the Celery worker service sets + an insecure umask by default (umask 0). + + This means that any files or directories created by the worker will + end up having world-writable permissions. + + Special thanks to Red Hat for originally discovering and reporting the + issue! + + This version will no longer set a default umask by default, so if unset + the umask of the parent process will be used. + +.. _`CELERYSA-0002`: + http://github.com/celery/celery/tree/master/docs/sec/CELERYSA-0002.txt + +News +---- + +- **Requirements** + + - Now depends on :ref:`Kombu 3.0.21 `. + + - Now depends on :mod:`billiard` 3.3.0.18. + + +- **App**: ``backend`` argument now also sets the :setting:`CELERY_RESULT_BACKEND` + setting. + +- **Task**: ``signature_from_request`` now propagates ``reply_to`` so that + the RPC backend works with retried tasks (Issue #2113). + +- **Task**: ``retry`` will no longer attempt to requeue the task if sending + the retry message fails. + + Unrelated exceptions being raised could cause a message loop, so it was + better to remove this behavior. + +- **Beat**: Accounts for standard 1ms drift by always waking up 0.010s + earlier. + + This will adjust the latency so that the periodic tasks will not move + 1ms after every invocation. + +- Documentation fixes + + Contributed by Yuval Greenfield, Lucas Wiman, nicholsonjf + +- **Worker**: Removed an outdated assert statement that could lead to errors + being masked (Issue #2086). + + + +.. _version-3.1.12: + +3.1.12 +====== +:release-date: 2014-06-09 10:12 P.M UTC +:release-by: Ask Solem + +- **Requirements** + + Now depends on :ref:`Kombu 3.0.19 `. + +- **App**: Connections were not being closed after fork due to an error in the + after fork handler (Issue #2055). + + This could manifest itself by causing framing errors when using RabbitMQ. + (``Unexpected frame``). + +- **Django**: ``django.setup()`` was being called too late when + using Django 1.7 (Issue #1802). + +- **Django**: Fixed problems with event timezones when using Django + (``Substantial drift``). + + Celery did not take into account that Django modifies the + ``time.timeone`` attributes and friends. + +- **Canvas**: ``Signature.link`` now works when the link option is a scalar + value (Issue #2019). + +- **Prefork pool**: Fixed race conditions for when file descriptors are + removed from the event loop. + + Fix contributed by Roger Hu. + +- **Prefork pool**: Improved solution for dividing tasks between child + processes. + + This change should improve performance when there are many child + processes, and also decrease the chance that two subsequent tasks are + written to the same child process. + +- **Worker**: Now ignores unknown event types, instead of crashing. + + Fix contributed by Illes Solt. + +- **Programs**: :program:`celery worker --detach` no longer closes open file + descriptors when :envvar:`C_FAKEFORK` is used so that the workers output + can be seen. + +- **Programs**: The default working directory for :program:`celery worker + --detach` is now the current working directory, not ``/``. + +- **Canvas**: ``signature(s, app=app)`` did not upgrade serialized signatures + to their original class (``subtask_type``) when the ``app`` keyword argument + was used. + +- **Control**: The ``duplicate nodename`` warning emitted by control commands + now shows the duplicate node name. + +- **Tasks**: Can now call ``ResultSet.get()`` on a result set without members. + + Fix contributed by Alexey Kotlyarov. + +- **App**: Fixed strange traceback mangling issue for + ``app.connection_or_acquire``. + +- **Programs**: The :program:`celery multi stopwait` command is now documented + in usage. + +- **Other**: Fixed cleanup problem with ``PromiseProxy`` when an error is + raised while trying to evaluate the promise. + +- **Other**: The utility used to censor configuration values now handles + non-string keys. + + Fix contributed by Luke Pomfrey. + +- **Other**: The ``inspect conf`` command did not handle non-string keys well. + + Fix contributed by Jay Farrimond. + +- **Programs**: Fixed argument handling problem in + :program:`celery worker --detach`. + + Fix contributed by Dmitry Malinovsky. + +- **Programs**: :program:`celery worker --detach` did not forward working + directory option (Issue #2003). + +- **Programs**: :program:`celery inspect registered` no longer includes + the list of built-in tasks. + +- **Worker**: The ``requires`` attribute for boot steps were not being handled + correctly (Issue #2002). + +- **Eventlet**: The eventlet pool now supports the ``pool_grow`` and + ``pool_shrink`` remote control commands. + + Contributed by Mher Movsisyan. + +- **Eventlet**: The eventlet pool now implements statistics for + :program:``celery inspect stats``. + + Contributed by Mher Movsisyan. + +- **Documentation**: Clarified ``Task.rate_limit`` behavior. + + Contributed by Jonas Haag. + +- **Documentation**: ``AbortableTask`` examples now updated to use the new + API (Issue #1993). + +- **Documentation**: The security documentation examples used an out of date + import. + + Fix contributed by Ian Dees. + +- **Init scripts**: The CentOS init scripts did not quote + :envvar:`CELERY_CHDIR`. + + Fix contributed by ffeast. + +.. _version-3.1.11: + +3.1.11 +====== +:release-date: 2014-04-16 11:00 P.M UTC +:release-by: Ask Solem + +- **Now compatible with RabbitMQ 3.3.0** + + You need to run Celery 3.1.11 or later when using RabbitMQ 3.3, + and if you use the ``librabbitmq`` module you also have to upgrade + to librabbitmq 1.5.0: + + .. code-block:: bash + + $ pip install -U librabbitmq + +- **Requirements**: + + - Now depends on :ref:`Kombu 3.0.15 `. + + - Now depends on `billiard 3.3.0.17`_. + + - Bundle ``celery[librabbitmq]`` now depends on :mod:`librabbitmq` 1.5.0. + +.. _`billiard 3.3.0.17`: + https://github.com/celery/billiard/blob/master/CHANGES.txt + +- **Tasks**: The :setting:`CELERY_DEFAULT_DELIVERY_MODE` setting was being + ignored (Issue #1953). + +- **Worker**: New :option:`--heartbeat-interval` can be used to change the + time (in seconds) between sending event heartbeats. + + Contributed by Matthew Duggan and Craig Northway. + +- **App**: Fixed memory leaks occurring when creating lots of temporary + app instances (Issue #1949). + +- **MongoDB**: SSL configuration with non-MongoDB transport breaks MongoDB + results backend (Issue #1973). + + Fix contributed by Brian Bouterse. + +- **Logging**: The color formatter accidentally modified ``record.msg`` + (Issue #1939). + +- **Results**: Fixed problem with task trails being stored multiple times, + causing ``result.collect()`` to hang (Issue #1936, Issue #1943). + +- **Results**: ``ResultSet`` now implements a ``.backend`` attribute for + compatibility with ``AsyncResult``. + +- **Results**: ``.forget()`` now also clears the local cache. + +- **Results**: Fixed problem with multiple calls to ``result._set_cache`` + (Issue #1940). + +- **Results**: ``join_native`` populated result cache even if disabled. + +- **Results**: The YAML result serializer should now be able to handle storing + exceptions. + +- **Worker**: No longer sends task error emails for expected errors (in + ``@task(throws=(..., )))``. + +- **Canvas**: Fixed problem with exception deserialization when using + the JSON serializer (Issue #1987). + +- **Eventlet**: Fixes crash when ``celery.contrib.batches`` attempted to + cancel a non-existing timer (Issue #1984). + +- Can now import ``celery.version_info_t``, and ``celery.five`` (Issue #1968). + + +.. _version-3.1.10: + +3.1.10 +====== +:release-date: 2014-03-22 09:40 P.M UTC +:release-by: Ask Solem + +- **Requirements**: + + - Now depends on :ref:`Kombu 3.0.14 `. + +- **Results**: + + Reliability improvements to the SQLAlchemy database backend. Previously the + connection from the MainProcess was improperly shared with the workers. + (Issue #1786) + +- **Redis:** Important note about events (Issue #1882). + + There is a new transport option for Redis that enables monitors + to filter out unwanted events. Enabling this option in the workers + will increase performance considerably: + + .. code-block:: python + + BROKER_TRANSPORT_OPTIONS = {'fanout_patterns': True} + + Enabling this option means that your workers will not be able to see + workers with the option disabled (or is running an older version of + Celery), so if you do enable it then make sure you do so on all + nodes. + + See :ref:`redis-caveats`. + + This will be the default in Celery 3.2. + +- **Results**: The :class:`@AsyncResult` object now keeps a local cache + of the final state of the task. + + This means that the global result cache can finally be disabled, + and you can do so by setting :setting:`CELERY_MAX_CACHED_RESULTS` to + :const:`-1`. The lifetime of the cache will then be bound to the + lifetime of the result object, which will be the default behavior + in Celery 3.2. + +- **Events**: The "Substantial drift" warning message is now logged once + per node name only (Issue #1802). + +- **Worker**: Ability to use one log file per child process when using the + prefork pool. + + This can be enabled by using the new ``%i`` and ``%I`` format specifiers + for the log file name. See :ref:`worker-files-process-index`. + +- **Redis**: New experimental chord join implementation. + + This is an optimization for chords when using the Redis result backend, + where the join operation is now considerably faster and using less + resources than the previous strategy. + + The new option can be set in the result backend URL: + + CELERY_RESULT_BACKEND = 'redis://localhost?new_join=1' + + This must be enabled manually as it's incompatible + with workers and clients not using it, so be sure to enable + the option in all clients and workers if you decide to use it. + +- **Multi**: With ``-opt:index`` (e.g. :option:`-c:1`) the index now always refers + to the position of a node in the argument list. + + This means that referring to a number will work when specifying a list + of node names and not just for a number range: + + .. code-block:: bash + + celery multi start A B C D -c:1 4 -c:2-4 8 + + In this example ``1`` refers to node A (as it's the first node in the + list). + +- **Signals**: The sender argument to ``Signal.connect`` can now be a proxy + object, which means that it can be used with the task decorator + (Issue #1873). + +- **Task**: A regression caused the ``queue`` argument to ``Task.retry`` to be + ignored (Issue #1892). + +- **App**: Fixed error message for :meth:`~@Celery.config_from_envvar`. + + Fix contributed by Dmitry Malinovsky. + +- **Canvas**: Chords can now contain a group of other chords (Issue #1921). + +- **Canvas**: Chords can now be combined when using the amqp result backend + (a chord where the callback is also a chord). + +- **Canvas**: Calling ``result.get()`` for a chain task will now complete + even if one of the tasks in the chain is ``ignore_result=True`` + (Issue #1905). + +- **Canvas**: Worker now also logs chord errors. + +- **Canvas**: A chord task raising an exception will now result in + any errbacks (``link_error``) to the chord callback to also be called. + +- **Results**: Reliability improvements to the SQLAlchemy database backend + (Issue #1786). + + Previously the connection from the ``MainProcess`` was improperly + inherited by child processes. + + Fix contributed by Ionel Cristian Mărieș. + +- **Task**: Task callbacks and errbacks are now called using the group + primitive. + +- **Task**: ``Task.apply`` now properly sets ``request.headers`` + (Issue #1874). + +- **Worker**: Fixed ``UnicodeEncodeError`` occuring when worker is started + by `supervisord`. + + Fix contributed by Codeb Fan. + +- **Beat**: No longer attempts to upgrade a newly created database file + (Issue #1923). + +- **Beat**: New setting :setting:``CELERYBEAT_SYNC_EVERY`` can be be used + to control file sync by specifying the number of tasks to send between + each sync. + + Contributed by Chris Clark. + +- **Commands**: :program:`celery inspect memdump` no longer crashes + if the :mod:`psutil` module is not installed (Issue #1914). + +- **Worker**: Remote control commands now always accepts json serialized + messages (Issue #1870). + +- **Worker**: Gossip will now drop any task related events it receives + by mistake (Issue #1882). + + +.. _version-3.1.9: + +3.1.9 +===== +:release-date: 2014-02-10 06:43 P.M UTC +:release-by: Ask Solem + +- **Requirements**: + + - Now depends on :ref:`Kombu 3.0.12 `. + +- **Prefork pool**: Better handling of exiting child processes. + + Fix contributed by Ionel Cristian Mărieș. + +- **Prefork pool**: Now makes sure all file descriptors are removed + from the hub when a process is cleaned up. + + Fix contributed by Ionel Cristian Mărieș. + +- **New Sphinx extension**: for autodoc documentation of tasks: + :mod:`celery.contrib.spinx` (Issue #1833). + +- **Django**: Now works with Django 1.7a1. + +- **Task**: Task.backend is now a property that forwards to ``app.backend`` + if no custom backend has been specified for the task (Issue #1821). + +- **Generic init scripts**: Fixed bug in stop command. + + Fix contributed by Rinat Shigapov. + +- **Generic init scripts**: Fixed compatibility with GNU :manpage:`stat`. + + Fix contributed by Paul Kilgo. + +- **Generic init scripts**: Fixed compatibility with the minimal + :program:`dash` shell (Issue #1815). + +- **Commands**: The :program:`celery amqp basic.publish` command was not + working properly. + + Fix contributed by Andrey Voronov. + +- **Commands**: Did no longer emit an error message if the pidfile exists + and the process is still alive (Issue #1855). + +- **Commands**: Better error message for missing arguments to preload + options (Issue #1860). + +- **Commands**: :program:`celery -h` did not work because of a bug in the + argument parser (Issue #1849). + +- **Worker**: Improved error message for message decoding errors. + +- **Time**: Now properly parses the `Z` timezone specifier in ISO 8601 date + strings. + + Fix contributed by Martin Davidsson. + +- **Worker**: Now uses the *negotiated* heartbeat value to calculate + how often to run the heartbeat checks. + +- **Beat**: Fixed problem with beat hanging after the first schedule + iteration (Issue #1822). + + Fix contributed by Roger Hu. + +- **Signals**: The header argument to :signal:`before_task_publish` is now + always a dictionary instance so that signal handlers can add headers. + +- **Worker**: A list of message headers is now included in message related + errors. + +.. _version-3.1.8: + +3.1.8 +===== +:release-date: 2014-01-17 10:45 P.M UTC +:release-by: Ask Solem + +- **Requirements**: + + - Now depends on :ref:`Kombu 3.0.10 `. + + - Now depends on `billiard 3.3.0.14`_. + +.. _`billiard 3.3.0.14`: + https://github.com/celery/billiard/blob/master/CHANGES.txt + +- **Worker**: The event loop was not properly reinitialized at consumer restart + which would force the worker to continue with a closed ``epoll`` instance on + Linux, resulting in a crash. + +- **Events:** Fixed issue with both heartbeats and task events that could + result in the data not being kept in sorted order. + + As a result this would force the worker to log "heartbeat missed" + events even though the remote node was sending heartbeats in a timely manner. + +- **Results:** The pickle serializer no longer converts group results to tuples, + and will keep the original type (*Issue #1750*). + +- **Results:** ``ResultSet.iterate`` is now pending deprecation. + + The method will be deprecated in version 3.2 and removed in version 3.3. + + Use ``result.get(callback=)`` (or ``result.iter_native()`` where available) + instead. + +- **Worker**\|eventlet/gevent: A regression caused ``Ctrl+C`` to be ineffective + for shutdown. + +- **Redis result backend:** Now using a pipeline to store state changes + for improved performance. + + Contributed by Pepijn de Vos. + +- **Redis result backend:** Will now retry storing the result if disconnected. + +- **Worker**\|gossip: Fixed attribute error occurring when another node leaves. + + Fix contributed by Brodie Rao. + +- **Generic init scripts:** Now runs a check at startup to verify + that any configuration scripts are owned by root and that they + are not world/group writeable. + + The init script configuration is a shell script executed by root, + so this is a preventive measure to ensure that users do not + leave this file vulnerable to changes by unprivileged users. + + .. note:: + + Note that upgrading celery will not update the init scripts, + instead you need to manually copy the improved versions from the + source distribution: + https://github.com/celery/celery/tree/3.1/extra/generic-init.d + +- **Commands**: The :program:`celery purge` command now warns that the operation + will delete all tasks and prompts the user for confirmation. + + A new :option:`-f` was added that can be used to disable + interactive mode. + +- **Task**: ``.retry()`` did not raise the value provided in the ``exc`` argument + when called outside of an error context (*Issue #1755*). + +- **Commands:** The :program:`celery multi` command did not forward command + line configuration to the target workers. + + The change means that multi will forward the special ``--`` argument and + configuration content at the end of the arguments line to the specified + workers. + + Example using command-line configuration to set a broker heartbeat + from :program:`celery multi`: + + .. code-block:: bash + + $ celery multi start 1 -c3 -- broker.heartbeat=30 + + Fix contributed by Antoine Legrand. + +- **Canvas:** ``chain.apply_async()`` now properly forwards execution options. + + Fix contributed by Konstantin Podshumok. + +- **Redis result backend:** Now takes ``connection_pool`` argument that can be + used to change the connection pool class/constructor. + +- **Worker:** Now truncates very long arguments and keyword arguments logged by + the pool at debug severity. + +- **Worker:** The worker now closes all open files on :sig:`SIGHUP` (regression) + (*Issue #1768*). + + Fix contributed by Brodie Rao + +- **Worker:** Will no longer accept remote control commands while the + worker startup phase is incomplete (*Issue #1741*). + +- **Commands:** The output of the event dump utility + (:program:`celery events -d`) can now be piped into other commands. + +- **Documentation:** The RabbitMQ installation instructions for OS X was + updated to use modern homebrew practices. + + Contributed by Jon Chen. + +- **Commands:** The :program:`celery inspect conf` utility now works. + +- **Commands:** The :option:`-no-color` argument was not respected by + all commands (*Issue #1799*). + +- **App:** Fixed rare bug with ``autodiscover_tasks()`` (*Issue #1797*). + +- **Distribution:** The sphinx docs will now always add the parent directory + to path so that the current celery source code is used as a basis for + API documentation (*Issue #1782*). + +- **Documentation:** Supervisord examples contained an extraneous '-' in a + `--logfile` argument example. + + Fix contributed by Mohammad Almeer. + +.. _version-3.1.7: + +3.1.7 +===== +:release-date: 2013-12-17 06:00 P.M UTC +:release-by: Ask Solem + +.. _v317-important: + +Important Notes +--------------- + +Init script security improvements +--------------------------------- + +Where the generic init scripts (for ``celeryd``, and ``celerybeat``) before +delegated the responsibility of dropping privileges to the target application, +it will now use ``su`` instead, so that the Python program is not trusted +with superuser privileges. + +This is not in reaction to any known exploit, but it will +limit the possibility of a privilege escalation bug being abused in the +future. + +You have to upgrade the init scripts manually from this directory: +https://github.com/celery/celery/tree/3.1/extra/generic-init.d + +AMQP result backend +~~~~~~~~~~~~~~~~~~~ + +The 3.1 release accidentally left the amqp backend configured to be +non-persistent by default. + +Upgrading from 3.0 would give a "not equivalent" error when attempting to +set or retrieve results for a task. That is unless you manually set the +persistence setting:: + + CELERY_RESULT_PERSISTENT = True + +This version restores the previous value so if you already forced +the upgrade by removing the existing exchange you must either +keep the configuration by setting ``CELERY_RESULT_PERSISTENT = False`` +or delete the ``celeryresults`` exchange again. + +Synchronous subtasks +~~~~~~~~~~~~~~~~~~~~ + +Tasks waiting for the result of a subtask will now emit +a :exc:`RuntimeWarning` warning when using the prefork pool, +and in 3.2 this will result in an exception being raised. + +It's not legal for tasks to block by waiting for subtasks +as this is likely to lead to resource starvation and eventually +deadlock when using the prefork pool (see also :ref:`task-synchronous-subtasks`). + +If you really know what you are doing you can avoid the warning (and +the future exception being raised) by moving the operation in a whitelist +block: + +.. code-block:: python + + from celery.result import allow_join_result + + @app.task + def misbehaving(): + result = other_task.delay() + with allow_join_result(): + result.get() + +Note also that if you wait for the result of a subtask in any form +when using the prefork pool you must also disable the pool prefetching +behavior with the worker :ref:`-Ofair option `. + +.. _v317-fixes: + +Fixes +----- + +- Now depends on :ref:`Kombu 3.0.8 `. + +- Now depends on :mod:`billiard` 3.3.0.13 + +- Events: Fixed compatibility with non-standard json libraries + that sends float as :class:`decimal.Decimal` (Issue #1731) + +- Events: State worker objects now always defines attributes: + ``active``, ``processed``, ``loadavg``, ``sw_ident``, ``sw_ver`` + and ``sw_sys``. + +- Worker: Now keeps count of the total number of tasks processed, + not just by type (``all_active_count``). + +- Init scripts: Fixed problem with reading configuration file + when the init script is symlinked to a runlevel (e.g. ``S02celeryd``). + (Issue #1740). + + This also removed a rarely used feature where you can symlink the script + to provide alternative configurations. You instead copy the script + and give it a new name, but perhaps a better solution is to provide + arguments to ``CELERYD_OPTS`` to separate them: + + .. code-block:: bash + + CELERYD_NODES="X1 X2 Y1 Y2" + CELERYD_OPTS="-A:X1 x -A:X2 x -A:Y1 y -A:Y2 y" + +- Fallback chord unlock task is now always called after the chord header + (Issue #1700). + + This means that the unlock task will not be started if there's + an error sending the header. + +- Celery command: Fixed problem with arguments for some control commands. + + Fix contributed by Konstantin Podshumok. + +- Fixed bug in ``utcoffset`` where the offset when in DST would be + completely wrong (Issue #1743). + +- Worker: Errors occurring while attempting to serialize the result of a + task will now cause the task to be marked with failure and a + :class:`kombu.exceptions.EncodingError` error. + + Fix contributed by Ionel Cristian Mărieș. + +- Worker with ``-B`` argument did not properly shut down the beat instance. + +- Worker: The ``%n`` and ``%h`` formats are now also supported by the + :option:`--logfile`, :option:`--pidfile` and :option:`--statedb` arguments. + + Example: + + .. code-block:: bash + + $ celery -A proj worker -n foo@%h --logfile=%n.log --statedb=%n.db + +- Redis/Cache result backends: Will now timeout if keys evicted while trying + to join a chord. + +- The fallbock unlock chord task now raises :exc:`Retry` so that the + retry even is properly logged by the worker. + +- Multi: Will no longer apply Eventlet/gevent monkey patches (Issue #1717). + +- Redis result backend: Now supports UNIX sockets. + + Like the Redis broker transport the result backend now also supports + using ``redis+socket:///tmp/redis.sock`` URLs. + + Contributed by Alcides Viamontes Esquivel. + +- Events: Events sent by clients was mistaken for worker related events + (Issue #1714). + + For ``events.State`` the tasks now have a ``Task.client`` attribute + that is set when a ``task-sent`` event is being received. + + Also, a clients logical clock is not in sync with the cluster so + they live in a "time bubble". So for this reason monitors will no + longer attempt to merge with the clock of an event sent by a client, + instead it will fake the value by using the current clock with + a skew of -1. + +- Prefork pool: The method used to find terminated processes was flawed + in that it did not also take into account missing popen objects. + +- Canvas: ``group`` and ``chord`` now works with anon signatures as long + as the group/chord object is associated with an app instance (Issue #1744). + + You can pass the app by using ``group(..., app=app)``. + +.. _version-3.1.6: + +3.1.6 +===== +:release-date: 2013-12-02 06:00 P.M UTC +:release-by: Ask Solem + +- Now depends on :mod:`billiard` 3.3.0.10. + +- Now depends on :ref:`Kombu 3.0.7 `. + +- Fixed problem where Mingle caused the worker to hang at startup + (Issue #1686). + +- Beat: Would attempt to drop privileges twice (Issue #1708). + +- Windows: Fixed error with ``geteuid`` not being available (Issue #1676). + +- Tasks can now provide a list of expected error classes (Issue #1682). + + The list should only include errors that the task is expected to raise + during normal operation:: + + @task(throws=(KeyError, HttpNotFound)) + + What happens when an exceptions is raised depends on the type of error: + + - Expected errors (included in ``Task.throws``) + + Will be logged using severity ``INFO``, and traceback is excluded. + + - Unexpected errors + + Will be logged using severity ``ERROR``, with traceback included. + +- Cache result backend now compatible with Python 3 (Issue #1697). + +- CentOS init script: Now compatible with sys-v style init symlinks. + + Fix contributed by Jonathan Jordan. + +- Events: Fixed problem when task name is not defined (Issue #1710). + + Fix contributed by Mher Movsisyan. + +- Task: Fixed unbound local errors (Issue #1684). + + Fix contributed by Markus Ullmann. + +- Canvas: Now unrolls groups with only one task (optimization) (Issue #1656). + +- Task: Fixed problem with eta and timezones. + + Fix contributed by Alexander Koval. + +- Django: Worker now performs model validation (Issue #1681). + +- Task decorator now emits less confusing errors when used with + incorrect arguments (Issue #1692). + +- Task: New method ``Task.send_event`` can be used to send custom events + to Flower and other monitors. + +- Fixed a compatibility issue with non-abstract task classes + +- Events from clients now uses new node name format (``gen@``). + +- Fixed rare bug with Callable not being defined at interpreter shutdown + (Issue #1678). + + Fix contributed by Nick Johnson. + +- Fixed Python 2.6 compatibility (Issue #1679). + +.. _version-3.1.5: + +3.1.5 +===== +:release-date: 2013-11-21 06:20 P.M UTC +:release-by: Ask Solem + +- Now depends on :ref:`Kombu 3.0.6 `. + +- Now depends on :mod:`billiard` 3.3.0.8 + +- App: ``config_from_object`` is now lazy (Issue #1665). + +- App: ``autodiscover_tasks`` is now lazy. + + Django users should now wrap access to the settings object + in a lambda:: + + app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) + + this ensures that the settings object is not prepared + prematurely. + +- Fixed regression for ``--app`` argument experienced by + some users (Issue #1653). + +- Worker: Now respects the ``--uid`` and ``--gid`` arguments + even if ``--detach`` is not enabled. + +- Beat: Now respects the ``--uid`` and ``--gid`` arguments + even if ``--detach`` is not enabled. + +- Python 3: Fixed unorderable error occuring with the worker ``-B`` + argument enabled. + +- ``celery.VERSION`` is now a named tuple. + +- ``maybe_signature(list)`` is now applied recursively (Issue #1645). + +- ``celery shell`` command: Fixed ``IPython.frontend`` deprecation warning. + +- The default app no longer includes the builtin fixups. + + This fixes a bug where ``celery multi`` would attempt + to load the Django settings module before entering + the target working directory. + +- The Django daemonization tutorial was changed. + + Users no longer have to explicitly export ``DJANGO_SETTINGS_MODULE`` + in :file:`/etc/default/celeryd` when the new project layout is used. + +- Redis result backend: expiry value can now be 0 (Issue #1661). + +- Censoring settings now accounts for non-string keys (Issue #1663). + +- App: New ``autofinalize`` option. + + Apps are automatically finalized when the task registry is accessed. + You can now disable this behavior so that an exception is raised + instead. + + Example: + + .. code-block:: python + + app = Celery(autofinalize=False) + + # raises RuntimeError + tasks = app.tasks + + @app.task + def add(x, y): + return x + y + + # raises RuntimeError + add.delay(2, 2) + + app.finalize() + # no longer raises: + tasks = app.tasks + add.delay(2, 2) + +- The worker did not send monitoring events during shutdown. + +- Worker: Mingle and gossip is now automatically disabled when + used with an unsupported transport (Issue #1664). + +- ``celery`` command: Preload options now supports + the rare ``--opt value`` format (Issue #1668). + +- ``celery`` command: Accidentally removed options + appearing before the subcommand, these are now moved to the end + instead. + +- Worker now properly responds to ``inspect stats`` commands + even if received before startup is complete (Issue #1659). + +- :signal:`task_postrun` is now sent within a finally block, to make + sure the signal is always sent. + +- Beat: Fixed syntax error in string formatting. + + Contributed by nadad. + +- Fixed typos in the documentation. + + Fixes contributed by Loic Bistuer, sunfinite. + +- Nested chains now works properly when constructed using the + ``chain`` type instead of the ``|`` operator (Issue #1656). + +.. _version-3.1.4: + +3.1.4 +===== +:release-date: 2013-11-15 11:40 P.M UTC +:release-by: Ask Solem + +- Now depends on :ref:`Kombu 3.0.5 `. + +- Now depends on :mod:`billiard` 3.3.0.7 + +- Worker accidentally set a default socket timeout of 5 seconds. + +- Django: Fixup now sets the default app so that threads will use + the same app instance (e.g. for manage.py runserver). + +- Worker: Fixed Unicode error crash at startup experienced by some users. + +- Calling ``.apply_async`` on an empty chain now works again (Issue #1650). + +- The ``celery multi show`` command now generates the same arguments + as the start command does. + +- The ``--app`` argument could end up using a module object instead + of an app instance (with a resulting crash). + +- Fixed a syntax error problem in the celerybeat init script. + + Fix contributed by Vsevolod. + +- Tests now passing on PyPy 2.1 and 2.2. + +.. _version-3.1.3: + +3.1.3 +===== +:release-date: 2013-11-13 00:55 A.M UTC +:release-by: Ask Solem + +- Fixed compatibility problem with Python 2.7.0 - 2.7.5 (Issue #1637) + + ``unpack_from`` started supporting ``memoryview`` arguments + in Python 2.7.6. + +- Worker: :option:`-B` argument accidentally closed files used + for logging. + +- Task decorated tasks now keep their docstring (Issue #1636) + +.. _version-3.1.2: + +3.1.2 +===== +:release-date: 2013-11-12 08:00 P.M UTC +:release-by: Ask Solem + +- Now depends on :mod:`billiard` 3.3.0.6 + +- No longer needs the billiard C extension to be installed. + +- The worker silently ignored task errors. + +- Django: Fixed ``ImproperlyConfigured`` error raised + when no database backend specified. + + Fix contributed by j0hnsmith + +- Prefork pool: Now using ``_multiprocessing.read`` with ``memoryview`` + if available. + +- ``close_open_fds`` now uses ``os.closerange`` if available. + +- ``get_fdmax`` now takes value from ``sysconfig`` if possible. + +.. _version-3.1.1: + +3.1.1 +===== +:release-date: 2013-11-11 06:30 P.M UTC +:release-by: Ask Solem + +- Now depends on :mod:`billiard` 3.3.0.4. + +- Python 3: Fixed compatibility issues. + +- Windows: Accidentally showed warning that the billiard C extension + was not installed (Issue #1630). + +- Django: Tutorial updated with a solution that sets a default + :envvar:`DJANGO_SETTINGS_MODULE` so that it doesn't have to be typed + in with the :program:`celery` command. + + Also fixed typos in the tutorial, and added the settings + required to use the Django database backend. + + Thanks to Chris Ward, orarbel. + +- Django: Fixed a problem when using the Django settings in Django 1.6. + +- Django: Fixup should not be applied if the django loader is active. + +- Worker: Fixed attribute error for ``human_write_stats`` when using the + compatibility prefork pool implementation. + +- Worker: Fixed compatibility with billiard without C extension. + +- Inspect.conf: Now supports a ``with_defaults`` argument. + +- Group.restore: The backend argument was not respected. + +.. _version-3.1.0: + +3.1.0 +======= +:release-date: 2013-11-09 11:00 P.M UTC +:release-by: Ask Solem + +See :ref:`whatsnew-3.1`. diff --git a/docs/history/index.rst b/docs/history/index.rst index 673532de1..cf6d0f96c 100644 --- a/docs/history/index.rst +++ b/docs/history/index.rst @@ -13,6 +13,7 @@ version please visit :ref:`changelog`. .. toctree:: :maxdepth: 2 + changelog-3.1 changelog-3.0 changelog-2.5 changelog-2.4 From 9e0eae688b35a09f292e9b3ec7ba0012c0010055 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 18 Nov 2014 20:20:03 +0000 Subject: [PATCH 0401/1103] MongoDB: Pickle url with backend. Closes #2347 --- celery/backends/mongodb.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 0e455497f..e3dd58917 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -59,7 +59,7 @@ class MongoBackend(BaseBackend): _connection = None - def __init__(self, **kwargs): + def __init__(self, app=None, url=None, **kwargs): """Initialize MongoDB backend instance. :raises celery.exceptions.ImproperlyConfigured: if @@ -68,7 +68,7 @@ def __init__(self, **kwargs): """ self.options = {} - super(MongoBackend, self).__init__(**kwargs) + super(MongoBackend, self).__init__(app, **kwargs) if not pymongo: raise ImproperlyConfigured( @@ -100,10 +100,10 @@ def __init__(self, **kwargs): self.options.setdefault('max_pool_size', self.max_pool_size) self.options.setdefault('auto_start_request', False) - url = kwargs.get('url') - if url: + self.url = url + if self.url: # Specifying backend as an URL - self.host = url + self.host = self.url def _get_connection(self): """Connect to the MongoDB server.""" @@ -232,9 +232,9 @@ def cleanup(self): ) def __reduce__(self, args=(), kwargs={}): - kwargs.update( - dict(expires=self.expires)) - return super(MongoBackend, self).__reduce__(args, kwargs) + return super(MongoBackend, self).__reduce__( + args, dict(kwargs, expires=self.expires, url=self.url), + ) def _get_database(self): conn = self._get_connection() From b2d948db5125bfcfe00eeb5362ba3c35f35e42f0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 18 Nov 2014 20:20:43 +0000 Subject: [PATCH 0402/1103] MongoDB: Fixes expires --- celery/backends/mongodb.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index e3dd58917..f82c5f559 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -8,7 +8,7 @@ """ from __future__ import absolute_import -from datetime import datetime +from datetime import datetime, timedelta try: import pymongo @@ -225,10 +225,10 @@ def _forget(self, task_id): def cleanup(self): """Delete expired metadata.""" self.collection.remove( - {'date_done': {'$lt': self.app.now() - self.expires}}, + {'date_done': {'$lt': self.app.now() - self.expires_delta}}, ) self.group_collection.remove( - {'date_done': {'$lt': self.app.now() - self.expires}}, + {'date_done': {'$lt': self.app.now() - self.expires_delta}}, ) def __reduce__(self, args=(), kwargs={}): @@ -271,3 +271,7 @@ def group_collection(self): # in the background. Once completed cleanup will be much faster collection.ensure_index('date_done', background='true') return collection + + @cached_property + def expires_delta(self): + return timedelta(seconds=self.expires) From f8b2efaa7e21f353f72529bf74f7d1ede91b2d80 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 18 Nov 2014 20:25:01 +0000 Subject: [PATCH 0403/1103] Optimization for #2358 --- celery/worker/strategy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index 49012476a..68115c06d 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -61,6 +61,7 @@ def default(task, app, consumer, call_at = consumer.timer.call_at apply_eta_task = consumer.apply_eta_task rate_limits_enabled = not consumer.disable_rate_limits + get_bucket = consumer.task_buckets.__getitem__ handle = consumer.on_task_request limit_task = consumer._limit_task body_can_be_buffer = consumer.pool.body_can_be_buffer @@ -115,7 +116,7 @@ def task_message_handler(message, body, ack, reject, callbacks, call_at(eta, apply_eta_task, (req, ), priority=6) else: if rate_limits_enabled: - bucket = consumer.task_buckets[task.name] + bucket = get_bucket(task.name) if bucket: return limit_task(req, bucket, 1) task_reserved(req) From a76798d3fe5719965dc639903d58ea31fabc7c78 Mon Sep 17 00:00:00 2001 From: Michael Permana Date: Tue, 18 Nov 2014 14:58:10 -0800 Subject: [PATCH 0404/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 53c62af09..03c3b6ac1 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -175,3 +175,4 @@ Joe Jevnik, 2014/10/22 Nathan Van Gheem, 2014/10/28 Gino Ledesma, 2014/10/28 Thomas French, 2014/11/10 +Michael Permana, 2014/11/6 From 0f9b201cb4b8cad585d54ac753333758906ac6c0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 18 Nov 2014 23:43:39 +0000 Subject: [PATCH 0405/1103] Removes unused method state.compare --- celery/states.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/celery/states.py b/celery/states.py index ad8feebca..054b448db 100644 --- a/celery/states.py +++ b/celery/states.py @@ -112,9 +112,6 @@ class state(str): """ - def compare(self, other, fun): - return fun(precedence(self), precedence(other)) - def __gt__(self, other): return precedence(self) < precedence(other) From 59725694892832c2576afd442a1be1b08a31f1ce Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 18 Nov 2014 23:44:11 +0000 Subject: [PATCH 0406/1103] Force ScheduleEntry to be orderable. Closes #2295 --- celery/beat.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/celery/beat.py b/celery/beat.py index c88f521c3..b17a2c295 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -17,6 +17,7 @@ import traceback from collections import namedtuple +from functools import total_ordering from threading import Event, Thread from billiard import ensure_multiprocessing @@ -50,6 +51,7 @@ class SchedulingError(Exception): """An error occured while scheduling a task.""" +@total_ordering class ScheduleEntry(object): """An entry in the scheduler. @@ -141,6 +143,11 @@ def __repr__(self): call=reprcall(self.task, self.args or (), self.kwargs or {}), ) + def __lt__(self, other): + if isinstance(other, ScheduleEntry): + return id(self) < id(other) + return NotImplemented + class Scheduler(object): """Scheduler for periodic tasks. From ca91b36767850996ea32130e192edd5436aa7c0d Mon Sep 17 00:00:00 2001 From: bee-keeper Date: Wed, 19 Nov 2014 14:19:03 +0000 Subject: [PATCH 0407/1103] (extras) added additional examples of running celery via supervisor --- extra/supervisord/celery.sh | 3 +++ extra/supervisord/celeryd.conf | 5 +++++ 2 files changed, 8 insertions(+) create mode 100644 extra/supervisord/celery.sh diff --git a/extra/supervisord/celery.sh b/extra/supervisord/celery.sh new file mode 100644 index 000000000..d49b3d123 --- /dev/null +++ b/extra/supervisord/celery.sh @@ -0,0 +1,3 @@ +#!/bin/bash +source {{ additional variables }} +exec celery --app={{ application_name }}._celery:app worker --loglevel=INFO -n worker.%%h \ No newline at end of file diff --git a/extra/supervisord/celeryd.conf b/extra/supervisord/celeryd.conf index f92293727..829c2f6d2 100644 --- a/extra/supervisord/celeryd.conf +++ b/extra/supervisord/celeryd.conf @@ -6,6 +6,11 @@ ; Set full path to celery program if using virtualenv command=celery worker -A proj --loglevel=INFO +; Alternatively, +;command=celery --app=your_app._celery:app worker --loglevel=INFO -n worker.%%h +; Or run a script +;command=celery.sh + directory=/path/to/project user=nobody numprocs=1 From 1436bc915af514f500b9a71af8a24bbc57c71f54 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 19 Nov 2014 14:19:08 +0000 Subject: [PATCH 0408/1103] some more for whatsnew3.2 --- docs/whatsnew-3.2.rst | 105 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 101 insertions(+), 4 deletions(-) diff --git a/docs/whatsnew-3.2.rst b/docs/whatsnew-3.2.rst index ffe60f796..95484420f 100644 --- a/docs/whatsnew-3.2.rst +++ b/docs/whatsnew-3.2.rst @@ -58,25 +58,117 @@ Celery now requires Python 2.7 or later. JSON is now the default serializer ---------------------------------- +Using one logfile per process by default +---------------------------------------- + +The Task base class no longer automatically register tasks +---------------------------------------------------------- + +The metaclass has been removed blah blah + + +Arguments now verified when calling a task +------------------------------------------ + .. _v320-news: News ==== -Item 1 ------- +New Task Message Protocol +========================= + + +``TaskProducer`` replaced by ``app.amqp.create_task_message`` and +``app.amqp.send_task_message``. + +- Worker stores results for internal errors like ``ContentDisallowed``, and + exceptions occurring outside of the task function. + + +Canvas Refactor +=============== + +Riak Result Backend +=================== + +Contributed by Gilles Dartiguelongue, Alman One and NoKriK. Bla bla - blah blah + +Event Batching +============== + +Events are now buffered in the worker and sent as a list + + +Task.replace +============ + Task.replace changed, removes Task.replace_in_chord. + + The two methods had almost the same functionality, but the old Task.replace + would force the new task to inherit the callbacks/errbacks of the existing + task. + + If you replace a node in a tree, then you would not expect the new node to + inherit the children of the old node, so this seems like unexpected + behavior. + + So self.replace(sig) now works for any task, in addition sig can now + be a group. + + Groups are automatically converted to a chord, where the callback + will "accumulate" the results of the group tasks. + + A new builtin task (`celery.accumulate` was added for this purpose) + + Closes #81 + + +Optimized Beat implementation +============================= + In Other News ------------- -- Now depends on :ref:`Kombu 3.1 `. +- **Requirements**: + + - Now depends on :ref:`Kombu 3.1 `. + + - Now depends on :mod:`billiard` version 3.4. + + - No longer depends on ``anyjson`` :sadface: + +- **Programs**: ``%n`` format for :program:`celery multi` is now synonym with + ``%N`` to be consistent with :program:`celery worker`. + +- **Programs**: celery inspect/control now supports --json argument -- Now depends on :mod:`billiard` version 3.4. +- **Programs**: :program:`celery logtool`: Utility for filtering and parsing celery worker logfiles + +- **Worker**: Gossip now sets ``x-message-ttl`` for event queue to heartbeat_interval s. + (Iss ue #2005). + +- **App**: New signals + + - :data:`app.on_configure <@on_configure>` + - :data:`app.on_after_configure <@on_after_configure>` + - :data:`app.on_after_finalize <@on_after_finalize>` + +- **Canvas**: ``chunks``/``map``/``starmap`` are now routed based on the target task. + +- Apps can now define how tasks are named (:meth:`@gen_task_name`). + + Contributed by Dmitry Malinovsky + +- Module ``celery.worker.job`` renamed to :mod:`celery.worker.request`. + +- Beat: ``Scheduler.Publisher``/``.publisher`` renamed to + ``.Producer``/``.producer``. .. _v320-removals: @@ -84,6 +176,11 @@ In Other News Scheduled Removals ================== +- The module ``celery.task.trace`` has been removed as scheduled for this + version. + +- Magic keyword arguments no longer supported. + .. _v320-deprecations: Deprecations From c67b0eaf11c4e0e8dbff8073d283d0650d029db7 Mon Sep 17 00:00:00 2001 From: bee-keeper Date: Wed, 19 Nov 2014 14:20:39 +0000 Subject: [PATCH 0409/1103] removed underscore --- extra/supervisord/celery.sh | 2 +- extra/supervisord/celeryd.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extra/supervisord/celery.sh b/extra/supervisord/celery.sh index d49b3d123..a5bcee09f 100644 --- a/extra/supervisord/celery.sh +++ b/extra/supervisord/celery.sh @@ -1,3 +1,3 @@ #!/bin/bash source {{ additional variables }} -exec celery --app={{ application_name }}._celery:app worker --loglevel=INFO -n worker.%%h \ No newline at end of file +exec celery --app={{ application_name }}.celery:app worker --loglevel=INFO -n worker.%%h \ No newline at end of file diff --git a/extra/supervisord/celeryd.conf b/extra/supervisord/celeryd.conf index 829c2f6d2..eaf59869d 100644 --- a/extra/supervisord/celeryd.conf +++ b/extra/supervisord/celeryd.conf @@ -7,7 +7,7 @@ command=celery worker -A proj --loglevel=INFO ; Alternatively, -;command=celery --app=your_app._celery:app worker --loglevel=INFO -n worker.%%h +;command=celery --app=your_app.celery:app worker --loglevel=INFO -n worker.%%h ; Or run a script ;command=celery.sh From 803639df7ab681c3922309e44b7d882996fddc3b Mon Sep 17 00:00:00 2001 From: William King Date: Fri, 21 Nov 2014 12:38:38 -0800 Subject: [PATCH 0410/1103] Add switch to reverse order of tasks_by_time, with test --- celery/events/state.py | 16 ++++++++++------ celery/tests/events/test_state.py | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/celery/events/state.py b/celery/events/state.py index 2a11891f7..96696aaa9 100644 --- a/celery/events/state.py +++ b/celery/events/state.py @@ -601,11 +601,15 @@ def itertasks(self, limit=None): if limit and index + 1 >= limit: break - def tasks_by_time(self, limit=None): + def tasks_by_time(self, limit=None, reverse=True): """Generator giving tasks ordered by time, in ``(uuid, Task)`` tuples.""" + _heap = self._taskheap + if reverse: + _heap = reversed(_heap) + seen = set() - for evtup in islice(reversed(self._taskheap), 0, limit): + for evtup in islice(_heap, 0, limit): task = evtup[3]() if task is not None: uuid = task.uuid @@ -614,24 +618,24 @@ def tasks_by_time(self, limit=None): seen.add(uuid) tasks_by_timestamp = tasks_by_time - def tasks_by_type(self, name, limit=None): + def tasks_by_type(self, name, limit=None, reverse=True): """Get all tasks by type. Return a list of ``(uuid, Task)`` tuples. """ return islice( - ((uuid, task) for uuid, task in self.tasks_by_time() + ((uuid, task) for uuid, task in self.tasks_by_time(reverse=reverse) if task.name == name), 0, limit, ) - def tasks_by_worker(self, hostname, limit=None): + def tasks_by_worker(self, hostname, limit=None, reverse=True): """Get all tasks by worker. """ return islice( - ((uuid, task) for uuid, task in self.tasks_by_time() + ((uuid, task) for uuid, task in self.tasks_by_time(reverse=reverse) if task.worker.hostname == hostname), 0, limit, ) diff --git a/celery/tests/events/test_state.py b/celery/tests/events/test_state.py index b7e35d7cf..f7b9946c2 100644 --- a/celery/tests/events/test_state.py +++ b/celery/tests/events/test_state.py @@ -318,6 +318,25 @@ def test_task_logical_clock_ordering(self): self.assertEqual(now[1][0], tC) self.assertEqual(now[2][0], tB) + def test_task_descending_clock_ordering(self): + state = State() + r = ev_logical_clock_ordering(state) + tA, tB, tC = r.uids + r.play() + now = list(state.tasks_by_time(reverse=False)) + self.assertEqual(now[0][0], tB) + self.assertEqual(now[1][0], tC) + self.assertEqual(now[2][0], tA) + for _ in range(1000): + shuffle(r.uids) + tA, tB, tC = r.uids + r.rewind_with_offset(r.current_clock + 1, r.uids) + r.play() + now = list(state.tasks_by_time(reverse=False)) + self.assertEqual(now[0][0], tB) + self.assertEqual(now[1][0], tC) + self.assertEqual(now[2][0], tA) + def test_worker_online_offline(self): r = ev_worker_online_offline(State()) next(r) From 9ff2bdde33087f2a0d45d581d743f3a369fbfad9 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Sat, 22 Nov 2014 00:43:14 +0000 Subject: [PATCH 0411/1103] Wording --- docs/internals/protocol.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/internals/protocol.rst b/docs/internals/protocol.rst index 6b3167c78..285ed9b06 100644 --- a/docs/internals/protocol.rst +++ b/docs/internals/protocol.rst @@ -23,9 +23,6 @@ Definition .. code-block:: python - # protocol v2 implies UTC=True - # 'class' header existing means protocol is v2 - properties = { 'correlation_id': uuid task_id, 'content_type': string mimetype, @@ -102,6 +99,11 @@ Changes from version 1 the payload (which may be language specific, e.g. serialized by the Python specific pickle serializer). +- Always UTC + + There's no ``utc`` flag anymore, so any time information missing timezone + will be expected to be in UTC time. + - Body is only for language specific data. - Python stores args/kwargs and embedded signatures in body. From 5ba2e75e73926df102f81b0269419e8175765b6b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 24 Nov 2014 20:53:34 +0000 Subject: [PATCH 0412/1103] Doc issues --- celery/events/state.py | 2 +- celery/tests/events/test_state.py | 6 +++--- docs/conf.py | 4 ++-- docs/configuration.rst | 1 + docs/history/changelog-3.1.rst | 4 ++-- docs/whatsnew-3.1.rst | 2 +- docs/whatsnew-3.2.rst | 2 +- 7 files changed, 11 insertions(+), 10 deletions(-) diff --git a/celery/events/state.py b/celery/events/state.py index 96696aaa9..74284a6d1 100644 --- a/celery/events/state.py +++ b/celery/events/state.py @@ -163,7 +163,7 @@ def event(type_, timestamp=None, if not local_received or not timestamp: return drift = abs(int(local_received) - int(timestamp)) - if drift > HEARTBEAT_DRIFT_MAX: + if drift > max_drift: _warn_drift(self.hostname, drift, local_received, timestamp) if local_received: diff --git a/celery/tests/events/test_state.py b/celery/tests/events/test_state.py index f7b9946c2..99911a897 100644 --- a/celery/tests/events/test_state.py +++ b/celery/tests/events/test_state.py @@ -324,9 +324,9 @@ def test_task_descending_clock_ordering(self): tA, tB, tC = r.uids r.play() now = list(state.tasks_by_time(reverse=False)) - self.assertEqual(now[0][0], tB) - self.assertEqual(now[1][0], tC) - self.assertEqual(now[2][0], tA) + self.assertEqual(now[0][0], tA) + self.assertEqual(now[1][0], tB) + self.assertEqual(now[2][0], tC) for _ in range(1000): shuffle(r.uids) tA, tB, tC = r.uids diff --git a/docs/conf.py b/docs/conf.py index e46cc67fc..efd7ea795 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -72,8 +72,8 @@ def linkcode_resolve(domain, info): intersphinx_mapping = { 'python': ('http://docs.python.org/dev', None), - 'kombu': ('http://kombu.readthedocs.org/en/latest/', None), - 'djcelery': ('http://django-celery.readthedocs.org/en/latest', None), + 'kombu': ('http://kombu.readthedocs.org/en/master/', None), + 'djcelery': ('http://django-celery.readthedocs.org/en/master', None), 'cyme': ('http://cyme.readthedocs.org/en/latest', None), 'amqp': ('http://amqp.readthedocs.org/en/latest', None), 'flower': ('http://flower.readthedocs.org/en/latest', None), diff --git a/docs/configuration.rst b/docs/configuration.rst index aab186901..9100f2e61 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -236,6 +236,7 @@ Can be one of the following: .. _`Redis`: http://redis.io .. _`Cassandra`: http://cassandra.apache.org/ .. _`IronCache`: http://www.iron.io/cache +.. _`CouchDB`: http://www.couchdb.com/ .. _`Couchbase`: http://www.couchbase.com/ diff --git a/docs/history/changelog-3.1.rst b/docs/history/changelog-3.1.rst index f128e79a1..f03e98869 100644 --- a/docs/history/changelog-3.1.rst +++ b/docs/history/changelog-3.1.rst @@ -135,8 +135,8 @@ If you're looking for versions prior to 3.1.x you should go to :ref:`history`. before importing any task modules (Django 1.7 compatibility, Issue #2227) - **Results**: ``result.get()`` was misbehaving by calling - ``backend.get_task_meta`` in a finally call leading to - AMQP result backend queues not being properly cleaned up (Issue #2245). + ``backend.get_task_meta`` in a finally call leading to + AMQP result backend queues not being properly cleaned up (Issue #2245). .. _version-3.1.14: diff --git a/docs/whatsnew-3.1.rst b/docs/whatsnew-3.1.rst index 62b16aa02..32bd47d39 100644 --- a/docs/whatsnew-3.1.rst +++ b/docs/whatsnew-3.1.rst @@ -76,7 +76,7 @@ so I cannot recommend them for production use. The next version of Celery 3.2 will focus on performance and removing rarely used parts of the library. Work has also started on a new message protocol, supporting multiple languages and more. The initial draft can -be found :ref:`here `. This has probably been the hardest release I've worked on, so no introduction to this changelog would be complete without a massive diff --git a/docs/whatsnew-3.2.rst b/docs/whatsnew-3.2.rst index 95484420f..d75b6e9a8 100644 --- a/docs/whatsnew-3.2.rst +++ b/docs/whatsnew-3.2.rst @@ -1,4 +1,4 @@ -.. _whatsnew-3.1: +.. _whatsnew-3.2: =========================================== What's new in Celery 3.2 (TBA) From 7265a6dc709e2877c1952de170023edd9c171db7 Mon Sep 17 00:00:00 2001 From: Markus Ullmann Date: Wed, 26 Nov 2014 14:37:03 +0100 Subject: [PATCH 0413/1103] Update redis.rst --- docs/getting-started/brokers/redis.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/getting-started/brokers/redis.rst b/docs/getting-started/brokers/redis.rst index 543f4ee90..485d15abb 100644 --- a/docs/getting-started/brokers/redis.rst +++ b/docs/getting-started/brokers/redis.rst @@ -34,6 +34,10 @@ Where the URL is in the format of:: all fields after the scheme are optional, and will default to localhost on port 6379, using database 0. +If a unix socket connection should be used, the URL needs to be in the format:: + + redis+socket:///path/to/redis.sock + .. _redis-visibility_timeout: Visibility Timeout From aac0bd4e3ba95a634b5364732a1fde8ca1a120a2 Mon Sep 17 00:00:00 2001 From: Emil Stanchev Date: Wed, 26 Nov 2014 16:13:21 +0200 Subject: [PATCH 0414/1103] Fix variable name in an example in the documentation --- docs/userguide/application.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/application.rst b/docs/userguide/application.rst index a4653dfe5..c29d4e16b 100644 --- a/docs/userguide/application.rst +++ b/docs/userguide/application.rst @@ -521,7 +521,7 @@ by changing its :meth:`@Task` attribute: >>> app.Task - >>> @x.task + >>> @app.task ... def add(x, y): ... return x + y From 267028aaf743ea8030fa4c12699929f4962e1b7b Mon Sep 17 00:00:00 2001 From: Andriy Yurchuk Date: Fri, 28 Nov 2014 10:36:13 +0200 Subject: [PATCH 0415/1103] Fix variable name in Task Cookbook tutorial --- docs/tutorials/task-cookbook.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/task-cookbook.rst b/docs/tutorials/task-cookbook.rst index ca3fa5065..e44722686 100644 --- a/docs/tutorials/task-cookbook.rst +++ b/docs/tutorials/task-cookbook.rst @@ -42,7 +42,7 @@ The cache key expires after some time in case something unexpected happens def import_feed(self, feed_url): # The cache key consists of the task name and the MD5 digest # of the feed URL. - feed_url_digest = md5(feed_url).hexdigest() + feed_url_hexdigest = md5(feed_url).hexdigest() lock_id = '{0}-lock-{1}'.format(self.name, feed_url_hexdigest) # cache.add fails if the key already exists From 85b9079908a9f00cd074abfea6693539fc11e584 Mon Sep 17 00:00:00 2001 From: Patrick Stegmann Date: Mon, 1 Dec 2014 22:28:19 +0100 Subject: [PATCH 0416/1103] Fixed #2391 --- docs/userguide/periodic-tasks.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/userguide/periodic-tasks.rst b/docs/userguide/periodic-tasks.rst index 7a8293a2a..857bcfd41 100644 --- a/docs/userguide/periodic-tasks.rst +++ b/docs/userguide/periodic-tasks.rst @@ -5,7 +5,7 @@ ================ .. contents:: - :local: +:local: Introduction ============ @@ -251,19 +251,19 @@ The syntax of these crontab expressions are very flexible. Some examples: | ``crontab(minute=0, hour='*/3,8-17')`` | Execute every hour divisible by 3, and | | | every hour during office hours (8am-5pm). | +-----------------------------------------+--------------------------------------------+ -| ``crontab(0, 0, 0, day_of_month='2')`` | Execute on the second day of every month. | +| ``crontab(0, 0, day_of_month='2')`` | Execute on the second day of every month. | | | | +-----------------------------------------+--------------------------------------------+ -| ``crontab(0, 0, 0,'`` | Execute on every even numbered day. | +| ``crontab(0, 0,`` | Execute on every even numbered day. | | ``day_of_month='2-30/3')`` | | +-----------------------------------------+--------------------------------------------+ -| ``crontab(0, 0, 0,`` | Execute on the first and third weeks of | +| ``crontab(0, 0,`` | Execute on the first and third weeks of | | ``day_of_month='1-7,15-21')`` | the month. | +-----------------------------------------+--------------------------------------------+ -| ``crontab(0, 0, 0, day_of_month='11',`` | Execute on 11th of May every year. | +| ``crontab(0, 0, day_of_month='11',`` | Execute on 11th of May every year. | | ``month_of_year='5')`` | | +-----------------------------------------+--------------------------------------------+ -| ``crontab(0, 0, 0,`` | Execute on the first month of every | +| ``crontab(0, 0,`` | Execute on the first month of every | | ``month_of_year='*/3')`` | quarter. | +-----------------------------------------+--------------------------------------------+ From 3272637ef35dd645a8dcb1f0dc342e5424209960 Mon Sep 17 00:00:00 2001 From: Patrick Stegmann Date: Tue, 2 Dec 2014 12:51:47 +0100 Subject: [PATCH 0417/1103] Reverted an unwanted formatting change --- docs/userguide/periodic-tasks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/periodic-tasks.rst b/docs/userguide/periodic-tasks.rst index 857bcfd41..d7ae86f95 100644 --- a/docs/userguide/periodic-tasks.rst +++ b/docs/userguide/periodic-tasks.rst @@ -5,7 +5,7 @@ ================ .. contents:: -:local: + :local: Introduction ============ From 21bdfd50089ee9318d9b4cb7bfa80d04a9347b96 Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Tue, 2 Dec 2014 20:24:28 +0600 Subject: [PATCH 0418/1103] Pass args and kwargs to the context when called locally --- celery/app/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/task.py b/celery/app/task.py index 8e1d791de..9c61edc4c 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -319,7 +319,7 @@ def add_around(self, attr, around): def __call__(self, *args, **kwargs): _task_stack.push(self) - self.push_request() + self.push_request(args=args, kwargs=kwargs) try: # add self if this is a bound task if self.__self__ is not None: From 2d5996484c70d1385707b73ed5517b6b1b58a2bf Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Tue, 2 Dec 2014 20:24:53 +0600 Subject: [PATCH 0419/1103] Fix message when soft timeout exceeded --- celery/worker/request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/worker/request.py b/celery/worker/request.py index 3a28def05..ecfab2679 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -285,8 +285,8 @@ def on_timeout(self, soft, timeout): task_ready(self) if soft: warn('Soft time limit (%ss) exceeded for %s[%s]', - timeout, self.name, self.id) - exc = SoftTimeLimitExceeded(timeout) + soft, self.name, self.id) + exc = SoftTimeLimitExceeded(soft) else: error('Hard time limit (%ss) exceeded for %s[%s]', timeout, self.name, self.id) From 9ea2393aa6dbec6a6645d99ddcfe548b3436706a Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Tue, 2 Dec 2014 20:40:24 +0600 Subject: [PATCH 0420/1103] Meth send_event checks if eventer is present and enabled --- celery/worker/request.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/celery/worker/request.py b/celery/worker/request.py index ecfab2679..f76be4c03 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -310,10 +310,7 @@ def on_success(self, failed__retval__runtime, **kwargs): if self.task.acks_late: self.acknowledge() - if self.eventer and self.eventer.enabled: - self.send_event( - 'task-succeeded', result=retval, runtime=runtime, - ) + self.send_event('task-succeeded', result=retval, runtime=runtime) def on_retry(self, exc_info): """Handler called if the task should be retried.""" From 6a1c30344b42d3fb49fa80e5a4e0d04493166f25 Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Wed, 3 Dec 2014 13:36:38 +0600 Subject: [PATCH 0421/1103] Remove duplicate entry in __all__ --- celery/worker/state.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/celery/worker/state.py b/celery/worker/state.py index 9a3ff49c1..51f55a44a 100644 --- a/celery/worker/state.py +++ b/celery/worker/state.py @@ -27,8 +27,7 @@ __all__ = ['SOFTWARE_INFO', 'reserved_requests', 'active_requests', 'total_count', 'revoked', 'task_reserved', 'maybe_shutdown', - 'task_accepted', 'task_ready', 'task_reserved', 'task_ready', - 'Persistent'] + 'task_accepted', 'task_reserved', 'task_ready', 'Persistent'] #: Worker software/platform information. SOFTWARE_INFO = {'sw_ident': 'py-celery', From 8558b8478ad3da794eb41dadf29727ec5e94c9aa Mon Sep 17 00:00:00 2001 From: GDvalle Date: Thu, 4 Dec 2014 11:21:03 -0600 Subject: [PATCH 0422/1103] Syntax fix in Abstract classes example --- docs/userguide/tasks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index be36a43ac..07f2f4b38 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -1164,7 +1164,7 @@ base class for new task types. abstract = True def after_return(self, *args, **kwargs): - print('Task returned: {0!r}'.format(self.request) + print('Task returned: {0!r}'.format(self.request)) @app.task(base=DebugTask) From 6a2a110e9aa417dd02de623d51b4dc5dff0a457c Mon Sep 17 00:00:00 2001 From: Vladimir Rutsky Date: Tue, 9 Dec 2014 02:03:59 +0300 Subject: [PATCH 0423/1103] remove redundant backticks --- docs/userguide/workers.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index a0ad3cdfd..e2e7a007b 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -191,8 +191,8 @@ is the *process index* not the process count or pid. three log files: - :file:`worker1.log` (main process) - - :file:`worker1-1.log`` (pool process 1) - - :file:`worker1-2.log`` (pool process 2) + - :file:`worker1-1.log` (pool process 1) + - :file:`worker1-2.log` (pool process 2) .. _worker-concurrency: From 75a0c13890c1c39c4a696843a16d95ae4089dcf1 Mon Sep 17 00:00:00 2001 From: Vladimir Rutsky Date: Tue, 9 Dec 2014 02:12:46 +0300 Subject: [PATCH 0424/1103] remove uneeded dot --- docs/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 7bfc544f1..2fd2d9bee 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -760,7 +760,7 @@ to use both. `Task.retry` is used to retry tasks, notably for expected errors that is catchable with the `try:` block. The AMQP transaction is not used -for these errors: **if the task raises an exception it is still acknowledged!**. +for these errors: **if the task raises an exception it is still acknowledged!** The `acks_late` setting would be used when you need the task to be executed again if the worker (for some reason) crashes mid-execution. From b575487b7862473d2c33fc77ab218600068157aa Mon Sep 17 00:00:00 2001 From: Vladimir Rutsky Date: Tue, 9 Dec 2014 02:23:59 +0300 Subject: [PATCH 0425/1103] Add missing dot --- docs/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 7bfc544f1..e720eaf7d 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -786,7 +786,7 @@ scenario of course, but you can probably imagine something far more sinister. So for ease of programming we have less reliability; It's a good default, users who require it and know what they are doing can still enable acks_late (and in the future hopefully -use manual acknowledgement) +use manual acknowledgement). In addition `Task.retry` has features not available in AMQP transactions: delay between retries, max retries, etc. From d07268291b4cde426594c1c99f1a1fbe24cecb25 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 9 Dec 2014 14:15:24 +0000 Subject: [PATCH 0426/1103] Disable failing test --- celery/tests/events/test_state.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/tests/events/test_state.py b/celery/tests/events/test_state.py index 99911a897..6ed41dad4 100644 --- a/celery/tests/events/test_state.py +++ b/celery/tests/events/test_state.py @@ -18,7 +18,7 @@ ) from celery.five import range from celery.utils import uuid -from celery.tests.case import AppCase, Mock, patch +from celery.tests.case import AppCase, Mock, SkipTest, patch try: Decimal(2.6) @@ -319,6 +319,7 @@ def test_task_logical_clock_ordering(self): self.assertEqual(now[2][0], tB) def test_task_descending_clock_ordering(self): + raise SkipTest('not working') state = State() r = ev_logical_clock_ordering(state) tA, tB, tC = r.uids From 740f017429b46d36e1b8dd07530b44debfa86b7a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 9 Dec 2014 16:17:39 +0000 Subject: [PATCH 0427/1103] Use format :#x --- celery/app/utils.py | 2 +- celery/apps/worker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/app/utils.py b/celery/app/utils.py index 62006c3e4..32ad7c24d 100644 --- a/celery/app/utils.py +++ b/celery/app/utils.py @@ -47,7 +47,7 @@ def appstr(app): """String used in __repr__ etc, to id app instances.""" - return '{0}:0x{1:x}'.format(app.main or '__main__', id(app)) + return '{0}:{1:#x}'.format(app.main or '__main__', id(app)) class Settings(ConfigurationView): diff --git a/celery/apps/worker.py b/celery/apps/worker.py index 887471a39..e5a12548d 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -206,7 +206,7 @@ def extra_info(self): def startup_info(self): app = self.app concurrency = string(self.concurrency) - appr = '{0}:0x{1:x}'.format(app.main or '__main__', id(app)) + appr = '{0}:{1:#x}'.format(app.main or '__main__', id(app)) if not isinstance(app.loader, AppLoader): loader = qualname(app.loader) if loader.startswith('celery.loaders'): From 3ef84b7b9d29c4402e9a1300ff76d13fda49c503 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 12 Dec 2014 12:43:40 +0000 Subject: [PATCH 0428/1103] Sets EVENT_QUEUE_TTL=5 and EVENT_QUEUE_EXPIRES=60 by default --- celery/app/defaults.py | 4 ++-- docs/configuration.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index bdbc52c56..ca819eb46 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -119,8 +119,8 @@ def __repr__(self): 'ENABLE_UTC': Option(True, type='bool'), 'ENABLE_REMOTE_CONTROL': Option(True, type='bool'), 'EVENT_SERIALIZER': Option('json'), - 'EVENT_QUEUE_EXPIRES': Option(None, type='float'), - 'EVENT_QUEUE_TTL': Option(None, type='float'), + 'EVENT_QUEUE_EXPIRES': Option(60.0, type='float'), + 'EVENT_QUEUE_TTL': Option(5.0, type='float'), 'IMPORTS': Option((), type='tuple'), 'INCLUDE': Option((), type='tuple'), 'IGNORE_RESULT': Option(False, type='bool'), diff --git a/docs/configuration.rst b/docs/configuration.rst index 9100f2e61..f275fdf86 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1618,7 +1618,7 @@ CELERY_EVENT_QUEUE_EXPIRES :transports supported: ``amqp`` -Expiry time in seconds (int/float) for when a monitor clients +Expiry time in seconds (int/float) for when after a monitor clients event queue will be deleted (``x-expires``). Default is never, relying on the queue autodelete setting. From 0a9fd7e21452641db0690bac5a8ea3e6795cc8f1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 12 Dec 2014 12:44:45 +0000 Subject: [PATCH 0429/1103] Now requires billiard 3.4b1 --- celery/concurrency/asynpool.py | 6 ++++- celery/platforms.py | 48 ++-------------------------------- 2 files changed, 7 insertions(+), 47 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 2a68d6dbc..656e4a0cf 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -179,7 +179,6 @@ def _select(readers=None, writers=None, err=None, timeout=0, class Worker(_pool.Worker): """Pool worker process.""" - dead = False def on_loop_start(self, pid): # our version sends a WORKER_UP message when the process is ready @@ -351,6 +350,11 @@ class AsynPool(_pool.Pool): ResultHandler = ResultHandler Worker = Worker + def WorkerProcess(self, worker): + worker = super(AsynPool, self).WorkerProcess(worker) + worker.dead = False + return worker + def __init__(self, processes=None, synack=False, sched_strategy=None, *args, **kwargs): self.sched_strategy = SCHED_STRATEGIES.get(sched_strategy, diff --git a/celery/platforms.py b/celery/platforms.py index 6b0900916..194c2b9bd 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -25,14 +25,14 @@ from billiard.process import current_process except ImportError: current_process = None +from billiard.compat import get_fdmax, close_open_fds # fileno used to be in this module from kombu.utils import maybe_fileno from kombu.utils.encoding import safe_str from contextlib import contextmanager from .local import try_import -from .five import items, range, reraise, string_t, zip_longest -from .utils.functional import uniq +from .five import items, reraise, string_t _setproctitle = try_import('setproctitle') resource = try_import('resource') @@ -110,26 +110,6 @@ class LockFailed(Exception): """Raised if a pidlock can't be acquired.""" -def get_fdmax(default=None): - """Return the maximum number of open file descriptors - on this system. - - :keyword default: Value returned if there's no file - descriptor limit. - - """ - try: - return os.sysconf('SC_OPEN_MAX') - except: - pass - if resource is None: # Windows - return default - fdmax = resource.getrlimit(resource.RLIMIT_NOFILE)[1] - if fdmax == resource.RLIM_INFINITY: - return default - return fdmax - - class Pidfile(object): """Pidfile @@ -268,30 +248,6 @@ def _create_pidlock(pidfile): return pidlock -if hasattr(os, 'closerange'): - - def close_open_fds(keep=None): - # must make sure this is 0-inclusive (Issue #1882) - keep = list(uniq(sorted( - f for f in map(maybe_fileno, keep or []) if f is not None - ))) - maxfd = get_fdmax(default=2048) - kL, kH = iter([-1] + keep), iter(keep + [maxfd]) - for low, high in zip_longest(kL, kH): - if low + 1 != high: - os.closerange(low + 1, high) - -else: - - def close_open_fds(keep=None): # noqa - keep = [maybe_fileno(f) - for f in (keep or []) if maybe_fileno(f) is not None] - for fd in reversed(range(get_fdmax(default=2048))): - if fd not in keep: - with ignore_errno(errno.EBADF): - os.close(fd) - - class DaemonContext(object): _is_open = False From 6c26da3db6fa68f34778a8d38cc3eba893f96803 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Sun, 14 Dec 2014 22:36:43 +0000 Subject: [PATCH 0430/1103] Autoscale: maybe_scale must accept argument. Closes #2411 --- celery/worker/autoscale.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/worker/autoscale.py b/celery/worker/autoscale.py index c8038b36d..265feda49 100644 --- a/celery/worker/autoscale.py +++ b/celery/worker/autoscale.py @@ -91,8 +91,8 @@ def _maybe_scale(self, req=None): self.scale_down((procs - cur) - self.min_concurrency) return True - def maybe_scale(self): - if self._maybe_scale(): + def maybe_scale(self, req=None): + if self._maybe_scale(req): self.pool.maintain_pool() def update(self, max=None, min=None): From 63dcb10179b234d0ad080c025248d86165cd6ded Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Sun, 14 Dec 2014 22:43:41 +0000 Subject: [PATCH 0431/1103] Chord unlock: Retry if deps.ready() raises. Closes #2404 --- celery/app/builtins.py | 64 +++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/celery/app/builtins.py b/celery/app/builtins.py index 90cc9c9b7..50db6ee7c 100644 --- a/celery/app/builtins.py +++ b/celery/app/builtins.py @@ -57,8 +57,8 @@ def add_unlock_chord_task(app): default_propagate = app.conf.CELERY_CHORD_PROPAGATES @app.task(name='celery.chord_unlock', max_retries=None, shared=False, - default_retry_delay=1, ignore_result=True, lazy=False) - def unlock_chord(group_id, callback, interval=None, propagate=None, + default_retry_delay=1, ignore_result=True, lazy=False, bind=True) + def unlock_chord(self, group_id, callback, interval=None, propagate=None, max_retries=None, result=None, Result=app.AsyncResult, GroupResult=app.GroupResult, result_from_tuple=result_from_tuple): @@ -69,45 +69,51 @@ def unlock_chord(group_id, callback, interval=None, propagate=None, # exception set to ChordError. propagate = default_propagate if propagate is None else propagate if interval is None: - interval = unlock_chord.default_retry_delay + interval = self.default_retry_delay # check if the task group is ready, and if so apply the callback. callback = maybe_signature(callback, app) deps = GroupResult( group_id, [result_from_tuple(r, app=app) for r in result], + app=app, ) j = deps.join_native if deps.supports_native_join else deps.join - if deps.ready(): - callback = maybe_signature(callback, app=app) + try: + ready = deps.ready() + except Exception as exc: + raise self.retry( + exc=exc, countdown=interval, max_retries=max_retries, + ) + else: + if not ready: + raise self.retry(countdown=interval, max_retries=max_retries) + + callback = maybe_signature(callback, app=app) + try: + with allow_join_result(): + ret = j(timeout=3.0, propagate=propagate) + except Exception as exc: + try: + culprit = next(deps._failed_join_report()) + reason = 'Dependency {0.id} raised {1!r}'.format( + culprit, exc, + ) + except StopIteration: + reason = repr(exc) + logger.error('Chord %r raised: %r', group_id, exc, exc_info=1) + app.backend.chord_error_from_stack(callback, + ChordError(reason)) + else: try: - with allow_join_result(): - ret = j(timeout=3.0, propagate=propagate) + callback.delay(ret) except Exception as exc: - try: - culprit = next(deps._failed_join_report()) - reason = 'Dependency {0.id} raised {1!r}'.format( - culprit, exc, - ) - except StopIteration: - reason = repr(exc) logger.error('Chord %r raised: %r', group_id, exc, exc_info=1) - app.backend.chord_error_from_stack(callback, - ChordError(reason)) - else: - try: - callback.delay(ret) - except Exception as exc: - logger.error('Chord %r raised: %r', group_id, exc, - exc_info=1) - app.backend.chord_error_from_stack( - callback, - exc=ChordError('Callback error: {0!r}'.format(exc)), - ) - else: - raise unlock_chord.retry(countdown=interval, - max_retries=max_retries) + app.backend.chord_error_from_stack( + callback, + exc=ChordError('Callback error: {0!r}'.format(exc)), + ) return unlock_chord From a6ff6c1d62d8747c60eb665e915b568755ab2f90 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Sun, 14 Dec 2014 23:03:49 +0000 Subject: [PATCH 0432/1103] Redis new_join: Must receive error not einfo --- celery/app/trace.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/celery/app/trace.py b/celery/app/trace.py index fa75c4a6e..8afc1988d 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -368,6 +368,8 @@ def trace_task(uuid, args, kwargs, request=None): ) except Exception as exc: I, R, state, retval = on_error(task_request, exc, uuid) + if task_request.chord: + on_chord_part_return(task, state, exc) except BaseException as exc: raise else: @@ -402,6 +404,8 @@ def trace_task(uuid, args, kwargs, request=None): except EncodeError as exc: I, R, state, retval = on_error(task_request, exc, uuid) else: + if task_request.chord: + on_chord_part_return(task, state, retval) if task_on_success: task_on_success(retval, uuid, args, kwargs) if success_receivers: @@ -416,8 +420,6 @@ def trace_task(uuid, args, kwargs, request=None): # -* POST *- if state not in IGNORE_STATES: - if task_request.chord: - on_chord_part_return(task, state, R) if task_after_return: task_after_return( state, retval, uuid, args, kwargs, None, From 21b86d3e9ee73a84d5ff458c8520314efe2138a8 Mon Sep 17 00:00:00 2001 From: Bert Vanderbauwhede Date: Thu, 18 Dec 2014 12:40:52 +0100 Subject: [PATCH 0433/1103] Add command option --executable --- celery/bin/base.py | 5 +++++ celery/bin/celeryd_detach.py | 5 ++++- celery/bin/multi.py | 5 +++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/celery/bin/base.py b/celery/bin/base.py index e9beb15ee..f74e1e7cb 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -63,6 +63,10 @@ Optional directory to change to after detaching. +.. cmdoption:: --executable + + Executable to use for the detached process. + """ from __future__ import absolute_import, print_function, unicode_literals @@ -651,4 +655,5 @@ def daemon_options(default_pidfile=None, default_logfile=None): Option('--uid', default=None), Option('--gid', default=None), Option('--umask', default=None), + Option('--executable', default=None), ) diff --git a/celery/bin/celeryd_detach.py b/celery/bin/celeryd_detach.py index 862fc8979..d9d6141d7 100644 --- a/celery/bin/celeryd_detach.py +++ b/celery/bin/celeryd_detach.py @@ -38,11 +38,14 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, - gid=None, umask=None, working_directory=None, fake=False, app=None): + gid=None, umask=None, working_directory=None, fake=False, app=None, + executable=None): fake = 1 if C_FAKEFORK else fake with detached(logfile, pidfile, uid, gid, umask, working_directory, fake, after_forkers=False): try: + if executable is not None: + path = executable os.execv(path, [path] + argv) except Exception: if app is None: diff --git a/celery/bin/multi.py b/celery/bin/multi.py index a7eb541d5..d0ea4a668 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -250,7 +250,7 @@ def start(self, argv, cmd): self.note('> Starting nodes...') for node in multi_args(p, cmd): self.note('\t> {0}: '.format(node.name), newline=False) - retcode = self.waitexec(node.argv) + retcode = self.waitexec(node.argv, path=p.options['--executable']) self.note(retcode and self.FAILED or self.OK) retcodes.append(retcode) self.retcode = int(any(retcodes)) @@ -262,6 +262,7 @@ def with_detacher_default_options(self, p): '--cmd', '-m {0}'.format(celery_exe('worker', '--detach')), ) + _setdefaultopt(p.options, ['--executable'], sys.executable) def signal_node(self, nodename, pid, sig): try: @@ -382,7 +383,7 @@ def restart(self, argv, cmd): def on_node_shutdown(nodename, argv, pid): self.note(self.colored.blue( '> Restarting node {0}: '.format(nodename)), newline=False) - retval = self.waitexec(argv) + retval = self.waitexec(argv, path=p.options['--executable']) self.note(retval and self.FAILED or self.OK) retvals.append(retval) From 8d146d8c14fad744b62694359afe5f02e141ace3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 18 Dec 2014 15:02:11 +0000 Subject: [PATCH 0434/1103] Tests passing --- celery/tests/bin/test_celeryd_detach.py | 2 +- celery/tests/bin/test_multi.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/tests/bin/test_celeryd_detach.py b/celery/tests/bin/test_celeryd_detach.py index 6c529e9c4..9aa80fa08 100644 --- a/celery/tests/bin/test_celeryd_detach.py +++ b/celery/tests/bin/test_celeryd_detach.py @@ -87,7 +87,7 @@ def test_execute_from_commandline(self, detach, exit): detach.assert_called_with( path=x.execv_path, uid=None, gid=None, umask=None, fake=False, logfile='/var/log', pidfile='celeryd.pid', - working_directory=None, + working_directory=None, executable=None, argv=x.execv_argv + [ '-c', '1', '-lDEBUG', '--logfile=/var/log', '--pidfile=celeryd.pid', diff --git a/celery/tests/bin/test_multi.py b/celery/tests/bin/test_multi.py index 653c8c126..2d81ccd17 100644 --- a/celery/tests/bin/test_multi.py +++ b/celery/tests/bin/test_multi.py @@ -240,7 +240,7 @@ def test_restart(self): waitexec.return_value = 0 callback('jerry', ['arg'], 13) - waitexec.assert_called_with(['arg']) + waitexec.assert_called_with(['arg'], path=sys.executable) self.assertIn('OK', self.fh.getvalue()) self.fh.seek(0) self.fh.truncate() From 4f09ed1c997deef799e98ccd7aba7b0070f9ab69 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Sun, 28 Dec 2014 01:16:40 -0800 Subject: [PATCH 0435/1103] This makes action=append with --arg 1 --arg 2 work --- celery/bin/base.py | 18 ++++++++++++++++-- celery/tests/bin/test_base.py | 16 ++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/celery/bin/base.py b/celery/bin/base.py index f74e1e7cb..c803ced2f 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -505,6 +505,14 @@ def process_cmdline_config(self, argv): def parse_preload_options(self, args): return self.preparse_options(args, self.preload_options) + def add_append_opt(self, acc, opt, value): + default = opt.default or [] + + if opt.dest not in acc: + acc[opt.dest] = default + + acc[opt.dest].append(value) + def preparse_options(self, args, options): acc = {} opts = {} @@ -520,13 +528,19 @@ def preparse_options(self, args, options): key, value = arg.split('=', 1) opt = opts.get(key) if opt: - acc[opt.dest] = value + if opt.action == 'append': + self.add_append_opt(acc, opt, value) + else: + acc[opt.dest] = value else: opt = opts.get(arg) if opt and opt.takes_value(): # optparse also supports ['--opt', 'value'] # (Issue #1668) - acc[opt.dest] = args[index + 1] + if opt.action == 'append': + self.add_append_opt(acc, opt, args[index + 1]) + else: + acc[opt.dest] = args[index + 1] index += 1 elif opt and opt.action == 'store_true': acc[opt.dest] = True diff --git a/celery/tests/bin/test_base.py b/celery/tests/bin/test_base.py index 8d1d0d55d..61d56fe0d 100644 --- a/celery/tests/bin/test_base.py +++ b/celery/tests/bin/test_base.py @@ -314,3 +314,19 @@ def test_parse_preload_options_shortopt(self): cmd.preload_options = (Option('-s', action='store', dest='silent'), ) acc = cmd.parse_preload_options(['-s', 'yes']) self.assertEqual(acc.get('silent'), 'yes') + + def test_parse_preload_options_with_equals_and_append(self): + cmd = Command() + opt = Option('--zoom', action='append', default=[]) + cmd.preload_options = (opt,) + acc = cmd.parse_preload_options(['--zoom=1', '--zoom=2']) + + self.assertEqual(acc, {'zoom': ['1', '2']}) + + def test_parse_preload_options_without_equals_and_append(self): + cmd = Command() + opt = Option('--zoom', action='append', default=[]) + cmd.preload_options = (opt,) + acc = cmd.parse_preload_options(['--zoom', '1', '--zoom', '2']) + + self.assertEqual(acc, {'zoom': ['1', '2']}) From 8fc1dec96ec9476c7250ae5aee69acf08a324241 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Sun, 28 Dec 2014 17:10:58 -0800 Subject: [PATCH 0436/1103] arguements -> arguments --- docs/userguide/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/signals.rst b/docs/userguide/signals.rst index fd6dae378..bfa2c5b5c 100644 --- a/docs/userguide/signals.rst +++ b/docs/userguide/signals.rst @@ -81,7 +81,7 @@ Note that this is executed in the process sending the task. Sender is the name of the task being sent. -Provides arguements: +Provides arguments: * body From 4e1909e35b21b791c560602df7434c22c998e861 Mon Sep 17 00:00:00 2001 From: Gunnlaugur Thor Briem Date: Fri, 2 Jan 2015 11:29:13 +0000 Subject: [PATCH 0437/1103] Fix typo in COMPAT_MODULES (Fixing this since I happened to came across it) --- celery/five.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/five.py b/celery/five.py index bfa42caf6..732ccde97 100644 --- a/celery/five.py +++ b/celery/five.py @@ -77,7 +77,7 @@ def _compat_periodic_task_decorator(*args, **kwargs): 'log': { 'get_default_logger': 'log.get_default_logger', 'setup_logger': 'log.setup_logger', - 'setup_loggig_subsystem': 'log.setup_logging_subsystem', + 'setup_logging_subsystem': 'log.setup_logging_subsystem', 'redirect_stdouts_to_logger': 'log.redirect_stdouts_to_logger', }, 'messaging': { From 01ca4edaefa4e8e889b0aaa0b7ee392531d783fd Mon Sep 17 00:00:00 2001 From: Corey Farwell Date: Tue, 6 Jan 2015 10:28:47 -0500 Subject: [PATCH 0438/1103] Specify return type for apply_sync --- celery/app/task.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/celery/app/task.py b/celery/app/task.py index 8e1d791de..3587b9776 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -439,6 +439,10 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, attribute. Trailing can also be disabled by default using the :attr:`trail` attribute :keyword publisher: Deprecated alias to ``producer``. + + :rtype :class:`celery.result.AsyncResult`: if + :setting:`CELERY_ALWAYS_EAGER` is not set, otherwise + :class:`celery.result.EagerResult`: Also supports all keyword arguments supported by :meth:`kombu.Producer.publish`. From 64116c8b91ab6252433039263a5290bba3ab4794 Mon Sep 17 00:00:00 2001 From: Brian Dixon Date: Wed, 7 Jan 2015 17:48:44 -0500 Subject: [PATCH 0439/1103] Update next-steps.rst Added note about running the worker in the directory above proj. If the worker is run within the proj directory resolving the error isn't intuitive from the resulting message --- docs/getting-started/next-steps.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/next-steps.rst b/docs/getting-started/next-steps.rst index b6a49a72f..25a2de336 100644 --- a/docs/getting-started/next-steps.rst +++ b/docs/getting-started/next-steps.rst @@ -70,7 +70,7 @@ you simply import this instance. Starting the worker ------------------- -The :program:`celery` program can be used to start the worker: +The :program:`celery` program can be used to start the worker (you need to run the worker in the directory above proj): .. code-block:: bash From b3d8ba2781189b7de0894f11295e815fa0bbd0b5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 12 Jan 2015 16:41:34 +0000 Subject: [PATCH 0440/1103] Closes #2326 for master branch --- celery/backends/base.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index a802bb1cf..50dec0c0a 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -439,18 +439,25 @@ def _strip_prefix(self, key): return bytes_to_str(key[len(prefix):]) return bytes_to_str(key) + def _filter_ready(self, values, READY_STATES=states.READY_STATES): + for k, v in values: + if v is not None: + v = self.decode_result(v) + if v['status'] in READY_STATES: + yield k, v + def _mget_to_results(self, values, keys): if hasattr(values, 'items'): # client returns dict so mapping preserved. return { - self._strip_prefix(k): self.decode_result(v) - for k, v in items(values) if v is not None + self._strip_prefix(k): v + for k, v in self._filter_ready(items(values)) } else: # client returns list so need to recreate mapping. return { - bytes_to_str(keys[i]): self.decode_result(value) - for i, value in enumerate(values) if value is not None + bytes_to_str(keys[i]): v + for i, v in self._filter_ready(enumerate(values)) } def get_many(self, task_ids, timeout=None, interval=0.5, no_ack=True, From 3f713bed7ce0db0b80c28715cb0c2508a1406dcc Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 13 Jan 2015 20:02:23 +0000 Subject: [PATCH 0441/1103] apply_async: Use specific queue is queue argument already is a Queue instance. Closes celery/kombu#438 --- celery/app/routes.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/celery/app/routes.py b/celery/app/routes.py index d654f9d70..c3952b10d 100644 --- a/celery/app/routes.py +++ b/celery/app/routes.py @@ -9,6 +9,8 @@ """ from __future__ import absolute_import +from kombu import Queue + from celery.exceptions import QueueNotFound from celery.five import string_t from celery.utils import lpmerge @@ -63,13 +65,14 @@ def expand_destination(self, route): queue = route.pop('queue', None) if queue: - try: - Q = self.queues[queue] # noqa - except KeyError: - raise QueueNotFound( - 'Queue {0!r} missing from CELERY_QUEUES'.format(queue)) - # needs to be declared by publisher - route['queue'] = Q + if isinstance(queue, Queue): + route['queue'] = queue + else: + try: + route['queue'] = self.queues[queue] + except KeyError: + raise QueueNotFound( + 'Queue {0!r} missing from CELERY_QUEUES'.format(queue)) return route def lookup_route(self, task, args=None, kwargs=None): From 3c25f3abddeab4c1efae037f164da26d5a8e6bbf Mon Sep 17 00:00:00 2001 From: Luke Burden Date: Wed, 14 Jan 2015 22:59:59 +1100 Subject: [PATCH 0442/1103] Fixes issue #2453 where django db connections are not closed during worker initialisation. --- celery/fixups/django.py | 2 +- celery/tests/fixups/test_django.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index c1ae62e21..d38b6f195 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -227,7 +227,7 @@ def close_database(self, **kwargs): def _close_database(self): try: - funs = [conn.close for conn in self._db.connections] + funs = [conn.close for conn in self._db.connections.all()] except AttributeError: if hasattr(self._db, 'close_old_connections'): # django 1.6 funs = [self._db.close_old_connections] diff --git a/celery/tests/fixups/test_django.py b/celery/tests/fixups/test_django.py index 17990a6e8..9235bd005 100644 --- a/celery/tests/fixups/test_django.py +++ b/celery/tests/fixups/test_django.py @@ -205,10 +205,13 @@ def test_close_database(self): def test__close_database(self): with self.fixup_context(self.app) as (f, _, _): - conns = f._db.connections = [Mock(), Mock(), Mock()] + conns = [Mock(), Mock(), Mock()] conns[1].close.side_effect = KeyError('already closed') f.database_errors = (KeyError, ) + f._db.connections = Mock() # ConnectionHandler + f._db.connections.all.side_effect = lambda: conns + f._close_database() conns[0].close.assert_called_with() conns[1].close.assert_called_with() From fbe2f8e4c5cb5d4bc42840c9025e27d40d41613c Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Sun, 18 Jan 2015 11:28:01 +0600 Subject: [PATCH 0443/1103] Fix __wrapped__ to work properly with inspect.Signature --- celery/app/base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index bc1eda601..cd68d5266 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -279,15 +279,16 @@ def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): base = base or self.Task if name not in self._tasks: + run = fun if bind else staticmethod(fun) task = type(fun.__name__, (base, ), dict({ 'app': self, 'name': name, - 'run': fun if bind else staticmethod(fun), + 'run': run, '_decorated': True, '__doc__': fun.__doc__, '__module__': fun.__module__, '__header__': staticmethod(head_from_fun(fun, bound=bind)), - '__wrapped__': fun}, **options))() + '__wrapped__': run}, **options))() self._tasks[task.name] = task task.bind(self) # connects task to this app else: From 1380902bb4e75d9c5aed4954032442339620de05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ionel=20Cristian=20M=C4=83rie=C8=99?= Date: Sat, 24 Jan 2015 10:44:53 +0200 Subject: [PATCH 0444/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 03c3b6ac1..65fb14ff0 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -176,3 +176,4 @@ Nathan Van Gheem, 2014/10/28 Gino Ledesma, 2014/10/28 Thomas French, 2014/11/10 Michael Permana, 2014/11/6 +Luke Burden, 2015/01/24 From 3635abb11d905201d2671c0889627f98b68e9c11 Mon Sep 17 00:00:00 2001 From: Vladislav Stepanov <8uk.8ak@gmail.com> Date: Thu, 29 Jan 2015 13:07:11 +0300 Subject: [PATCH 0445/1103] Configuration parameter name was part of `p #id` --- docs/configuration.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index f275fdf86..ee599af3a 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -310,8 +310,11 @@ the :setting:`CELERY_RESULT_ENGINE_OPTIONS` setting:: # echo enables verbose logging from SQLAlchemy. CELERY_RESULT_ENGINE_OPTIONS = {'echo': True} - .. setting:: CELERY_RESULT_DB_SHORT_LIVED_SESSIONS + +Short lived sessions +~~~~~~~~~~~~~~~~~~~~ + CELERY_RESULT_DB_SHORT_LIVED_SESSIONS = True Short lived sessions are disabled by default. If enabled they can drastically reduce From 106b40b01e203f2796686685a0ff107d22b780dc Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sun, 1 Feb 2015 20:51:38 +0100 Subject: [PATCH 0446/1103] add note about Django 1.6 transaction changes to userguide See #2472. Django 1.6 introduced a change to the tranasction model, switching to autocommit by default and deprecating much of the old transaction API, with plans to remove it completely in 1.8. This commit adds a note to the userguide section that discusses race conditions involving database transactions, pointing out the ramifications of the example code in Django 1.6+ --- docs/userguide/tasks.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index be36a43ac..0ccb956b4 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -1548,6 +1548,16 @@ depending on state from the current transaction*: transaction.commit() expand_abbreviations.delay(article.pk) +Note that Django 1.6 and later enable autocommit mode by default +(deprecating `commit_on_success` and `commit_manually`), automatically +wrapping each SQL query in its own transaction, avoiding the race +condition by default and making it less likely that you'll encounter +the above problem. However, enabling `ATOMIC_REQUESTS` on the database +connection will bring back the transaction per request model and the +race condition along with it. In this case, the simplest solution is +just to use the `@transaction.non_atomic_requests` to switch it back +to autocommit for that view. + .. _task-example: Example From d38faad887b64f047075ed11405ee452e1e52b9e Mon Sep 17 00:00:00 2001 From: David Baumgold Date: Sun, 1 Feb 2015 15:18:39 -0500 Subject: [PATCH 0447/1103] Add Travis and Coveralls badges to README --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 7bffaab40..a6c753ce7 100644 --- a/README.rst +++ b/README.rst @@ -4,6 +4,8 @@ .. image:: http://cloud.github.com/downloads/celery/celery/celery_128.png +|build-status| |coverage-status| + :Version: 3.2.0a1 (Cipater) :Web: http://celeryproject.org/ :Download: http://pypi.python.org/pypi/celery/ @@ -425,3 +427,7 @@ file in the top distribution directory for the full license text. :alt: Bitdeli badge :target: https://bitdeli.com/free +.. |build-status| image:: https://travis-ci.org/celery/celery.svg?branch=master + :target: https://travis-ci.org/celery/celery +.. |coverage-status| image:: https://coveralls.io/repos/celery/celery/badge.svg + :target: https://coveralls.io/r/celery/celery From d80a749cec7528da44e5b76ad52a905ddc04793c Mon Sep 17 00:00:00 2001 From: Jelle Verstraaten Date: Wed, 11 Feb 2015 14:50:36 +0100 Subject: [PATCH 0448/1103] Update celerybeat to use sh instead of bash Updated the init script to work without assuming bash is always available from /bin/bash/ Also see issue #2496 --- extra/generic-init.d/celerybeat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/generic-init.d/celerybeat b/extra/generic-init.d/celerybeat index 27f31111e..fb31ca292 100755 --- a/extra/generic-init.d/celerybeat +++ b/extra/generic-init.d/celerybeat @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh -e # ========================================================= # celerybeat - Starts the Celery periodic task scheduler. # ========================================================= From d332e1960f43b7fd88e1f0632e6d4e8a10e3e88f Mon Sep 17 00:00:00 2001 From: PMickael Date: Thu, 12 Feb 2015 21:19:51 +0100 Subject: [PATCH 0449/1103] Celery 3.2 : Contrib.Batches, adapt to new task message protocol --- celery/contrib/batches.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/celery/contrib/batches.py b/celery/contrib/batches.py index 5bfa3a902..da04c0577 100644 --- a/celery/contrib/batches.py +++ b/celery/contrib/batches.py @@ -90,6 +90,7 @@ def wot_api_real(urls): from celery.utils.log import get_logger from celery.worker.request import Request from celery.utils import noop +from celery.worker.strategy import proto1_to_proto2 __all__ = ['Batches'] @@ -163,8 +164,8 @@ def __init__(self, id, name, args, kwargs, delivery_info, hostname): @classmethod def from_request(cls, request): - return cls(request.id, request.name, request.args, - request.kwargs, request.delivery_info, request.hostname) + return cls(request.id, request.name, request.body[0], + request.body[1], request.delivery_info, request.hostname) class Batches(Task): @@ -196,10 +197,21 @@ def Strategy(self, task, app, consumer): flush_buffer = self._do_flush def task_message_handler(message, body, ack, reject, callbacks, **kw): - request = Req(body, on_ack=ack, app=app, hostname=hostname, - events=eventer, task=task, - connection_errors=connection_errors, - delivery_info=message.delivery_info) + if body is None: 31513 ? S 125:09 /usr/bin/python -m celery worker --without-heartbeat -c 50 --pool=eventlet -n celery6@ns326150.ip-37-187-158.eu --app=mai + body, headers, decoded, utc = ( n -Q rss --without-gossip --logfile=/home/logs/rss.log --pidfile=celery6.pid + message.body, message.headers, False, True, 31528 ? R 128:34 /usr/bin/python -m celery worker --without-heartbeat -c 50 --pool=eventlet -n celery7@ns326150.ip-37-187-158.eu --app=mai + ) n -Q rss --without-gossip --logfile=/home/logs/rss.log --pidfile=celery7.pid + if not body_can_be_buffer: 31543 ? S 124:32 /usr/bin/python -m celery worker --without-heartbeat -c 50 --pool=eventlet -n celery8@ns326150.ip-37-187-158.eu --app=mai + body = bytes(body) if isinstance(body, buffer_t) else body n -Q rss --without-gossip --logfile=/home/logs/rss.log --pidfile=celery8.pid + else: 26150 ? S 0:50 /usr/bin/python -m celery worker --without-heartbeat -c 2 --pool=eventlet -n engines@ns326150.ip-37-187-158.eu --app=main + body, headers, decoded, utc = proto1_to_proto2(message, body) -Q engines --without-gossip --logfile=/home/logs/engines.log --pidfile=/home/logs/pid-engines.pid + 22409 ? S 0:00 /usr/bin/python -m celery worker --without-heartbeat -c 1 -n elasticsearch_bulk_actions@ns326150.ip-37-187-158.eu --app=m + request = Req( ain -Q elasticsearch_bulk_actions --without-gossip --logfile=/home/logs/elasticsearch_bulk_actions.log --pidfile=elasticsearch_bulk_actions.pid + message, 22459 ? S 0:00 \_ /usr/bin/python -m celery worker --without-heartbeat -c 1 -n elasticsearch_bulk_actions@ns326150.ip-37-187-158.eu --a + on_ack=ack, on_reject=reject, app=app, hostname=hostname, pp=main -Q elasticsearch_bulk_actions --without-gossip --logfile=/home/logs/elasticsearch_bulk_actions.log --pidfile=elasticsearch_bulk_actions.pid + eventer=eventer, task=task, connection_errors=connection_errors, 22419 ? S 0:00 /usr/bin/python -m celery worker --without-heartbeat -c 1 -n celery@ns326150.ip-37-187-158.eu --app=main -Q elasticsearch + body=body, headers=headers, decoded=decoded, utc=utc, _bulk_actions --without-gossip --logfile=/home/logs/elasticsearch_bulk_actions.log --pidfile=celery.pid + ) put_buffer(request) if self._tref is None: # first request starts flush timer. From f27d9582c19e7d9338f7d11fbf5bf2556097e071 Mon Sep 17 00:00:00 2001 From: PMickael Date: Thu, 12 Feb 2015 21:46:01 +0100 Subject: [PATCH 0450/1103] Worker option : Prefetch-multiplier Use with batches --- celery/bin/worker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/celery/bin/worker.py b/celery/bin/worker.py index 05b249d69..d01be1097 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -242,6 +242,8 @@ def get_options(self): default=conf.CELERYD_TASK_SOFT_TIME_LIMIT, type='float'), Option('--maxtasksperchild', dest='max_tasks_per_child', default=conf.CELERYD_MAX_TASKS_PER_CHILD, type='int'), + Option('--prefetch-multiplier', dest='prefetch_multiplier', + default=conf.CELERYD_PREFETCH_MULTIPLIER, type='int'), Option('--queues', '-Q', default=[]), Option('--exclude-queues', '-X', default=[]), Option('--include', '-I', default=[]), From 5392550384986e4232a4d9f3d0170bfb45ea3b29 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 12 Feb 2015 21:23:58 +0000 Subject: [PATCH 0451/1103] 2015 --- LICENSE | 3 ++- celery/__init__.py | 3 ++- docs/conf.py | 2 +- docs/copyright.rst | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/LICENSE b/LICENSE index aeb3da0c0..736d82a97 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,6 @@ -Copyright (c) 2009, 2010, 2011, 2012 Ask Solem, and individual contributors. All Rights Reserved. +Copyright (c) 2015 Ask Solem & contributors. All rights reserved. Copyright (c) 2012-2014 GoPivotal, Inc. All rights reserved. +Copyright (c) 2009, 2010, 2011, 2012 Ask Solem, and individual contributors. All rights reserved. Celery is licensed under The BSD License (3 Clause, also known as the new BSD license). The license is an OSI approved Open Source diff --git a/celery/__init__.py b/celery/__init__.py index 67355fbb5..1fc03e81a 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- """Distributed Task Queue""" +# :copyright: (c) 2015 Ask Solem. All rights reserved. +# :copyright: (c) 2012-2014 GoPivotal, Inc., All rights reserved. # :copyright: (c) 2009 - 2012 Ask Solem and individual contributors, # All rights reserved. -# :copyright: (c) 2012-2014 GoPivotal, Inc., All rights reserved. # :license: BSD (3 Clause), see LICENSE for more details. from __future__ import absolute_import, print_function, unicode_literals diff --git a/docs/conf.py b/docs/conf.py index efd7ea795..c23728e83 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -54,7 +54,7 @@ def linkcode_resolve(domain, info): # General information about the project. project = 'Celery' -copyright = '2009-2014, Ask Solem & Contributors' +copyright = '2009-2015, Ask Solem & Contributors' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/docs/copyright.rst b/docs/copyright.rst index bfffb3019..cf2885186 100644 --- a/docs/copyright.rst +++ b/docs/copyright.rst @@ -7,7 +7,7 @@ by Ask Solem .. |copy| unicode:: U+000A9 .. COPYRIGHT SIGN -Copyright |copy| 2009-2014, Ask Solem. +Copyright |copy| 2009-2015, Ask Solem. All rights reserved. This material may be copied or distributed only subject to the terms and conditions set forth in the `Creative Commons From 1c6ebe30bc2f41635f39e19584d282b7ceded0d7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 12 Feb 2015 21:53:20 +0000 Subject: [PATCH 0452/1103] Wording --- docs/userguide/tasks.rst | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 0ccb956b4..139e204e8 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -1548,15 +1548,19 @@ depending on state from the current transaction*: transaction.commit() expand_abbreviations.delay(article.pk) -Note that Django 1.6 and later enable autocommit mode by default -(deprecating `commit_on_success` and `commit_manually`), automatically -wrapping each SQL query in its own transaction, avoiding the race -condition by default and making it less likely that you'll encounter -the above problem. However, enabling `ATOMIC_REQUESTS` on the database -connection will bring back the transaction per request model and the -race condition along with it. In this case, the simplest solution is -just to use the `@transaction.non_atomic_requests` to switch it back -to autocommit for that view. +.. note:: + Django 1.6 (and later) now enables autocommit mode by default, + and ``commit_on_success``/``commit_manually`` are depreacated. + + This means each SQL query is wrapped and executed in individual + transactions, making it less likely to experience the + problem described above. + + However, enabling ``ATOMIC_REQUESTS`` on the database + connection will bring back the transaction-per-request model and the + race condition along with it. In this case, the simple solution is + using the ``@transaction.non_atomic_requests`` decorator to go back + to autocommit for that view only. .. _task-example: From 663486b213b57104c9bdce90ea8c1dbf15dcafb7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 12 Feb 2015 21:54:46 +0000 Subject: [PATCH 0453/1103] Attempt to fix pypy tests --- celery/utils/functional.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 83b5ba29c..7cdb71671 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -25,6 +25,8 @@ 'first', 'firstmethod', 'chunks', 'padlist', 'mattrgetter', 'uniq', 'regen', 'dictfilter', 'lazy', 'maybe_evaluate', 'head_from_fun'] +IS_PYPY = hasattr(sys, 'pypy_version_info') + KEYWORD_MARK = object() FUNHEAD_TEMPLATE = """ @@ -33,6 +35,15 @@ def {fun_name}({fun_args}): """ +class DummyContext(object): + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + pass + + class LRUCache(UserDict): """LRU Cache implementation using a doubly linked list to track access. @@ -81,12 +92,15 @@ def _iterate_items(self): pass iteritems = _iterate_items - def _iterate_values(self): - for k in self: - try: - yield self.data[k] - except KeyError: # pragma: no cover - pass + def _iterate_values(self, _need_lock=IS_PYPY): + ctx = self.mutex if _need_lock else DummyContext() + with ctx: + for k in self: + try: + yield self.data[k] + except KeyError: # pragma: no cover + pass + itervalues = _iterate_values def _iterate_keys(self): From 102ee8ff0e9ac548372e06fd7a1680ed6f93419c Mon Sep 17 00:00:00 2001 From: John Anderson Date: Fri, 2 Jan 2015 14:31:03 -0800 Subject: [PATCH 0454/1103] Added sontek (John Anderson) to contributors Conflicts: CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 65fb14ff0..44a31f8e3 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -176,4 +176,5 @@ Nathan Van Gheem, 2014/10/28 Gino Ledesma, 2014/10/28 Thomas French, 2014/11/10 Michael Permana, 2014/11/6 +John Anderson, 2014/12/27 Luke Burden, 2015/01/24 From 31109051a1bccee6a5ae8ef6401fdd92738ddbbb Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 12 Feb 2015 22:14:56 +0000 Subject: [PATCH 0455/1103] Sphinx build now requires billiard --- requirements/docs.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/docs.txt b/requirements/docs.txt index 70028e681..e9da93cb3 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,3 +1,4 @@ +billiard Sphinx SQLAlchemy https://github.com/celery/py-amqp/zipball/master From 8dde1c7d0fa23f35dbf05c2b7dcfb0c81c084790 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 12 Feb 2015 22:15:46 +0000 Subject: [PATCH 0456/1103] IRC: Report builds only on first failure --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 867986b15..eae9ac385 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,4 +24,4 @@ notifications: channels: - "chat.freenode.net#celery" on_success: change - on_failure: always + on_failure: change From 43c2e7deab39bc267102b27142b6aca1e50d5579 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 12 Feb 2015 22:16:01 +0000 Subject: [PATCH 0457/1103] Another attempt at fixing pypy tests --- celery/tests/utils/test_functional.py | 2 +- celery/utils/functional.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/celery/tests/utils/test_functional.py b/celery/tests/utils/test_functional.py index 79085417c..e564a4120 100644 --- a/celery/tests/utils/test_functional.py +++ b/celery/tests/utils/test_functional.py @@ -79,7 +79,7 @@ def __init__(self, cache): def run(self): while not self.__is_shutdown.isSet(): try: - self.cache.data.popitem(last=False) + self.cache.popitem(last=False) except KeyError: break self.__is_stopped.set() diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 7cdb71671..afee84d11 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -74,6 +74,12 @@ def update(self, *args, **kwargs): for item in islice(iter(data), len(data) - limit): data.pop(item) + def popitem(self, last=True, _needs_lock=IS_PYPY): + if not _needs_lock: + return self.data.popitem(last) + with self.mutex: + return self.data.popitem(last) + def __setitem__(self, key, value): # remove least recently used key. with self.mutex: From 25178208dcedce742604bb181dcb05bfb1506882 Mon Sep 17 00:00:00 2001 From: Bert Vanderbauwhede Date: Fri, 19 Dec 2014 09:15:26 +0100 Subject: [PATCH 0458/1103] Update CONTRIBUTORS.txt As requested. Conflicts: CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 44a31f8e3..18617379c 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -176,5 +176,6 @@ Nathan Van Gheem, 2014/10/28 Gino Ledesma, 2014/10/28 Thomas French, 2014/11/10 Michael Permana, 2014/11/6 +Bert Vanderbauwhede, 2014/12/18 John Anderson, 2014/12/27 Luke Burden, 2015/01/24 From 63f6c9826f22bc2757b4b7674b15838d4554c7f2 Mon Sep 17 00:00:00 2001 From: PMickael Date: Thu, 12 Feb 2015 23:15:45 +0100 Subject: [PATCH 0459/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 18617379c..574f43919 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -179,3 +179,4 @@ Michael Permana, 2014/11/6 Bert Vanderbauwhede, 2014/12/18 John Anderson, 2014/12/27 Luke Burden, 2015/01/24 +Mickaël Penhard, 2015/02/15 From ac2512b021f6a873fff8b2b1a5b0f74d33df8ce2 Mon Sep 17 00:00:00 2001 From: Mark Parncutt Date: Mon, 16 Feb 2015 17:20:30 +1100 Subject: [PATCH 0460/1103] Allow scheduling according to sunrise, sunset, dawn and dusk --- CONTRIBUTORS.txt | 1 + celery/schedules.py | 149 +++++++++++++++++++++++++++++- docs/AUTHORS.txt | 1 + docs/userguide/periodic-tasks.rst | 102 ++++++++++++++++++++ 4 files changed, 251 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 574f43919..40342b473 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -180,3 +180,4 @@ Bert Vanderbauwhede, 2014/12/18 John Anderson, 2014/12/27 Luke Burden, 2015/01/24 Mickaël Penhard, 2015/02/15 +Mark Parncutt, 2015/02/16 diff --git a/celery/schedules.py b/celery/schedules.py index be6832151..9a50d6935 100644 --- a/celery/schedules.py +++ b/celery/schedules.py @@ -22,12 +22,12 @@ from .utils import is_iterable from .utils.timeutils import ( weekday, maybe_timedelta, remaining, humanize_seconds, - timezone, maybe_make_aware, ffwd + timezone, maybe_make_aware, ffwd, localize ) from .datastructures import AttributeDict __all__ = ['ParseException', 'schedule', 'crontab', 'crontab_parser', - 'maybe_schedule'] + 'maybe_schedule', 'solar'] schedstate = namedtuple('schedstate', ('is_due', 'next')) @@ -591,3 +591,148 @@ def maybe_schedule(s, relative=False, app=None): else: s.app = app return s + +SOLAR_INVALID_LATITUDE = """\ +Argument latitude {lat} is invalid, must be between -90 and 90.\ +""" + +SOLAR_INVALID_LONGITUDE = """\ +Argument longitude {lon} is invalid, must be between -180 and 180.\ +""" + +SOLAR_INVALID_EVENT = """\ +Argument event \"{event}\" is invalid, must be one of {all_events}.\ +""" + +class solar(schedule): + """A solar event can be used as the `run_every` value of a + :class:`PeriodicTask` to schedule based on certain solar events. + + :param event: Solar event that triggers this task. Available + values are: dawn_astronomical, dawn_nautical, dawn_civil, + sunrise, solar_noon, sunset, dusk_civil, dusk_nautical, + dusk_astronomical + :param lat: The latitude of the observer. + :param lon: The longitude of the observer. + :param nowfun: Function returning the current date and time + (class:`~datetime.datetime`). + :param app: Celery app instance. + """ + + + _all_events = ['dawn_astronomical', + 'dawn_nautical', + 'dawn_civil', + 'sunrise', + 'solar_noon', + 'sunset', + 'dusk_civil', + 'dusk_nautical', + 'dusk_astronomical'] + _horizons = {'dawn_astronomical': '-18', + 'dawn_nautical': '-12', + 'dawn_civil': '-6', + 'sunrise': '-0:34', + 'solar_noon': '0', + 'sunset': '-0:34', + 'dusk_civil': '-6', + 'dusk_nautical': '-12', + 'dusk_astronomical': '18'} + _methods = {'dawn_astronomical': 'next_rising', + 'dawn_nautical': 'next_rising', + 'dawn_civil': 'next_rising', + 'sunrise': 'next_rising', + 'solar_noon': 'next_transit', + 'sunset': 'next_setting', + 'dusk_civil': 'next_setting', + 'dusk_nautical': 'next_setting', + 'dusk_astronomical': 'next_setting'} + _use_center_l = {'dawn_astronomical': True, + 'dawn_nautical': True, + 'dawn_civil': True, + 'sunrise': False, + 'solar_noon': True, + 'sunset': False, + 'dusk_civil': True, + 'dusk_nautical': True, + 'dusk_astronomical': True} + + def __init__(self, event, lat, lon, nowfun=None, app=None): + self.ephem = __import__('ephem') + self.event = event + self.lat = lat + self.lon = lon + self.nowfun = nowfun + self._app = app + + if event not in self._all_events: + raise ValueError(SOLAR_INVALID_EVENT.format(event=event, all_events=', '.join(self._all_events))) + if lat < -90 or lat > 90: + raise ValueError(SOLAR_INVALID_LATITUDE.format(lat=lat)) + if lon < -180 or lon > 180: + raise ValueError(SOLAR_INVALID_LONGITUDE.format(lon=lon)) + + cal = self.ephem.Observer() + cal.lat = str(lat) + cal.lon = str(lon) + cal.elev = 0 + cal.horizon = self._horizons[event] + cal.pressure = 0 + self.cal = cal + + self.method = self._methods[event] + self.use_center = self._use_center_l[event] + + def now(self): + return (self.nowfun or self.app.now)() + + def __reduce__(self): + return (self.__class__, (self.event, + self.lat, + self.lon), None) + + def __repr__(self): + return "" + + def remaining_estimate(self, last_run_at): + """Returns when the periodic task should run next as a timedelta, + or if it shouldn't run today (e.g. the sun does not rise today), + returns the time when the next check should take place.""" + last_run_at = self.maybe_make_aware(last_run_at) + last_run_at_utc = localize(last_run_at, timezone.utc) + self.cal.date = last_run_at_utc + try: + next_utc = getattr(self.cal, self.method)(self.ephem.Sun(), start=last_run_at_utc, use_center=self.use_center) + except self.ephem.CircumpolarError: + """Sun will not rise/set today. Check again tomorrow + (specifically, after the next anti-transit).""" + next_utc = self.cal.next_antitransit(self.ephem.Sun()) + timedelta(minutes=1) + next = self.maybe_make_aware(next_utc.datetime()) + now = self.maybe_make_aware(self.now()) + delta = next - now + return delta + + def is_due(self, last_run_at): + """Returns tuple of two items `(is_due, next_time_to_run)`, + where next time to run is in seconds. + + See :meth:`celery.schedules.schedule.is_due` for more information. + + """ + rem_delta = self.remaining_estimate(last_run_at) + rem = max(rem_delta.total_seconds(), 0) + due = rem == 0 + if due: + rem_delta = self.remaining_estimate(self.now()) + rem = max(rem_delta.total_seconds(), 0) + return schedstate(due, rem) + + def __eq__(self, other): + if isinstance(other, solar): + return (other.event == self.event and + other.lat == self.lat and + other.lon == self.lon) + return NotImplemented + + def __ne__(self, other): + return not self.__eq__(other) diff --git a/docs/AUTHORS.txt b/docs/AUTHORS.txt index 5c4f055db..8ff42cbbb 100644 --- a/docs/AUTHORS.txt +++ b/docs/AUTHORS.txt @@ -89,6 +89,7 @@ Marcin Kuźmiński Marcin Lulek Mark Hellewell Mark Lavin +Mark Parncutt Mark Stover Mark Thurman Martin Galpin diff --git a/docs/userguide/periodic-tasks.rst b/docs/userguide/periodic-tasks.rst index d7ae86f95..b6804dd4e 100644 --- a/docs/userguide/periodic-tasks.rst +++ b/docs/userguide/periodic-tasks.rst @@ -269,6 +269,108 @@ The syntax of these crontab expressions are very flexible. Some examples: See :class:`celery.schedules.crontab` for more documentation. +.. _beat-solar: + +Solar schedules +================= + +If you have a task that should be executed according to sunrise, +sunset, dawn or dusk, you can use the +:class:`~celery.schedules.solar` schedule type: + +.. code-block:: python + + from celery.schedules import solar + + CELERYBEAT_SCHEDULE = { + # Executes at sunset in Melbourne + 'add-at-melbourne-sunset': { + 'task': 'tasks.add', + 'schedule': solar('sunset', -37.81753, 144.96715), + 'args': (16, 16), + }, + } + +The arguments are simply: ``solar(event, latitude, longitude)`` + +Be sure to use the correct sign for latitude and longitude: + ++---------------+-------------------+----------------------+ +| **Sign** | **Argument** | **Meaning** | ++---------------+-------------------+----------------------+ +| ``+`` | ``latitude`` | North | ++---------------+-------------------+----------------------+ +| ``-`` | ``latitude`` | South | ++---------------+-------------------+----------------------+ +| ``+`` | ``longitude`` | East | ++---------------+-------------------+----------------------+ +| ``-`` | ``longitude`` | West | ++---------------+-------------------+----------------------+ + +Possible event types are: + ++-----------------------------------------+--------------------------------------------+ +| **Event** | **Meaning** | ++-----------------------------------------+--------------------------------------------+ +| ``dawn_astronomical`` | Execute at the moment after which the sky | +| | is no longer completely dark. This is when | +| | the sun is 18 degrees below the horizon. | ++-----------------------------------------+--------------------------------------------+ +| ``dawn_nautical`` | Execute when there is enough sunlight for | +| | the horizon and some objects to be | +| | distinguishable; formally, when the sun is | +| | 12 degrees below the horizon. | ++-----------------------------------------+--------------------------------------------+ +| ``dawn_civil`` | Execute when there is enough light for | +| | objects to be distinguishable so that | +| | outdoor activities can commence; | +| | formally, when the Sun is 6 degrees below | +| | the horizon. | ++-----------------------------------------+--------------------------------------------+ +| ``sunrise`` | Execute when the upper edge of the sun | +| | appears over the eastern horizon in the | +| | morning. | ++-----------------------------------------+--------------------------------------------+ +| ``solar_noon`` | Execute when the sun is highest above the | +| | horizon on that day. | ++-----------------------------------------+--------------------------------------------+ +| ``sunset`` | Execute when the trailing edge of the sun | +| | disappears over the western horizon in the | +| | evening. | ++-----------------------------------------+--------------------------------------------+ +| ``dusk_civil`` | Execute at the end of civil twilight, when | +| | objects are still distinguishable and some | +| | stars and planets are visible. Formally, | +| | when the sun is 6 degrees below the | +| | horizon. | ++-----------------------------------------+--------------------------------------------+ +| ``dusk_nautical`` | Execute when the sun is 12 degrees below | +| | the horizon. Objects are no longer | +| | distinguishable, and the horizon is no | +| | longer visible to the naked eye. | ++-----------------------------------------+--------------------------------------------+ +| ``dusk_astronomical`` | Execute at the moment after which the sky | +| | becomes completely dark; formally, when | +| | the sun is 18 degrees below the horizon. | ++-----------------------------------------+--------------------------------------------+ + +All solar events are calculated using UTC, and are therefore +unaffected by your timezone setting. + +In polar regions, the sun may not rise or set every day. The scheduler +is able to handle these cases, i.e. a ``sunrise`` event won't run on a day +when the sun doesn't rise. The one exception is ``solar_noon``, which is +formally defined as the moment the sun transits the celestial meridian, +and will occur every day even if the sun is below the horizon. + +Twilight is defined as the period between dawn and sunrise, and between +sunset and dusk. You can schedule an event according to "twilight" +depending on your definition of twilight (civil, nautical or astronomical), +and whether you want the event to take place at the beginning or end +of twilight, using the appropriate event from the list above. + +See :class:`celery.schedules.solar` for more documentation. + .. _beat-starting: Starting the Scheduler From 80150fb6bd4f4b7c4b014846061541cd62e74e94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A8=B1=E9=82=B1=E7=BF=94?= Date: Mon, 16 Feb 2015 16:59:13 +0800 Subject: [PATCH 0461/1103] Update Celery on FreeBSD in FAQ --- docs/faq.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index ae82a216a..86ae18396 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -281,8 +281,9 @@ When using the RabbitMQ (AMQP) and Redis transports it should work out of the box. For other transports the compatibility prefork pool is -used which requires a working POSIX semaphore implementation, and this isn't -enabled in FreeBSD by default. You have to enable +used which requires a working POSIX semaphore implementation, +this is enabled in FreeBSD by default since FreeBSD 8.x. +For older version of FreeBSD, you have to enable POSIX semaphores in the kernel and manually recompile billiard. Luckily, Viktor Petersson has written a tutorial to get you started with From fd90aeea1a9c0370041b1dc1080d9f8218e9b909 Mon Sep 17 00:00:00 2001 From: Ken Reese Date: Fri, 27 Feb 2015 12:24:22 -0700 Subject: [PATCH 0462/1103] Update configuration.rst Fixes a minor spelling error in the documentation. --- docs/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index ee599af3a..e1b032924 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1061,7 +1061,7 @@ manner using TCP/IP alone, so AMQP defines something called heartbeats that's is used both by the client and the broker to detect if a connection was closed. -Hartbeats are disabled by default. +Heartbeats are disabled by default. If the heartbeat value is 10 seconds, then the heartbeat will be monitored at the interval specified From 0331cb726605a8cd859cabfae7be60872de624e0 Mon Sep 17 00:00:00 2001 From: Wil Langford Date: Fri, 27 Feb 2015 20:53:19 -0800 Subject: [PATCH 0463/1103] Removes what looks like a copy/paste artifact in batches.py. --- celery/contrib/batches.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/celery/contrib/batches.py b/celery/contrib/batches.py index da04c0577..ad41c1903 100644 --- a/celery/contrib/batches.py +++ b/celery/contrib/batches.py @@ -197,20 +197,20 @@ def Strategy(self, task, app, consumer): flush_buffer = self._do_flush def task_message_handler(message, body, ack, reject, callbacks, **kw): - if body is None: 31513 ? S 125:09 /usr/bin/python -m celery worker --without-heartbeat -c 50 --pool=eventlet -n celery6@ns326150.ip-37-187-158.eu --app=mai - body, headers, decoded, utc = ( n -Q rss --without-gossip --logfile=/home/logs/rss.log --pidfile=celery6.pid - message.body, message.headers, False, True, 31528 ? R 128:34 /usr/bin/python -m celery worker --without-heartbeat -c 50 --pool=eventlet -n celery7@ns326150.ip-37-187-158.eu --app=mai - ) n -Q rss --without-gossip --logfile=/home/logs/rss.log --pidfile=celery7.pid - if not body_can_be_buffer: 31543 ? S 124:32 /usr/bin/python -m celery worker --without-heartbeat -c 50 --pool=eventlet -n celery8@ns326150.ip-37-187-158.eu --app=mai - body = bytes(body) if isinstance(body, buffer_t) else body n -Q rss --without-gossip --logfile=/home/logs/rss.log --pidfile=celery8.pid - else: 26150 ? S 0:50 /usr/bin/python -m celery worker --without-heartbeat -c 2 --pool=eventlet -n engines@ns326150.ip-37-187-158.eu --app=main - body, headers, decoded, utc = proto1_to_proto2(message, body) -Q engines --without-gossip --logfile=/home/logs/engines.log --pidfile=/home/logs/pid-engines.pid - 22409 ? S 0:00 /usr/bin/python -m celery worker --without-heartbeat -c 1 -n elasticsearch_bulk_actions@ns326150.ip-37-187-158.eu --app=m - request = Req( ain -Q elasticsearch_bulk_actions --without-gossip --logfile=/home/logs/elasticsearch_bulk_actions.log --pidfile=elasticsearch_bulk_actions.pid - message, 22459 ? S 0:00 \_ /usr/bin/python -m celery worker --without-heartbeat -c 1 -n elasticsearch_bulk_actions@ns326150.ip-37-187-158.eu --a - on_ack=ack, on_reject=reject, app=app, hostname=hostname, pp=main -Q elasticsearch_bulk_actions --without-gossip --logfile=/home/logs/elasticsearch_bulk_actions.log --pidfile=elasticsearch_bulk_actions.pid - eventer=eventer, task=task, connection_errors=connection_errors, 22419 ? S 0:00 /usr/bin/python -m celery worker --without-heartbeat -c 1 -n celery@ns326150.ip-37-187-158.eu --app=main -Q elasticsearch - body=body, headers=headers, decoded=decoded, utc=utc, _bulk_actions --without-gossip --logfile=/home/logs/elasticsearch_bulk_actions.log --pidfile=celery.pid + if body is None: + body, headers, decoded, utc = ( + message.body, message.headers, False, True, + ) + if not body_can_be_buffer: + body = bytes(body) if isinstance(body, buffer_t) else body + else: + body, headers, decoded, utc = proto1_to_proto2(message, body) + + request = Req( + message, + on_ack=ack, on_reject=reject, app=app, hostname=hostname, + eventer=eventer, task=task, connection_errors=connection_errors, + body=body, headers=headers, decoded=decoded, utc=utc, ) put_buffer(request) From dc8923827fcf4701eb2df55e2a574bc24b69ede1 Mon Sep 17 00:00:00 2001 From: Cullen Rhodes Date: Fri, 13 Mar 2015 10:45:16 +0000 Subject: [PATCH 0464/1103] Fixed broken Pylons link in README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a6c753ce7..464d5da02 100644 --- a/README.rst +++ b/README.rst @@ -185,7 +185,7 @@ development easier, and sometimes they add important hooks like closing database connections at ``fork``. .. _`Django`: http://djangoproject.com/ -.. _`Pylons`: http://pylonshq.com/ +.. _`Pylons`: http://www.pylonsproject.org/ .. _`Flask`: http://flask.pocoo.org/ .. _`web2py`: http://web2py.com/ .. _`Bottle`: http://bottlepy.org/ From 5005b5a3fdb1e883bcbaf29b7afbeb1749992927 Mon Sep 17 00:00:00 2001 From: Bence Tamas Date: Thu, 19 Mar 2015 16:34:33 +0100 Subject: [PATCH 0465/1103] Fix TypeError raised by Django's SystemCheck --- celery/fixups/django.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index d38b6f195..3db7f4b65 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -156,9 +156,10 @@ def validate_models(self): try: from django.core.management.validation import get_validation_errors except ImportError: - from django.core.management.base import BaseCommand + from django.core.management.base import BaseCommand, OutputWrapper cmd = BaseCommand() - cmd.stdout, cmd.stderr = sys.stdout, sys.stderr + cmd.stdout = OutputWrapper(sys.stdout) + cmd.stderr = OutputWrapper(sys.stderr) cmd.check() else: num_errors = get_validation_errors(s, None) From 38eae2697198d2d795c838a358a951702674847a Mon Sep 17 00:00:00 2001 From: PMickael Date: Thu, 19 Mar 2015 18:35:50 +0100 Subject: [PATCH 0466/1103] Remove unnecessary space --- celery/schedules.py | 14 +++++++------- docs/userguide/periodic-tasks.rst | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/celery/schedules.py b/celery/schedules.py index 9a50d6935..917a8e2d7 100644 --- a/celery/schedules.py +++ b/celery/schedules.py @@ -613,13 +613,13 @@ class solar(schedule): sunrise, solar_noon, sunset, dusk_civil, dusk_nautical, dusk_astronomical :param lat: The latitude of the observer. - :param lon: The longitude of the observer. + :param lon: The longitude of the observer. :param nowfun: Function returning the current date and time (class:`~datetime.datetime`). :param app: Celery app instance. """ - + _all_events = ['dawn_astronomical', 'dawn_nautical', 'dawn_civil', @@ -656,7 +656,7 @@ class solar(schedule): 'dusk_civil': True, 'dusk_nautical': True, 'dusk_astronomical': True} - + def __init__(self, event, lat, lon, nowfun=None, app=None): self.ephem = __import__('ephem') self.event = event @@ -664,14 +664,14 @@ def __init__(self, event, lat, lon, nowfun=None, app=None): self.lon = lon self.nowfun = nowfun self._app = app - + if event not in self._all_events: raise ValueError(SOLAR_INVALID_EVENT.format(event=event, all_events=', '.join(self._all_events))) if lat < -90 or lat > 90: raise ValueError(SOLAR_INVALID_LATITUDE.format(lat=lat)) if lon < -180 or lon > 180: raise ValueError(SOLAR_INVALID_LONGITUDE.format(lon=lon)) - + cal = self.ephem.Observer() cal.lat = str(lat) cal.lon = str(lon) @@ -679,13 +679,13 @@ def __init__(self, event, lat, lon, nowfun=None, app=None): cal.horizon = self._horizons[event] cal.pressure = 0 self.cal = cal - + self.method = self._methods[event] self.use_center = self._use_center_l[event] def now(self): return (self.nowfun or self.app.now)() - + def __reduce__(self): return (self.__class__, (self.event, self.lat, diff --git a/docs/userguide/periodic-tasks.rst b/docs/userguide/periodic-tasks.rst index b6804dd4e..a1546bdf5 100644 --- a/docs/userguide/periodic-tasks.rst +++ b/docs/userguide/periodic-tasks.rst @@ -281,12 +281,12 @@ sunset, dawn or dusk, you can use the .. code-block:: python from celery.schedules import solar - + CELERYBEAT_SCHEDULE = { # Executes at sunset in Melbourne 'add-at-melbourne-sunset': { 'task': 'tasks.add', - 'schedule': solar('sunset', -37.81753, 144.96715), + 'schedule': solar('sunset', -37.81753, 144.96715), 'args': (16, 16), }, } From 8a2c3b18a754b13b8458c71bd976d1d74a0cbbb4 Mon Sep 17 00:00:00 2001 From: PMickael Date: Fri, 20 Mar 2015 11:20:32 +0100 Subject: [PATCH 0467/1103] OutputWrapper only available since django 1.5 --- celery/fixups/django.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index 3db7f4b65..eb5c8d235 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -156,10 +156,16 @@ def validate_models(self): try: from django.core.management.validation import get_validation_errors except ImportError: - from django.core.management.base import BaseCommand, OutputWrapper + from django.core.management.base import BaseCommand cmd = BaseCommand() - cmd.stdout = OutputWrapper(sys.stdout) - cmd.stderr = OutputWrapper(sys.stderr) + try: + # since django 1.5 + from django.core.management.base import OutputWrapper + cmd.stdout = OutputWrapper(sys.stdout) + cmd.stderr = OutputWrapper(sys.stderr) + except ImportError: + cmd.stdout, cmd.stderr = sys.stdout, sys.stderr + cmd.check() else: num_errors = get_validation_errors(s, None) From 357939d802f7c7b1adf9499b3727c932c7a0953b Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 22 Mar 2015 19:50:47 +0100 Subject: [PATCH 0468/1103] Fix typo in task docs --- docs/userguide/tasks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 993cc6eb3..aeb5077eb 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -1550,7 +1550,7 @@ depending on state from the current transaction*: .. note:: Django 1.6 (and later) now enables autocommit mode by default, - and ``commit_on_success``/``commit_manually`` are depreacated. + and ``commit_on_success``/``commit_manually`` are deprecated. This means each SQL query is wrapped and executed in individual transactions, making it less likely to experience the From 27f85e786c34d7445e054d0418f15c7de7933bdf Mon Sep 17 00:00:00 2001 From: Ilya Georgievsky Date: Mon, 23 Mar 2015 17:38:54 +0300 Subject: [PATCH 0469/1103] on_message callback added --- celery/backends/amqp.py | 8 +++++--- celery/result.py | 14 +++++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index 5111d5936..596a4c667 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -231,7 +231,7 @@ def consume(self, task_id, timeout=None, no_ack=True, on_interval=None): def _many_bindings(self, ids): return [self._create_binding(task_id) for task_id in ids] - def get_many(self, task_ids, timeout=None, no_ack=True, + def get_many(self, task_ids, timeout=None, no_ack=True, on_message=None, now=monotonic, getfields=itemgetter('status', 'task_id'), READY_STATES=states.READY_STATES, PROPAGATE_STATES=states.PROPAGATE_STATES, **kwargs): @@ -254,15 +254,17 @@ def get_many(self, task_ids, timeout=None, no_ack=True, push_cache = self._cache.__setitem__ decode_result = self.meta_from_decoded - def on_message(message): + def _on_message(message): body = decode_result(message.decode()) + if on_message is not None: + on_message(body) state, uid = getfields(body) if state in READY_STATES: push_result(body) \ if uid in task_ids else push_cache(uid, body) bindings = self._many_bindings(task_ids) - with self.Consumer(channel, bindings, on_message=on_message, + with self.Consumer(channel, bindings, on_message=_on_message, accept=self.accept, no_ack=no_ack): wait = conn.drain_events popleft = results.popleft diff --git a/celery/result.py b/celery/result.py index 3784547f0..221f1c7f9 100644 --- a/celery/result.py +++ b/celery/result.py @@ -567,7 +567,7 @@ def iterate(self, timeout=None, propagate=True, interval=0.5): raise TimeoutError('The operation timed out') def get(self, timeout=None, propagate=True, interval=0.5, - callback=None, no_ack=True): + callback=None, no_ack=True, on_message=None): """See :meth:`join` This is here for API compatibility with :class:`AsyncResult`, @@ -577,7 +577,7 @@ def get(self, timeout=None, propagate=True, interval=0.5, """ return (self.join_native if self.supports_native_join else self.join)( timeout=timeout, propagate=propagate, - interval=interval, callback=callback, no_ack=no_ack) + interval=interval, callback=callback, no_ack=no_ack, on_message=on_message) def join(self, timeout=None, propagate=True, interval=0.5, callback=None, no_ack=True): @@ -649,7 +649,8 @@ def join(self, timeout=None, propagate=True, interval=0.5, results.append(value) return results - def iter_native(self, timeout=None, interval=0.5, no_ack=True): + def iter_native(self, timeout=None, interval=0.5, no_ack=True, + on_message=None): """Backend optimized version of :meth:`iterate`. .. versionadded:: 2.2 @@ -667,10 +668,12 @@ def iter_native(self, timeout=None, interval=0.5, no_ack=True): return self.backend.get_many( set(r.id for r in results), timeout=timeout, interval=interval, no_ack=no_ack, + on_message=on_message, ) def join_native(self, timeout=None, propagate=True, - interval=0.5, callback=None, no_ack=True): + interval=0.5, callback=None, no_ack=True, + on_message=None): """Backend optimized version of :meth:`join`. .. versionadded:: 2.2 @@ -687,7 +690,8 @@ def join_native(self, timeout=None, propagate=True, result.id: i for i, result in enumerate(self.results) } acc = None if callback else [None for _ in range(len(self))] - for task_id, meta in self.iter_native(timeout, interval, no_ack): + for task_id, meta in self.iter_native(timeout, interval, no_ack, + on_message): value = meta['result'] if propagate and meta['status'] in states.PROPAGATE_STATES: raise value From e362cde9984920f15ab61b380b34856dddb89761 Mon Sep 17 00:00:00 2001 From: Ilya Georgievsky Date: Mon, 23 Mar 2015 18:57:49 +0300 Subject: [PATCH 0470/1103] fix build error --- celery/result.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/celery/result.py b/celery/result.py index 221f1c7f9..2524c2291 100644 --- a/celery/result.py +++ b/celery/result.py @@ -580,7 +580,7 @@ def get(self, timeout=None, propagate=True, interval=0.5, interval=interval, callback=callback, no_ack=no_ack, on_message=on_message) def join(self, timeout=None, propagate=True, interval=0.5, - callback=None, no_ack=True): + callback=None, no_ack=True, on_message=None): """Gathers the results of all tasks as a list in order. .. note:: @@ -632,6 +632,9 @@ def join(self, timeout=None, propagate=True, interval=0.5, time_start = monotonic() remaining = None + if on_message is not None: + raise Exception('Your backend not suppored on_message callback') + results = [] for result in self.results: remaining = None From 4ad7533a9bae2dae2e510b42c7fac3149f70c671 Mon Sep 17 00:00:00 2001 From: Ilya Georgievsky Date: Mon, 23 Mar 2015 19:17:15 +0300 Subject: [PATCH 0471/1103] test ci fix --- requirements/test-ci.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/requirements/test-ci.txt b/requirements/test-ci.txt index 8385252ae..7c4f39865 100644 --- a/requirements/test-ci.txt +++ b/requirements/test-ci.txt @@ -1,7 +1,4 @@ coverage>=3.0 coveralls redis -#riak >=2.0 -#pymongo -#SQLAlchemy PyOpenSSL From 5d73c6d85a777080122cd09fc2f6d064eef6eb88 Mon Sep 17 00:00:00 2001 From: Ori Hoch Date: Tue, 24 Mar 2015 11:59:20 +0200 Subject: [PATCH 0472/1103] change method for detection if process was killed --- extra/generic-init.d/celerybeat | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/extra/generic-init.d/celerybeat b/extra/generic-init.d/celerybeat index fb31ca292..85785caa5 100755 --- a/extra/generic-init.d/celerybeat +++ b/extra/generic-init.d/celerybeat @@ -202,14 +202,17 @@ create_paths () { create_default_dir "$CELERYBEAT_PID_DIR" } +is_running() { + pid=$1 + ps $pid > /dev/null 2>&1 +} wait_pid () { pid=$1 forever=1 i=0 while [ $forever -gt 0 ]; do - kill -0 $pid 1>/dev/null 2>&1 - if [ $? -eq 1 ]; then + if ! is_running $pid; then echo "OK" forever=0 else From 665940624edb0f9ccd5cd6b2a28b68cdf85520ef Mon Sep 17 00:00:00 2001 From: samjy Date: Tue, 24 Mar 2015 22:55:09 +0100 Subject: [PATCH 0473/1103] Improve use of uri in mongo backend - use database name, user, password from the uri if provided - uri configurations can be overwritten in CELERY_MONGODB_BACKEND_SETTINGS --- celery/backends/mongodb.py | 79 ++++++++++++++++++--------- celery/tests/backends/test_mongodb.py | 57 +++++++++++++++++++ 2 files changed, 111 insertions(+), 25 deletions(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index f82c5f559..456ae6034 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -45,6 +45,7 @@ def __init__(self, **kw): class MongoBackend(BaseBackend): + mongo_host = None host = 'localhost' port = 27017 user = None @@ -75,6 +76,28 @@ def __init__(self, app=None, url=None, **kwargs): 'You need to install the pymongo library to use the ' 'MongoDB backend.') + self.url = url + + # default options + self.options.setdefault('max_pool_size', self.max_pool_size) + self.options.setdefault('auto_start_request', False) + + # update conf with mongo uri data, only if uri was given + if self.url: + uri_data = pymongo.uri_parser.parse_uri(self.url) + # build the hosts list to create a mongo connection + make_host_str = lambda x: "{0}:{1}".format(x[0], x[1]) + hostslist = map(make_host_str, uri_data['nodelist']) + self.user = uri_data['username'] + self.password = uri_data['password'] + self.mongo_host = hostslist + if uri_data['database']: + # if no database is provided in the uri, use default + self.database_name = uri_data['database'] + + self.options.update(uri_data['options']) + + # update conf with specific settings config = self.app.conf.get('CELERY_MONGODB_BACKEND_SETTINGS') if config is not None: if not isinstance(config, dict): @@ -82,8 +105,13 @@ def __init__(self, app=None, url=None, **kwargs): 'MongoDB backend settings should be grouped in a dict') config = dict(config) # do not modify original + if 'host' in config or 'port' in config: + # these should take over uri conf + self.mongo_host = None + self.host = config.pop('host', self.host) - self.port = int(config.pop('port', self.port)) + self.port = config.pop('port', self.port) + self.mongo_host = config.pop('mongo_host', self.mongo_host) self.user = config.pop('user', self.user) self.password = config.pop('password', self.password) self.database_name = config.pop('database', self.database_name) @@ -94,37 +122,38 @@ def __init__(self, app=None, url=None, **kwargs): 'groupmeta_collection', self.groupmeta_collection, ) - self.options = dict(config, **config.pop('options', None) or {}) - - # Set option defaults - self.options.setdefault('max_pool_size', self.max_pool_size) - self.options.setdefault('auto_start_request', False) - - self.url = url - if self.url: - # Specifying backend as an URL - self.host = self.url + self.options.update(config.pop('options', {})) + self.options.update(config) def _get_connection(self): """Connect to the MongoDB server.""" if self._connection is None: from pymongo import MongoClient - # The first pymongo.Connection() argument (host) can be - # a list of ['host:port'] elements or a mongodb connection - # URI. If this is the case, don't use self.port - # but let pymongo get the port(s) from the URI instead. - # This enables the use of replica sets and sharding. - # See pymongo.Connection() for more info. - url = self.host - if isinstance(url, string_t) \ - and not url.startswith('mongodb://'): - url = 'mongodb://{0}:{1}'.format(url, self.port) - if url == 'mongodb://': - url = url + 'localhost' + host = self.mongo_host + if not host: + # The first pymongo.Connection() argument (host) can be + # a list of ['host:port'] elements or a mongodb connection + # URI. If this is the case, don't use self.port + # but let pymongo get the port(s) from the URI instead. + # This enables the use of replica sets and sharding. + # See pymongo.Connection() for more info. + host = self.host + if isinstance(host, string_t) \ + and not host.startswith('mongodb://'): + host = 'mongodb://{0}:{1}'.format(host, self.port) + + if host == 'mongodb://': + host += 'localhost' + + # don't change self.options + conf = dict(self.options) + conf['host'] = host + if detect_environment() != 'default': - self.options['use_greenlets'] = True - self._connection = MongoClient(host=url, **self.options) + conf['use_greenlets'] = True + + self._connection = MongoClient(**conf) return self._connection diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index 801da3c1b..7f6597bd0 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -67,6 +67,63 @@ def test_init_settings_is_None(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None MongoBackend(app=self.app) + def test_init_with_settings(self): + self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None + # empty settings + mb = MongoBackend(app=self.app) + + # uri + uri = 'mongodb://localhost:27017' + mb = MongoBackend(app=self.app, url=uri) + self.assertEqual(mb.mongo_host, ['localhost:27017']) + self.assertEqual(mb.options, {'auto_start_request': False, + 'max_pool_size': 10}) + self.assertEqual(mb.database_name, 'celery') + + # uri with database name + uri = 'mongodb://localhost:27017/celerydb' + mb = MongoBackend(app=self.app, url=uri) + self.assertEqual(mb.database_name, 'celerydb') + + # uri with user, password, database name, replica set + uri = ('mongodb://' + 'celeryuser:celerypassword@' + 'mongo1.example.com:27017,' + 'mongo2.example.com:27017,' + 'mongo3.example.com:27017/' + 'celerydatabase?replicaSet=rs0') + mb = MongoBackend(app=self.app, url=uri) + self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017', + 'mongo2.example.com:27017', + 'mongo3.example.com:27017']) + self.assertEqual(mb.options, {'auto_start_request': False, + 'max_pool_size': 10, + 'replicaset': 'rs0'}) + self.assertEqual(mb.user, 'celeryuser') + self.assertEqual(mb.password, 'celerypassword') + self.assertEqual(mb.database_name, 'celerydatabase') + + # same uri, change some parameters in backend settings + self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = { + 'replicaset': 'rs1', + 'user': 'backenduser', + 'database': 'another_db', + 'options': { + 'socketKeepAlive': True, + }, + } + mb = MongoBackend(app=self.app, url=uri) + self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017', + 'mongo2.example.com:27017', + 'mongo3.example.com:27017']) + self.assertEqual(mb.options, {'auto_start_request': False, + 'max_pool_size': 10, + 'replicaset': 'rs1', + 'socketKeepAlive': True}) + self.assertEqual(mb.user, 'backenduser') + self.assertEqual(mb.password, 'celerypassword') + self.assertEqual(mb.database_name, 'another_db') + @depends_on_current_app def test_reduce(self): x = MongoBackend(app=self.app) From 00b394da7232439d1edfb3b979800667a1448c5c Mon Sep 17 00:00:00 2001 From: samjy Date: Thu, 26 Mar 2015 13:38:22 +0100 Subject: [PATCH 0474/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 40342b473..73327e53b 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -181,3 +181,4 @@ John Anderson, 2014/12/27 Luke Burden, 2015/01/24 Mickaël Penhard, 2015/02/15 Mark Parncutt, 2015/02/16 +Samuel Jaillet, 2015/03/24 From 60c11b681e40f11b90bc5ee69917b45c7ceb9fa8 Mon Sep 17 00:00:00 2001 From: Ilya Georgievsky Date: Mon, 30 Mar 2015 20:20:36 +0300 Subject: [PATCH 0475/1103] revert "test ci fix", get_many_on_message test added --- celery/tests/backends/test_amqp.py | 30 ++++++++++++++++++++++++++++++ requirements/test-ci.txt | 3 +++ 2 files changed, 33 insertions(+) diff --git a/celery/tests/backends/test_amqp.py b/celery/tests/backends/test_amqp.py index 6ca5441de..3a17ef41e 100644 --- a/celery/tests/backends/test_amqp.py +++ b/celery/tests/backends/test_amqp.py @@ -294,6 +294,36 @@ def test_get_many(self): b.store_result(tids[0], i, states.PENDING) list(b.get_many(tids, timeout=0.01)) + def test_get_many_on_message(self): + b = self.create_backend(max_cached_results=10) + + tids = [] + for i in range(10): + tid = uuid() + b.store_result(tid, '', states.PENDING) + b.store_result(tid, 'comment_%i_1' % i, states.STARTED) + b.store_result(tid, 'comment_%i_2' % i, states.STARTED) + b.store_result(tid, 'final result %i' % i, states.SUCCESS) + tids.append(tid) + + + expected_messages = {} + for i, _tid in enumerate(tids): + expected_messages[_tid] = [] + expected_messages[_tid].append( (states.PENDING, '') ) + expected_messages[_tid].append( (states.STARTED, 'comment_%i_1' % i) ) + expected_messages[_tid].append( (states.STARTED, 'comment_%i_2' % i) ) + expected_messages[_tid].append( (states.SUCCESS, 'final result %i' % i) ) + + on_message_results = {} + def on_message(body): + if not body['task_id'] in on_message_results: + on_message_results[body['task_id']] = [] + on_message_results[body['task_id']].append( (body['status'], body['result']) ) + + b.get_many(tids, timeout=1, on_message=on_message) + self.assertEqual(sorted(on_message_results), sorted(expected_messages)) + def test_get_many_raises_outer_block(self): class Backend(AMQPBackend): diff --git a/requirements/test-ci.txt b/requirements/test-ci.txt index 7c4f39865..8385252ae 100644 --- a/requirements/test-ci.txt +++ b/requirements/test-ci.txt @@ -1,4 +1,7 @@ coverage>=3.0 coveralls redis +#riak >=2.0 +#pymongo +#SQLAlchemy PyOpenSSL From 2522eb0a6d717b496c04f2d41bbf9c3b0200d9b6 Mon Sep 17 00:00:00 2001 From: Ilya Georgievsky Date: Tue, 31 Mar 2015 12:44:27 +0300 Subject: [PATCH 0476/1103] test fix --- celery/tests/backends/test_amqp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/tests/backends/test_amqp.py b/celery/tests/backends/test_amqp.py index 3a17ef41e..031481c8d 100644 --- a/celery/tests/backends/test_amqp.py +++ b/celery/tests/backends/test_amqp.py @@ -321,7 +321,7 @@ def on_message(body): on_message_results[body['task_id']] = [] on_message_results[body['task_id']].append( (body['status'], body['result']) ) - b.get_many(tids, timeout=1, on_message=on_message) + res = list(b.get_many(tids, timeout=1, on_message=on_message)) self.assertEqual(sorted(on_message_results), sorted(expected_messages)) def test_get_many_raises_outer_block(self): From 053858c267e4770d5e2976d1ab14a73e9d86db75 Mon Sep 17 00:00:00 2001 From: PMickael Date: Tue, 31 Mar 2015 13:03:29 +0200 Subject: [PATCH 0477/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 73327e53b..a0bcbd34a 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -182,3 +182,4 @@ Luke Burden, 2015/01/24 Mickaël Penhard, 2015/02/15 Mark Parncutt, 2015/02/16 Samuel Jaillet, 2015/03/24 +Ilya Georgievsky, 2015/03/31 From d76838ab311f6869ab76354a1127f1aa663c796f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dud=C3=A1s=20=C3=81d=C3=A1m?= Date: Fri, 10 Apr 2015 13:36:21 +0200 Subject: [PATCH 0478/1103] fix typo --- celery/result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/result.py b/celery/result.py index 2524c2291..d40d5fc42 100644 --- a/celery/result.py +++ b/celery/result.py @@ -633,7 +633,7 @@ def join(self, timeout=None, propagate=True, interval=0.5, remaining = None if on_message is not None: - raise Exception('Your backend not suppored on_message callback') + raise Exception('Your backend not supported on_message callback') results = [] for result in self.results: From af3e046f00aa5da2d996ecd14b566f7360e6683d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 13 Apr 2015 16:38:10 +0100 Subject: [PATCH 0479/1103] Fixes infinite recursion when logger_isa receives patched logger object --- celery/utils/log.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/celery/utils/log.py b/celery/utils/log.py index ccb715a6d..778519001 100644 --- a/celery/utils/log.py +++ b/celery/utils/log.py @@ -77,9 +77,9 @@ def in_sighandler(): set_in_sighandler(False) -def logger_isa(l, p): +def logger_isa(l, p, max=1000): this, seen = l, set() - while this: + for _ in range(max): if this == p: return True else: @@ -89,6 +89,10 @@ def logger_isa(l, p): ) seen.add(this) this = this.parent + if not this: + break + else: + raise RuntimeError('Logger hierarchy exceeds {0}'.format(max)) return False From c8ffcc4aea3d548bd81a765d6960318aa5d19d52 Mon Sep 17 00:00:00 2001 From: Wiliam Souza Date: Wed, 15 Apr 2015 14:53:16 -0300 Subject: [PATCH 0480/1103] Changed _apply_chord_incr to use int value --- celery/backends/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/cache.py b/celery/backends/cache.py index 7062a001a..b9480fb31 100644 --- a/celery/backends/cache.py +++ b/celery/backends/cache.py @@ -129,7 +129,7 @@ def delete(self, key): return self.client.delete(key) def _apply_chord_incr(self, header, partial_args, group_id, body, **opts): - self.client.set(self.get_key_for_chord(group_id), '0', time=86400) + self.client.set(self.get_key_for_chord(group_id), 0, time=86400) return super(CacheBackend, self)._apply_chord_incr( header, partial_args, group_id, body, **opts ) From d93bceecbf360bde9096748d1806402f02b366fe Mon Sep 17 00:00:00 2001 From: James Pulec Date: Wed, 15 Apr 2015 14:09:00 -0700 Subject: [PATCH 0481/1103] Correct spelling error --- docs/userguide/tasks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index aeb5077eb..bd23625bf 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -1567,7 +1567,7 @@ depending on state from the current transaction*: Example ======= -Let's take a real wold example; A blog where comments posted needs to be +Let's take a real world example; A blog where comments posted needs to be filtered for spam. When the comment is created, the spam filter runs in the background, so the user doesn't have to wait for it to finish. From 772bcb832b91bdc361beaf3bb775a9f4a481faab Mon Sep 17 00:00:00 2001 From: fatihsucu Date: Thu, 16 Apr 2015 14:31:59 +0300 Subject: [PATCH 0482/1103] mongodb default options removed. --- celery/backends/mongodb.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 456ae6034..624807aad 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -78,9 +78,6 @@ def __init__(self, app=None, url=None, **kwargs): self.url = url - # default options - self.options.setdefault('max_pool_size', self.max_pool_size) - self.options.setdefault('auto_start_request', False) # update conf with mongo uri data, only if uri was given if self.url: From abdecca1337b7331d394aad06e0595659e3a394e Mon Sep 17 00:00:00 2001 From: Paul English Date: Thu, 16 Apr 2015 15:28:49 -0600 Subject: [PATCH 0483/1103] Update task-cookbook.rst `django.utils.hashcompat` has been deprecated and removed in 1.6. It's recommended to use the builtin python `hashlib` instead. https://docs.djangoproject.com/en/1.5/internals/deprecation/ --- docs/tutorials/task-cookbook.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/task-cookbook.rst b/docs/tutorials/task-cookbook.rst index e44722686..a4c01868f 100644 --- a/docs/tutorials/task-cookbook.rst +++ b/docs/tutorials/task-cookbook.rst @@ -31,7 +31,7 @@ The cache key expires after some time in case something unexpected happens from celery import task from celery.utils.log import get_task_logger from django.core.cache import cache - from django.utils.hashcompat import md5_constructor as md5 + from hashlib import md5 from djangofeeds.models import Feed logger = get_task_logger(__name__) From e53e170b629dd5fd45bad64880c7b9ca248f28cb Mon Sep 17 00:00:00 2001 From: fatihsucu Date: Fri, 17 Apr 2015 12:12:50 +0300 Subject: [PATCH 0484/1103] mongodb backend version controller and defaults added. --- celery/backends/mongodb.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 624807aad..278eb6496 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -79,6 +79,14 @@ def __init__(self, app=None, url=None, **kwargs): self.url = url + # default options according to pymongo version + if pymongo.version_tuple >= 3: + self.options.setdefault('maxPoolSize', self.max_pool_size) + else: + self.options.setdefault('max_pool_size', self.max_pool_size) + self.options.setdefault('auto_start_request', False) + + # update conf with mongo uri data, only if uri was given if self.url: uri_data = pymongo.uri_parser.parse_uri(self.url) From c0b366d558e9d51c2e5ad62d482eab45e903d7db Mon Sep 17 00:00:00 2001 From: fatihsucu Date: Fri, 17 Apr 2015 12:25:16 +0300 Subject: [PATCH 0485/1103] fixed type in version controller --- celery/backends/mongodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 278eb6496..1abb1bbe0 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -80,7 +80,7 @@ def __init__(self, app=None, url=None, **kwargs): # default options according to pymongo version - if pymongo.version_tuple >= 3: + if pymongo.version_tuple >= (3,): self.options.setdefault('maxPoolSize', self.max_pool_size) else: self.options.setdefault('max_pool_size', self.max_pool_size) From 33a74c6873fa34220a196e8123a7b4bb6b88f5d7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 17 Apr 2015 14:21:21 +0100 Subject: [PATCH 0486/1103] flakes --- celery/worker/request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/worker/request.py b/celery/worker/request.py index f76be4c03..2b0ca1f58 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -103,7 +103,7 @@ def __init__(self, message, on_ack=noop, else: self.content_type, self.content_encoding = ( message.content_type, message.content_encoding, - ) + ) name = self.name = headers['task'] self.id = headers['id'] From 41347903e1e32bd421aaa29ed06589f558e9c204 Mon Sep 17 00:00:00 2001 From: Fatih Sucu Date: Fri, 17 Apr 2015 22:40:51 +0300 Subject: [PATCH 0487/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index a0bcbd34a..78599ac9f 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -183,3 +183,4 @@ Mickaël Penhard, 2015/02/15 Mark Parncutt, 2015/02/16 Samuel Jaillet, 2015/03/24 Ilya Georgievsky, 2015/03/31 +Fatih Sucu, 2015/04/17 From 15c00792d48fcb2b76d3af0b717a254bd8bd5dc9 Mon Sep 17 00:00:00 2001 From: Dmitry Malinovsky Date: Sun, 19 Apr 2015 16:58:37 +0600 Subject: [PATCH 0488/1103] Add William King to contributors --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 78599ac9f..aeb1101a4 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -176,6 +176,7 @@ Nathan Van Gheem, 2014/10/28 Gino Ledesma, 2014/10/28 Thomas French, 2014/11/10 Michael Permana, 2014/11/6 +William King, 2014/11/21 Bert Vanderbauwhede, 2014/12/18 John Anderson, 2014/12/27 Luke Burden, 2015/01/24 From dad37c741dbc4374b3ce231add6448b11f5e22fa Mon Sep 17 00:00:00 2001 From: PMickael Date: Mon, 20 Apr 2015 09:35:50 +0200 Subject: [PATCH 0489/1103] Process import change from billiard since 3.4 multiprocessing Fix #2530 [https://github.com/celery/billiard/commit/c7eedbd0ee1498e76d4fa1affac5b 1a275660ee7] --- celery/beat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/beat.py b/celery/beat.py index b17a2c295..8bb023b91 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -21,7 +21,7 @@ from threading import Event, Thread from billiard import ensure_multiprocessing -from billiard.process import Process +from billiard.context import Process from billiard.common import reset_signals from kombu.utils import cached_property, reprcall from kombu.utils.functional import maybe_evaluate From d1a0086858007f5dc1cc397ddd3a0e6baebc77fe Mon Sep 17 00:00:00 2001 From: James Pulec Date: Mon, 20 Apr 2015 09:52:38 -0700 Subject: [PATCH 0490/1103] Add James Pulec to CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index a0bcbd34a..b0b4cd611 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -183,3 +183,5 @@ Mickaël Penhard, 2015/02/15 Mark Parncutt, 2015/02/16 Samuel Jaillet, 2015/03/24 Ilya Georgievsky, 2015/03/31 +Fatih Sucu, 2015/04/17 +James Pulec, 2015/04/19 From d22e17ddc9ac4bc513e8bf7000f5fb0979ebb434 Mon Sep 17 00:00:00 2001 From: Alexander Date: Sat, 14 Mar 2015 16:18:24 +0500 Subject: [PATCH 0491/1103] Fix TypeError raised in logging (validate_models) --- celery/fixups/django.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index eb5c8d235..05c68d022 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -1,10 +1,14 @@ from __future__ import absolute_import -import io import os import sys import warnings +if sys.version_info[0] < 3 and not hasattr(sys, 'pypy_version_info'): + from StringIO import StringIO +else: + from io import StringIO + from kombu.utils import cached_property, symbol_by_name from datetime import datetime @@ -152,7 +156,7 @@ def validate_models(self): pass else: django_setup() - s = io.StringIO() + s = StringIO() try: from django.core.management.validation import get_validation_errors except ImportError: From 9223daf2c88f2fe48dc01a389b3a820b23a0acb2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 22 Apr 2015 17:24:48 +0100 Subject: [PATCH 0492/1103] Update link to Pylons. Issue #2535 --- docs/includes/introduction.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index e178f0422..da5fda4a1 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -177,7 +177,7 @@ development easier, and sometimes they add important hooks like closing database connections at ``fork``. .. _`Django`: http://djangoproject.com/ -.. _`Pylons`: http://pylonshq.com/ +.. _`Pylons`: http://pylonsproject.org/ .. _`Flask`: http://flask.pocoo.org/ .. _`web2py`: http://web2py.com/ .. _`Bottle`: http://bottlepy.org/ From 46dd54dca414195b1c31ba1f6e2e9d4fcafc03bf Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 22 Apr 2015 18:25:17 +0100 Subject: [PATCH 0493/1103] Please PyPy --- celery/utils/functional.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index afee84d11..63242bdbb 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -90,17 +90,17 @@ def __setitem__(self, key, value): def __iter__(self): return iter(self.data) - def _iterate_items(self): - for k in self: - try: - yield (k, self.data[k]) - except KeyError: # pragma: no cover - pass + def _iterate_items(self, _need_lock=IS_PYPY): + with self.mutex if _need_lock else DummyContext(): + for k in self: + try: + yield (k, self.data[k]) + except KeyError: # pragma: no cover + pass iteritems = _iterate_items def _iterate_values(self, _need_lock=IS_PYPY): - ctx = self.mutex if _need_lock else DummyContext() - with ctx: + with self.mutex if _need_lock else DummyContext(): for k in self: try: yield self.data[k] From db2caf540ae9e420e2336958d3ef1fe5b859eba5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 22 Apr 2015 21:01:48 +0100 Subject: [PATCH 0494/1103] Pool: Fall back to using select if poll is not supported. Closes #2430 --- celery/concurrency/asynpool.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 656e4a0cf..37263c7a5 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -111,8 +111,9 @@ def _get_job_writer(job): def _select(readers=None, writers=None, err=None, timeout=0, - poll=select.poll, POLLIN=select.POLLIN, - POLLOUT=select.POLLOUT, POLLERR=select.POLLERR): + poll=getattr(select, 'poll', select.select), + POLLIN=select.POLLIN, POLLOUT=select.POLLOUT, + POLLERR=select.POLLERR): """Simple wrapper to :class:`~select.select`, using :`~select.poll` as the implementation. From 9ea86e9eea0452f2bfaccd1eb7ab2c3a673ac0d6 Mon Sep 17 00:00:00 2001 From: Alexander Date: Sat, 25 Apr 2015 20:11:15 +0500 Subject: [PATCH 0495/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 3baad1f23..bd30be9ac 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -186,3 +186,4 @@ Samuel Jaillet, 2015/03/24 Ilya Georgievsky, 2015/03/31 Fatih Sucu, 2015/04/17 James Pulec, 2015/04/19 +Alexander Lebedev, 2015/04/25 From c2273f49f397d63987f933eb422a234b1776e1cd Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 29 Apr 2015 12:40:25 +0100 Subject: [PATCH 0496/1103] Doc wording --- celery/app/task.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index 01ff2935b..bd35028d3 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -390,11 +390,12 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, :keyword retry: If enabled sending of the task message will be retried in the event of connection loss or failure. Default is taken from the :setting:`CELERY_TASK_PUBLISH_RETRY` - setting. Note you need to handle the + setting. Note that you need to handle the producer/connection manually for this to work. :keyword retry_policy: Override the retry policy used. See the - :setting:`CELERY_TASK_PUBLISH_RETRY` setting. + :setting:`CELERY_TASK_PUBLISH_RETRY_POLICY` + setting. :keyword routing_key: Custom routing key used to route the task to a worker server. If in combination with a @@ -544,14 +545,14 @@ def retry(self, args=None, kwargs=None, exc=None, throw=True, >>> from imaginary_twitter_lib import Twitter >>> from proj.celery import app - >>> @app.task() - ... def tweet(auth, message): + >>> @app.task(bind=True) + ... def tweet(self, auth, message): ... twitter = Twitter(oauth=auth) ... try: ... twitter.post_status_update(message) ... except twitter.FailWhale as exc: ... # Retry in 5 minutes. - ... raise tweet.retry(countdown=60 * 5, exc=exc) + ... raise self.retry(countdown=60 * 5, exc=exc) Although the task will never return above as `retry` raises an exception to notify the worker, we use `raise` in front of the retry @@ -818,9 +819,8 @@ def after_return(self, status, retval, task_id, args, kwargs, einfo): :param status: Current task state. :param retval: Task return value/exception. :param task_id: Unique id of the task. - :param args: Original arguments for the task that failed. - :param kwargs: Original keyword arguments for the task - that failed. + :param args: Original arguments for the task. + :param kwargs: Original keyword arguments for the task. :keyword einfo: :class:`~billiard.einfo.ExceptionInfo` instance, containing the traceback (if any). From 6592ff64b6b024a4b68abcc53b151888fdf0dee3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 29 Apr 2015 12:48:31 +0100 Subject: [PATCH 0497/1103] MongoDB backend tests now passing with pymongo 3.x. Closes #2589 --- celery/backends/mongodb.py | 20 +++++++++++--------- celery/tests/backends/test_mongodb.py | 26 +++++++++++++------------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 1abb1bbe0..17332338d 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -30,7 +30,7 @@ from kombu.exceptions import EncodeError from celery import states from celery.exceptions import ImproperlyConfigured -from celery.five import string_t +from celery.five import string_t, items from .base import BaseBackend @@ -78,14 +78,9 @@ def __init__(self, app=None, url=None, **kwargs): self.url = url - - # default options according to pymongo version - if pymongo.version_tuple >= (3,): - self.options.setdefault('maxPoolSize', self.max_pool_size) - else: - self.options.setdefault('max_pool_size', self.max_pool_size) - self.options.setdefault('auto_start_request', False) - + # Set option defaults + for key, value in items(self._prepare_client_options()): + self.options.setdefault(key, value) # update conf with mongo uri data, only if uri was given if self.url: @@ -130,6 +125,13 @@ def __init__(self, app=None, url=None, **kwargs): self.options.update(config.pop('options', {})) self.options.update(config) + def _prepare_client_options(self): + if pymongo.version_tuple >= (3, ): + return {'maxPoolSize': self.max_pool_size} + else: # pragma: no cover + return {'max_pool_size': max_pool_size, + 'auto_start_request': False} + def _get_connection(self): """Connect to the MongoDB server.""" if self._connection is None: diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index 7f6597bd0..2d656a6d5 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -76,8 +76,7 @@ def test_init_with_settings(self): uri = 'mongodb://localhost:27017' mb = MongoBackend(app=self.app, url=uri) self.assertEqual(mb.mongo_host, ['localhost:27017']) - self.assertEqual(mb.options, {'auto_start_request': False, - 'max_pool_size': 10}) + self.assertEqual(mb.options, mb._prepare_client_options()) self.assertEqual(mb.database_name, 'celery') # uri with database name @@ -96,9 +95,9 @@ def test_init_with_settings(self): self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017', 'mongo2.example.com:27017', 'mongo3.example.com:27017']) - self.assertEqual(mb.options, {'auto_start_request': False, - 'max_pool_size': 10, - 'replicaset': 'rs0'}) + self.assertEqual( + mb.options, dict(mb._prepare_client_options(), replicaset='rs0'), + ) self.assertEqual(mb.user, 'celeryuser') self.assertEqual(mb.password, 'celerypassword') self.assertEqual(mb.database_name, 'celerydatabase') @@ -116,10 +115,10 @@ def test_init_with_settings(self): self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017', 'mongo2.example.com:27017', 'mongo3.example.com:27017']) - self.assertEqual(mb.options, {'auto_start_request': False, - 'max_pool_size': 10, - 'replicaset': 'rs1', - 'socketKeepAlive': True}) + self.assertEqual( + mb.options, dict(mb._prepare_client_options(), + replicaset='rs1', socketKeepAlive=True), + ) self.assertEqual(mb.user, 'backenduser') self.assertEqual(mb.password, 'celerypassword') self.assertEqual(mb.database_name, 'another_db') @@ -149,8 +148,9 @@ def test_get_connection_no_connection_host(self): connection = self.backend._get_connection() mock_Connection.assert_called_once_with( - host='mongodb://localhost:27017', max_pool_size=10, - auto_start_request=False) + host='mongodb://localhost:27017', + **self.backend._prepare_client_options() + ) self.assertEqual(sentinel.connection, connection) def test_get_connection_no_connection_mongodb_uri(self): @@ -164,8 +164,8 @@ def test_get_connection_no_connection_mongodb_uri(self): connection = self.backend._get_connection() mock_Connection.assert_called_once_with( - host=mongodb_uri, max_pool_size=10, - auto_start_request=False) + host=mongodb_uri, **self.backend._prepare_client_options() + ) self.assertEqual(sentinel.connection, connection) @patch('celery.backends.mongodb.MongoBackend._get_connection') From 33e72fdbc7b07fc26d13bcdc36fb6f42c8291b66 Mon Sep 17 00:00:00 2001 From: Allard Hoeve Date: Wed, 29 Apr 2015 16:41:00 +0200 Subject: [PATCH 0498/1103] Fix Exception marshalling with JSON serializer The code in `drain_events` in `amqp.py` naively sets the result dict to a plain meta dict without transforming the dict structure back into an actual Exception through `exception_to_python`. When a task raises an exception, `AsyncResult.get` tries to raise the exception, which is actually still a dict and fails with: ``` TypeError: exceptions must be old-style classes or derived from BaseException, not dict ``` This patch makes `drain_events` call `meta_from_decoded` which is responsible for that, just like it is called in `get_many`. Then, raising the exception in `AsyncResult.get` works fine. To reproduce, see the testcase in #2518. Then, apply the patch and see stuff start to work again. closes #2518 --- celery/backends/amqp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index 596a4c667..4871e0623 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -195,7 +195,7 @@ def drain_events(self, connection, consumer, def callback(meta, message): if meta['status'] in states.READY_STATES: - results[meta['task_id']] = meta + results[meta['task_id']] = self.meta_from_decoded(meta) consumer.callbacks[:] = [callback] time_start = now() From 3dd71214eab7b1ed427457203863a1b9803337d9 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 29 Apr 2015 22:26:16 +0100 Subject: [PATCH 0499/1103] Pool: select api is not the same as poll (Issue #2430) --- celery/concurrency/asynpool.py | 63 ++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 37263c7a5..501707f06 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -110,10 +110,43 @@ def _get_job_writer(job): return writer() # is a weakref +if hasattr(select, 'poll', None): + def _select_imp(readers=None, writers=None, err=None, timeout=0, + poll=select.poll, POLLIN=select.POLLIN, + POLLOUT=select.POLLOUT, POLLERR=select.POLLERR): + poller = poll() + register = poller.register + + if readers: + [register(fd, POLLIN) for fd in readers] + if writers: + [register(fd, POLLOUT) for fd in writers] + if err: + [register(fd, POLLERR) for fd in err] + + R, W = set(), set() + timeout = 0 if timeout and timeout < 0 else round(timeout * 1e3) + events = poller.poll(timeout) + for fd, event in events: + if not isinstance(fd, Integral): + fd = fd.fileno() + if event & POLLIN: + R.add(fd) + if event & POLLOUT: + W.add(fd) + if event & POLLERR: + R.add(fd) + return R, W, 0 +else: + def _select_imp(readers=None, writers=None, err=None, timeout=0): + r, w, e = select.select(readers, writers, err, timeout) + if e: + r = list(set(r) | set(e)) + return r, w, 0 + + def _select(readers=None, writers=None, err=None, timeout=0, - poll=getattr(select, 'poll', select.select), - POLLIN=select.POLLIN, POLLOUT=select.POLLOUT, - POLLERR=select.POLLERR): + _select_imp=_select_imp): """Simple wrapper to :class:`~select.select`, using :`~select.poll` as the implementation. @@ -136,30 +169,8 @@ def _select(readers=None, writers=None, err=None, timeout=0, readers = set() if readers is None else readers writers = set() if writers is None else writers err = set() if err is None else err - poller = poll() - register = poller.register - - if readers: - [register(fd, POLLIN) for fd in readers] - if writers: - [register(fd, POLLOUT) for fd in writers] - if err: - [register(fd, POLLERR) for fd in err] - - R, W = set(), set() - timeout = 0 if timeout and timeout < 0 else round(timeout * 1e3) try: - events = poller.poll(timeout) - for fd, event in events: - if not isinstance(fd, Integral): - fd = fd.fileno() - if event & POLLIN: - R.add(fd) - if event & POLLOUT: - W.add(fd) - if event & POLLERR: - R.add(fd) - return R, W, 0 + return _select_imp(readers, writers, err, timeout) except (select.error, socket.error) as exc: if exc.errno == errno.EINTR: return set(), set(), 1 From e544b4ea63275bbaf31f305133b7a931def983cf Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 29 Apr 2015 23:00:17 +0100 Subject: [PATCH 0500/1103] sloppy --- celery/concurrency/asynpool.py | 6 ++--- celery/tests/concurrency/test_prefork.py | 34 ++++++++++++------------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 501707f06..b1cb64751 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -110,7 +110,7 @@ def _get_job_writer(job): return writer() # is a weakref -if hasattr(select, 'poll', None): +if hasattr(select, 'poll'): def _select_imp(readers=None, writers=None, err=None, timeout=0, poll=select.poll, POLLIN=select.POLLIN, POLLOUT=select.POLLOUT, POLLERR=select.POLLERR): @@ -146,7 +146,7 @@ def _select_imp(readers=None, writers=None, err=None, timeout=0): def _select(readers=None, writers=None, err=None, timeout=0, - _select_imp=_select_imp): + poll=_select_imp): """Simple wrapper to :class:`~select.select`, using :`~select.poll` as the implementation. @@ -170,7 +170,7 @@ def _select(readers=None, writers=None, err=None, timeout=0, writers = set() if writers is None else writers err = set() if err is None else err try: - return _select_imp(readers, writers, err, timeout) + return poll(readers, writers, err, timeout) except (select.error, socket.error) as exc: if exc.errno == errno.EINTR: return set(), set(), 1 diff --git a/celery/tests/concurrency/test_prefork.py b/celery/tests/concurrency/test_prefork.py index 00fec85d9..23c35ad0b 100644 --- a/celery/tests/concurrency/test_prefork.py +++ b/celery/tests/concurrency/test_prefork.py @@ -154,46 +154,46 @@ def test_select(self, __select): ebadf.errno = errno.EBADF with patch('select.poll', create=True) as poller: poll = poller.return_value = Mock(name='poll.poll') - poll.poll.return_value = [(3, select.POLLIN)] + poll.return_value = {3}, set(), 0 self.assertEqual( - asynpool._select({3}, poll=poller), + asynpool._select({3}, poll=poll), ({3}, set(), 0), ) - poll.poll.return_value = [(3, select.POLLERR)] + poll.return_value = {3}, set(), 0 self.assertEqual( - asynpool._select({3}, None, {3}, poll=poller), + asynpool._select({3}, None, {3}, poll=poll), ({3}, set(), 0), ) eintr = socket.error() eintr.errno = errno.EINTR - poll.poll.side_effect = eintr + poll.side_effect = eintr readers = {3} self.assertEqual( - asynpool._select(readers, poll=poller), + asynpool._select(readers, poll=poll), (set(), set(), 1), ) self.assertIn(3, readers) with patch('select.poll') as poller: poll = poller.return_value = Mock(name='poll.poll') - poll.poll.side_effect = ebadf + poll.side_effect = ebadf with patch('select.select') as selcheck: selcheck.side_effect = ebadf readers = {3} self.assertEqual( - asynpool._select(readers, poll=poller), + asynpool._select(readers, poll=poll), (set(), set(), 1), ) self.assertNotIn(3, readers) with patch('select.poll') as poller: poll = poller.return_value = Mock(name='poll.poll') - poll.poll.side_effect = MemoryError() + poll.side_effect = MemoryError() with self.assertRaises(MemoryError): - asynpool._select({1}, poll=poller) + asynpool._select({1}, poll=poll) with patch('select.poll') as poller: poll = poller.return_value = Mock(name='poll.poll') @@ -202,9 +202,9 @@ def test_select(self, __select): def se(*args): selcheck.side_effect = MemoryError() raise ebadf - poll.poll.side_effect = se + poll.side_effect = se with self.assertRaises(MemoryError): - asynpool._select({3}, poll=poller) + asynpool._select({3}, poll=poll) with patch('select.poll') as poller: poll = poller.return_value = Mock(name='poll.poll') @@ -214,17 +214,17 @@ def se2(*args): selcheck.side_effect = socket.error() selcheck.side_effect.errno = 1321 raise ebadf - poll.poll.side_effect = se2 + poll.side_effect = se2 with self.assertRaises(socket.error): - asynpool._select({3}, poll=poller) + asynpool._select({3}, poll=poll) with patch('select.poll') as poller: poll = poller.return_value = Mock(name='poll.poll') - poll.poll.side_effect = socket.error() - poll.poll.side_effect.errno = 34134 + poll.side_effect = socket.error() + poll.side_effect.errno = 34134 with self.assertRaises(socket.error): - asynpool._select({3}, poll=poller) + asynpool._select({3}, poll=poll) def test_promise(self): fun = Mock() From 9f682d198147e30ff19ea654e64b263ba5cfc490 Mon Sep 17 00:00:00 2001 From: Allard Hoeve Date: Thu, 30 Apr 2015 09:29:47 +0200 Subject: [PATCH 0501/1103] Attempt to pass Travis test on pypy - supress progress bar by pip when running pypy as this seems to crash the test on Travis: https://travis-ci.org/celery/celery/jobs/60549687 - this prevents dev.txt from being properly installed and the result is tests failing with import errors --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 80cfd5c55..4977e8e4b 100644 --- a/tox.ini +++ b/tox.ini @@ -48,7 +48,7 @@ deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/dev.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - pip install -U -r{toxinidir}/requirements/dev.txt + pip install -q -U -r{toxinidir}/requirements/dev.txt nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:pypy3] @@ -59,7 +59,7 @@ deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/dev.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - pip install -U -r{toxinidir}/requirements/dev.txt + pip install -q -U -r{toxinidir}/requirements/dev.txt nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] [testenv:docs] From a84d67ce3334ec558a2160dac9bfd40f06b72aa0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 30 Apr 2015 16:23:55 +0100 Subject: [PATCH 0502/1103] Embedded beat must set app for thread/process. Closes #2594 --- celery/beat.py | 20 ++++++++++++-------- celery/tests/app/test_beat.py | 4 ++-- celery/worker/components.py | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/celery/beat.py b/celery/beat.py index 8bb023b91..21d1316c6 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -535,13 +535,15 @@ def scheduler(self): class _Threaded(Thread): """Embedded task scheduler using threading.""" - def __init__(self, *args, **kwargs): + def __init__(self, app, **kwargs): super(_Threaded, self).__init__() - self.service = Service(*args, **kwargs) + self.app = app + self.service = Service(app, **kwargs) self.daemon = True self.name = 'Beat' def run(self): + self.app.set_current() self.service.start() def stop(self): @@ -555,9 +557,10 @@ def stop(self): else: class _Process(Process): # noqa - def __init__(self, *args, **kwargs): + def __init__(self, app, **kwargs): super(_Process, self).__init__() - self.service = Service(*args, **kwargs) + self.app = app + self.service = Service(app, **kwargs) self.name = 'Beat' def run(self): @@ -565,6 +568,8 @@ def run(self): platforms.close_open_fds([ sys.__stdin__, sys.__stdout__, sys.__stderr__, ] + list(iter_open_logger_fds())) + self.app.set_default() + self.app.set_current() self.service.start(embedded_process=True) def stop(self): @@ -572,7 +577,7 @@ def stop(self): self.terminate() -def EmbeddedService(*args, **kwargs): +def EmbeddedService(app, max_interval=None, **kwargs): """Return embedded clock service. :keyword thread: Run threaded instead of as a separate process. @@ -582,6 +587,5 @@ def EmbeddedService(*args, **kwargs): if kwargs.pop('thread', False) or _Process is None: # Need short max interval to be able to stop thread # in reasonable time. - kwargs.setdefault('max_interval', 1) - return _Threaded(*args, **kwargs) - return _Process(*args, **kwargs) + return _Threaded(app, max_interval=1, **kwargs) + return _Process(app, max_interval=max_interval, **kwargs) diff --git a/celery/tests/app/test_beat.py b/celery/tests/app/test_beat.py index 362fbf9b4..40b8c8589 100644 --- a/celery/tests/app/test_beat.py +++ b/celery/tests/app/test_beat.py @@ -478,7 +478,7 @@ def test_start_stop_process(self): from billiard.process import Process - s = beat.EmbeddedService(app=self.app) + s = beat.EmbeddedService(self.app) self.assertIsInstance(s, Process) self.assertIsInstance(s.service, beat.Service) s.service = MockService() @@ -499,7 +499,7 @@ def terminate(self): self.assertTrue(s._popen.terminated) def test_start_stop_threaded(self): - s = beat.EmbeddedService(thread=True, app=self.app) + s = beat.EmbeddedService(self.app, thread=True) from threading import Thread self.assertIsInstance(s, Thread) self.assertIsInstance(s.service, beat.Service) diff --git a/celery/worker/components.py b/celery/worker/components.py index d23a3b6b8..bb02f4e9e 100644 --- a/celery/worker/components.py +++ b/celery/worker/components.py @@ -203,7 +203,7 @@ def create(self, w): from celery.beat import EmbeddedService if w.pool_cls.__module__.endswith(('gevent', 'eventlet')): raise ImproperlyConfigured(ERR_B_GREEN) - b = w.beat = EmbeddedService(app=w.app, + b = w.beat = EmbeddedService(w.app, schedule_filename=w.schedule_filename, scheduler_cls=w.scheduler_cls) return b From 3d88ede2760f857c0e800e1f641c84cd61ef64d2 Mon Sep 17 00:00:00 2001 From: Allard Hoeve Date: Thu, 30 Apr 2015 18:09:05 +0200 Subject: [PATCH 0503/1103] Add test --- celery/tests/backends/test_amqp.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/celery/tests/backends/test_amqp.py b/celery/tests/backends/test_amqp.py index 031481c8d..32bda1c9c 100644 --- a/celery/tests/backends/test_amqp.py +++ b/celery/tests/backends/test_amqp.py @@ -13,6 +13,7 @@ from celery.backends.amqp import AMQPBackend from celery.exceptions import TimeoutError from celery.five import Empty, Queue, range +from celery.result import AsyncResult from celery.utils import uuid from celery.tests.case import ( @@ -246,10 +247,20 @@ def test_wait_for(self): with self.assertRaises(TimeoutError): b.wait_for(tid, timeout=0.01, cache=False) - def test_drain_events_remaining_timeouts(self): + def test_drain_events_decodes_exceptions_in_meta(self): + tid = uuid() + b = self.create_backend(serializer="json") + b.store_result(tid, RuntimeError("aap"), states.FAILURE) + result = AsyncResult(tid, backend=b) - class Connection(object): + with self.assertRaises(Exception) as cm: + result.get() + self.assertEqual(cm.exception.__class__.__name__, "RuntimeError") + self.assertEqual(str(cm.exception), "aap") + + def test_drain_events_remaining_timeouts(self): + class Connection(object): def drain_events(self, timeout=None): pass From 7bf736bfae86766819c3af5da661f7152b364c85 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Tue, 5 May 2015 21:53:48 -0400 Subject: [PATCH 0504/1103] Update task.py The `throw` behaviour should be honored in 'eager' mode as well. --- celery/app/task.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/celery/app/task.py b/celery/app/task.py index bd35028d3..f775ed287 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -595,6 +595,8 @@ def retry(self, args=None, kwargs=None, exc=None, throw=True, # if task was executed eagerly using apply(), # then the retry must also be executed eagerly. S.apply().get() + if throw: + raise ret return ret try: From 5222f8bd8ad1c6548ae9d7d38cbd9a9949cec193 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 7 May 2015 14:07:55 +0100 Subject: [PATCH 0505/1103] Respect Exchange.delivery_mode (depends on celery/kombu@b6c3f99f66ccdcd359ed92dd8c59174cc0f1c0d3) Closes #1953 --- celery/app/amqp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 85d3f5bea..2e738e19c 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -438,7 +438,8 @@ def publish_task(producer, name, message, try: delivery_mode = queue.exchange.delivery_mode except AttributeError: - delivery_mode = default_delivery_mode + pass + delivery_mode = delivery_mode or default_delivery_mode exchange = exchange or queue.exchange.name routing_key = routing_key or queue.routing_key if declare is None and queue and not isinstance(queue, Broadcast): From 4fc01fa7eb70930388c63e0871688079dacb3594 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 7 May 2015 14:22:17 +0100 Subject: [PATCH 0506/1103] Fixes celery amqp when using pyamqp://. Closes #2013 --- celery/bin/amqp.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/celery/bin/amqp.py b/celery/bin/amqp.py index 638b5ed7a..40e858e25 100644 --- a/celery/bin/amqp.py +++ b/celery/bin/amqp.py @@ -182,6 +182,16 @@ class AMQShell(cmd.Cmd): 'basic.ack': Spec(('delivery_tag', int)), } + def _prepare_spec(self, conn): + # XXX Hack to fix Issue #2013 + from amqp import Connection, Message + if isinstance(conn.connection, Connection): + self.amqp['basic.publish'] = Spec(('msg', Message), + ('exchange', str), + ('routing_key', str), + ('mandatory', bool, 'no'), + ('immediate', bool, 'no')) + def __init__(self, *args, **kwargs): self.connect = kwargs.pop('connect') self.silent = kwargs.pop('silent', False) @@ -298,6 +308,7 @@ def respond(self, retval): def _reconnect(self): """Re-establish connection to the AMQP server.""" self.conn = self.connect(self.conn) + self._prepare_spec(self.conn) self.chan = self.conn.default_channel self.needs_reconnect = False From ec73ceb38c91e49d47b06c6eaa879daf675e917b Mon Sep 17 00:00:00 2001 From: JTill Date: Fri, 8 May 2015 07:09:55 +0000 Subject: [PATCH 0507/1103] Only set default app if there isn't one already --- celery/fixups/django.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index 05c68d022..66b76f4db 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -15,6 +15,7 @@ from importlib import import_module from celery import signals +from celery.app import default_app from celery.exceptions import FixupWarning __all__ = ['DjangoFixup', 'fixup'] @@ -48,7 +49,8 @@ class DjangoFixup(object): def __init__(self, app): self.app = app - self.app.set_default() + if default_app is None: + self.app.set_default() self._worker_fixup = None def install(self): From 2d931569c95711713e9409a0caf3824776a0dacf Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 12 May 2015 22:45:52 +0100 Subject: [PATCH 0508/1103] Implements the task protocol 2 shadow field --- celery/app/amqp.py | 4 ++-- celery/app/base.py | 6 ++++-- celery/app/task.py | 40 +++++++++++++++++++++++++++++++++++----- celery/worker/request.py | 24 +++++++++++++++--------- 4 files changed, 56 insertions(+), 18 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 2e738e19c..09320be04 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -270,7 +270,7 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, callbacks=None, errbacks=None, reply_to=None, time_limit=None, soft_time_limit=None, create_sent_event=False, root_id=None, parent_id=None, - now=None, timezone=None): + shadow=None, now=None, timezone=None): args = args or () kwargs = kwargs or {} utc = self.utc @@ -337,7 +337,7 @@ def as_task_v1(self, task_id, name, args=None, kwargs=None, chord=None, callbacks=None, errbacks=None, reply_to=None, time_limit=None, soft_time_limit=None, create_sent_event=False, root_id=None, parent_id=None, - now=None, timezone=None): + shadow=None, now=None, timezone=None): args = args or () kwargs = kwargs or {} utc = self.utc diff --git a/celery/app/base.py b/celery/app/base.py index cd68d5266..2f40a509b 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -366,7 +366,8 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, publisher=None, link=None, link_error=None, add_to_parent=True, group_id=None, retries=0, chord=None, reply_to=None, time_limit=None, soft_time_limit=None, - root_id=None, parent_id=None, route_name=None, **options): + root_id=None, parent_id=None, route_name=None, + shadow=None, **options): amqp = self.amqp task_id = task_id or uuid() producer = producer or publisher # XXX compat @@ -383,7 +384,8 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, expires, retries, chord, maybe_list(link), maybe_list(link_error), reply_to or self.oid, time_limit, soft_time_limit, - self.conf.CELERY_SEND_TASK_SENT_EVENT, root_id, parent_id, + self.conf.CELERY_SEND_TASK_SENT_EVENT, + root_id, parent_id, shadow, ) if connection: diff --git a/celery/app/task.py b/celery/app/task.py index bd35028d3..a65703595 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -360,7 +360,7 @@ def delay(self, *args, **kwargs): return self.apply_async(args, kwargs) def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, - link=None, link_error=None, **options): + link=None, link_error=None, shadow=None, **options): """Apply tasks asynchronously by sending a message. :keyword args: The positional arguments to pass on to the @@ -384,6 +384,9 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, the task should expire. The task will not be executed after the expiration time. + :keyword shadow: Override task name used in logs/monitoring + (default from :meth:`shadow_name`). + :keyword connection: Re-use existing broker connection instead of establishing a new one. @@ -440,9 +443,9 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, attribute. Trailing can also be disabled by default using the :attr:`trail` attribute :keyword publisher: Deprecated alias to ``producer``. - - :rtype :class:`celery.result.AsyncResult`: if - :setting:`CELERY_ALWAYS_EAGER` is not set, otherwise + + :rtype :class:`celery.result.AsyncResult`: if + :setting:`CELERY_ALWAYS_EAGER` is not set, otherwise :class:`celery.result.EagerResult`: Also supports all keyword arguments supported by @@ -468,12 +471,39 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, if self.__self__ is not None: args = args if isinstance(args, tuple) else tuple(args or ()) args = (self.__self__, ) + args + final_options = self._get_exec_options() + if options: + final_options = dict(final_options, **options) return app.send_task( self.name, args, kwargs, task_id=task_id, producer=producer, link=link, link_error=link_error, result_cls=self.AsyncResult, - **dict(self._get_exec_options(), **options) + shadow=shadow or self.shadow_name(args, kwargs, final_options), + **final_options ) + def shadow_name(self, args, kwargs, options): + """Override for custom task name in worker logs/monitoring. + + :param args: Task positional arguments. + :param kwargs: Task keyword arguments. + :param options: Task execution options. + + **Example**: + + .. code-block:: python + + from celery.utils.imports import qualname + + def shadow_name(task, args, kwargs, options): + return qualname(args[0]) + + @app.task(shadow_name=shadow_name, serializer='pickle') + def apply_function_async(fun, *args, **kwargs): + return fun(*args, **kwargs) + + """ + pass + def signature_from_request(self, request=None, args=None, kwargs=None, queue=None, **extra_options): request = self.request if request is None else request diff --git a/celery/worker/request.py b/celery/worker/request.py index 2b0ca1f58..194358045 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -76,7 +76,7 @@ class Request(object): if not IS_PYPY: # pragma: no cover __slots__ = ( - 'app', 'name', 'id', 'on_ack', 'body', + 'app', 'type', 'name', 'id', 'on_ack', 'body', 'hostname', 'eventer', 'connection_errors', 'task', 'eta', 'expires', 'request_dict', 'on_reject', 'utc', 'content_type', 'content_encoding', @@ -105,8 +105,10 @@ def __init__(self, message, on_ack=noop, message.content_type, message.content_encoding, ) - name = self.name = headers['task'] self.id = headers['id'] + type = self.type = self.name = headers['task'] + if 'shadow' in headers: + self.name = headers['shadow'] if 'timelimit' in headers: self.time_limits = headers['timelimit'] self.on_ack = on_ack @@ -114,7 +116,7 @@ def __init__(self, message, on_ack=noop, self.hostname = hostname or socket.gethostname() self.eventer = eventer self.connection_errors = connection_errors or () - self.task = task or self.app.tasks[name] + self.task = task or self.app.tasks[type] # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. @@ -178,7 +180,7 @@ def execute_using_pool(self, pool, **kwargs): soft_time_limit = soft_time_limit or task.soft_time_limit result = pool.apply_async( trace_task_ret, - args=(self.name, task_id, self.request_dict, self.body, + args=(self.type, task_id, self.request_dict, self.body, self.content_type, self.content_encoding), accept_callback=self.on_accepted, timeout_callback=self.on_timeout, @@ -377,6 +379,7 @@ def reject(self, requeue=False): def info(self, safe=False): return {'id': self.id, 'name': self.name, + 'type': self.type, 'body': self.body, 'hostname': self.hostname, 'time_start': self.time_start, @@ -385,15 +388,18 @@ def info(self, safe=False): 'worker_pid': self.worker_pid} def __str__(self): - return '{0.name}[{0.id}]{1}{2}'.format( - self, + return ' '.join([ + self.humaninfo(), ' eta:[{0}]'.format(self.eta) if self.eta else '', ' expires:[{0}]'.format(self.expires) if self.expires else '', - ) + ]) shortinfo = __str__ + def humaninfo(self): + return '{0.name}[{0.id}]'.format(self) + def __repr__(self): - return '<{0} {1}: {2}>'.format(type(self).__name__, self.id, self.name) + return '<{0}: {1}>'.format(type(self).__name__, self.humaninfo()) @property def tzlocal(self): @@ -457,7 +463,7 @@ def execute_using_pool(self, pool, **kwargs): soft_time_limit = soft_time_limit or default_soft_time_limit result = apply_async( trace, - args=(self.name, task_id, self.request_dict, self.body, + args=(self.type, task_id, self.request_dict, self.body, self.content_type, self.content_encoding), accept_callback=self.on_accepted, timeout_callback=self.on_timeout, From 0731623271251c6595c15e758cd539c46b32c6c9 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 12 May 2015 22:56:38 +0100 Subject: [PATCH 0509/1103] flakes --- celery/app/builtins.py | 4 +- celery/backends/mongodb.py | 2 +- celery/bin/base.py | 2 +- celery/contrib/batches.py | 6 +- celery/result.py | 4 +- celery/schedules.py | 71 +++++++++++++++--------- celery/tests/backends/test_amqp.py | 22 +++++--- celery/tests/concurrency/test_prefork.py | 1 - celery/tests/fixups/test_django.py | 2 +- 9 files changed, 72 insertions(+), 42 deletions(-) diff --git a/celery/app/builtins.py b/celery/app/builtins.py index 50db6ee7c..645611dfd 100644 --- a/celery/app/builtins.py +++ b/celery/app/builtins.py @@ -84,8 +84,8 @@ def unlock_chord(self, group_id, callback, interval=None, propagate=None, ready = deps.ready() except Exception as exc: raise self.retry( - exc=exc, countdown=interval, max_retries=max_retries, - ) + exc=exc, countdown=interval, max_retries=max_retries, + ) else: if not ready: raise self.retry(countdown=interval, max_retries=max_retries) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 17332338d..926ef454b 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -129,7 +129,7 @@ def _prepare_client_options(self): if pymongo.version_tuple >= (3, ): return {'maxPoolSize': self.max_pool_size} else: # pragma: no cover - return {'max_pool_size': max_pool_size, + return {'max_pool_size': self.max_pool_size, 'auto_start_request': False} def _get_connection(self): diff --git a/celery/bin/base.py b/celery/bin/base.py index c803ced2f..7c029d0f9 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -509,7 +509,7 @@ def add_append_opt(self, acc, opt, value): default = opt.default or [] if opt.dest not in acc: - acc[opt.dest] = default + acc[opt.dest] = default acc[opt.dest].append(value) diff --git a/celery/contrib/batches.py b/celery/contrib/batches.py index ad41c1903..e3d2e86c5 100644 --- a/celery/contrib/batches.py +++ b/celery/contrib/batches.py @@ -85,6 +85,8 @@ def wot_api_real(urls): from itertools import count +from kombu.five import buffer_t + from celery.task import Task from celery.five import Empty, Queue from celery.utils.log import get_logger @@ -195,6 +197,7 @@ def Strategy(self, task, app, consumer): timer = consumer.timer put_buffer = self._buffer.put flush_buffer = self._do_flush + body_can_be_buffer = consumer.pool.body_can_be_buffer def task_message_handler(message, body, ack, reject, callbacks, **kw): if body is None: @@ -209,8 +212,9 @@ def task_message_handler(message, body, ack, reject, callbacks, **kw): request = Req( message, on_ack=ack, on_reject=reject, app=app, hostname=hostname, - eventer=eventer, task=task, connection_errors=connection_errors, + eventer=eventer, task=task, body=body, headers=headers, decoded=decoded, utc=utc, + connection_errors=connection_errors, ) put_buffer(request) diff --git a/celery/result.py b/celery/result.py index d40d5fc42..df8880d11 100644 --- a/celery/result.py +++ b/celery/result.py @@ -577,7 +577,9 @@ def get(self, timeout=None, propagate=True, interval=0.5, """ return (self.join_native if self.supports_native_join else self.join)( timeout=timeout, propagate=propagate, - interval=interval, callback=callback, no_ack=no_ack, on_message=on_message) + interval=interval, callback=callback, no_ack=no_ack, + on_message=on_message, + ) def join(self, timeout=None, propagate=True, interval=0.5, callback=None, no_ack=True, on_message=None): diff --git a/celery/schedules.py b/celery/schedules.py index 917a8e2d7..4b3ffeaa1 100644 --- a/celery/schedules.py +++ b/celery/schedules.py @@ -47,6 +47,18 @@ {0._orig_day_of_month} {0._orig_month_of_year} (m/h/d/dM/MY)>\ """ +SOLAR_INVALID_LATITUDE = """\ +Argument latitude {lat} is invalid, must be between -90 and 90.\ +""" + +SOLAR_INVALID_LONGITUDE = """\ +Argument longitude {lon} is invalid, must be between -180 and 180.\ +""" + +SOLAR_INVALID_EVENT = """\ +Argument event "{event}" is invalid, must be one of {all_events}.\ +""" + def cronfield(s): return '*' if s is None else s @@ -592,17 +604,6 @@ def maybe_schedule(s, relative=False, app=None): s.app = app return s -SOLAR_INVALID_LATITUDE = """\ -Argument latitude {lat} is invalid, must be between -90 and 90.\ -""" - -SOLAR_INVALID_LONGITUDE = """\ -Argument longitude {lon} is invalid, must be between -180 and 180.\ -""" - -SOLAR_INVALID_EVENT = """\ -Argument event \"{event}\" is invalid, must be one of {all_events}.\ -""" class solar(schedule): """A solar event can be used as the `run_every` value of a @@ -619,8 +620,8 @@ class solar(schedule): :param app: Celery app instance. """ - - _all_events = ['dawn_astronomical', + _all_events = [ + 'dawn_astronomical', 'dawn_nautical', 'dawn_civil', 'sunrise', @@ -628,8 +629,10 @@ class solar(schedule): 'sunset', 'dusk_civil', 'dusk_nautical', - 'dusk_astronomical'] - _horizons = {'dawn_astronomical': '-18', + 'dusk_astronomical', + ] + _horizons = { + 'dawn_astronomical': '-18', 'dawn_nautical': '-12', 'dawn_civil': '-6', 'sunrise': '-0:34', @@ -637,8 +640,10 @@ class solar(schedule): 'sunset': '-0:34', 'dusk_civil': '-6', 'dusk_nautical': '-12', - 'dusk_astronomical': '18'} - _methods = {'dawn_astronomical': 'next_rising', + 'dusk_astronomical': '18', + } + _methods = { + 'dawn_astronomical': 'next_rising', 'dawn_nautical': 'next_rising', 'dawn_civil': 'next_rising', 'sunrise': 'next_rising', @@ -646,8 +651,10 @@ class solar(schedule): 'sunset': 'next_setting', 'dusk_civil': 'next_setting', 'dusk_nautical': 'next_setting', - 'dusk_astronomical': 'next_setting'} - _use_center_l = {'dawn_astronomical': True, + 'dusk_astronomical': 'next_setting', + } + _use_center_l = { + 'dawn_astronomical': True, 'dawn_nautical': True, 'dawn_civil': True, 'sunrise': False, @@ -655,7 +662,8 @@ class solar(schedule): 'sunset': False, 'dusk_civil': True, 'dusk_nautical': True, - 'dusk_astronomical': True} + 'dusk_astronomical': True, + } def __init__(self, event, lat, lon, nowfun=None, app=None): self.ephem = __import__('ephem') @@ -666,7 +674,9 @@ def __init__(self, event, lat, lon, nowfun=None, app=None): self._app = app if event not in self._all_events: - raise ValueError(SOLAR_INVALID_EVENT.format(event=event, all_events=', '.join(self._all_events))) + raise ValueError(SOLAR_INVALID_EVENT.format( + event=event, all_events=', '.join(self._all_events), + )) if lat < -90 or lat > 90: raise ValueError(SOLAR_INVALID_LATITUDE.format(lat=lat)) if lon < -180 or lon > 180: @@ -687,12 +697,13 @@ def now(self): return (self.nowfun or self.app.now)() def __reduce__(self): - return (self.__class__, (self.event, - self.lat, - self.lon), None) + return (self.__class__, ( + self.event, self.lat, self.lon), None) def __repr__(self): - return "" + return ''.format( + self.event, self.lat, self.lon, + ) def remaining_estimate(self, last_run_at): """Returns when the periodic task should run next as a timedelta, @@ -702,11 +713,17 @@ def remaining_estimate(self, last_run_at): last_run_at_utc = localize(last_run_at, timezone.utc) self.cal.date = last_run_at_utc try: - next_utc = getattr(self.cal, self.method)(self.ephem.Sun(), start=last_run_at_utc, use_center=self.use_center) + next_utc = getattr(self.cal, self.method)( + self.ephem.Sun(), + start=last_run_at_utc, use_center=self.use_center, + ) except self.ephem.CircumpolarError: """Sun will not rise/set today. Check again tomorrow (specifically, after the next anti-transit).""" - next_utc = self.cal.next_antitransit(self.ephem.Sun()) + timedelta(minutes=1) + next_utc = ( + self.cal.next_antitransit(self.ephem.Sun()) + + timedelta(minutes=1) + ) next = self.maybe_make_aware(next_utc.datetime()) now = self.maybe_make_aware(self.now()) delta = next - now diff --git a/celery/tests/backends/test_amqp.py b/celery/tests/backends/test_amqp.py index 031481c8d..3d0b4706b 100644 --- a/celery/tests/backends/test_amqp.py +++ b/celery/tests/backends/test_amqp.py @@ -306,22 +306,30 @@ def test_get_many_on_message(self): b.store_result(tid, 'final result %i' % i, states.SUCCESS) tids.append(tid) - expected_messages = {} for i, _tid in enumerate(tids): expected_messages[_tid] = [] - expected_messages[_tid].append( (states.PENDING, '') ) - expected_messages[_tid].append( (states.STARTED, 'comment_%i_1' % i) ) - expected_messages[_tid].append( (states.STARTED, 'comment_%i_2' % i) ) - expected_messages[_tid].append( (states.SUCCESS, 'final result %i' % i) ) + expected_messages[_tid].append((states.PENDING, '')) + expected_messages[_tid].append( + (states.STARTED, 'comment_%i_1' % i), + ) + expected_messages[_tid].append( + (states.STARTED, 'comment_%i_2' % i), + ) + expected_messages[_tid].append( + (states.SUCCESS, 'final result %i' % i), + ) on_message_results = {} + def on_message(body): if not body['task_id'] in on_message_results: on_message_results[body['task_id']] = [] - on_message_results[body['task_id']].append( (body['status'], body['result']) ) + on_message_results[body['task_id']].append( + (body['status'], body['result']), + ) - res = list(b.get_many(tids, timeout=1, on_message=on_message)) + list(b.get_many(tids, timeout=1, on_message=on_message)) self.assertEqual(sorted(on_message_results), sorted(expected_messages)) def test_get_many_raises_outer_block(self): diff --git a/celery/tests/concurrency/test_prefork.py b/celery/tests/concurrency/test_prefork.py index 23c35ad0b..47081ce26 100644 --- a/celery/tests/concurrency/test_prefork.py +++ b/celery/tests/concurrency/test_prefork.py @@ -1,7 +1,6 @@ from __future__ import absolute_import import errno -import select import socket import time diff --git a/celery/tests/fixups/test_django.py b/celery/tests/fixups/test_django.py index 9235bd005..94b755eee 100644 --- a/celery/tests/fixups/test_django.py +++ b/celery/tests/fixups/test_django.py @@ -209,7 +209,7 @@ def test__close_database(self): conns[1].close.side_effect = KeyError('already closed') f.database_errors = (KeyError, ) - f._db.connections = Mock() # ConnectionHandler + f._db.connections = Mock() # ConnectionHandler f._db.connections.all.side_effect = lambda: conns f._close_database() From 589a6998ef935ca9cd87b40744a16f9ff09c8e52 Mon Sep 17 00:00:00 2001 From: PMickael Date: Tue, 19 May 2015 10:45:36 +0200 Subject: [PATCH 0510/1103] Fix task protocol shadow exception when self.__self__ is None --- celery/app/task.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/celery/app/task.py b/celery/app/task.py index a65703595..4a0bbf83d 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -471,13 +471,15 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, if self.__self__ is not None: args = args if isinstance(args, tuple) else tuple(args or ()) args = (self.__self__, ) + args + shadow = shadow or self.shadow_name(args, kwargs, final_options) + final_options = self._get_exec_options() if options: final_options = dict(final_options, **options) return app.send_task( self.name, args, kwargs, task_id=task_id, producer=producer, link=link, link_error=link_error, result_cls=self.AsyncResult, - shadow=shadow or self.shadow_name(args, kwargs, final_options), + shadow=shadow, **final_options ) From ee461c8a80f9601d0724b42777ab2ed7f574739b Mon Sep 17 00:00:00 2001 From: Alex Rattray Date: Tue, 19 May 2015 15:54:29 +0530 Subject: [PATCH 0511/1103] Add Maintainers to README See https://github.com/celery/celery/issues/2534 --- README.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.rst b/README.rst index 464d5da02..39dc545e4 100644 --- a/README.rst +++ b/README.rst @@ -395,6 +395,21 @@ Wiki http://wiki.github.com/celery/celery/ + +.. _maintainers: + +Maintainers +=========== + +- @ask (primary maintainer) +- @thedrow +- @chrisgogreen +- @PMickael +- @malinoff +- @raghuramos1987 +- And you? We really need more: https://github.com/celery/celery/issues/2534 + + .. _contributing-short: Contributing From f15bc44f473b65fbeb5ddec2056f171a7de7c17c Mon Sep 17 00:00:00 2001 From: Alex Rattray Date: Wed, 20 May 2015 13:23:52 +0530 Subject: [PATCH 0512/1103] Remove @raghuramos1987 From his request in PR --- README.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/README.rst b/README.rst index 39dc545e4..f4ae4629c 100644 --- a/README.rst +++ b/README.rst @@ -406,7 +406,6 @@ Maintainers - @chrisgogreen - @PMickael - @malinoff -- @raghuramos1987 - And you? We really need more: https://github.com/celery/celery/issues/2534 From 0c34f81dd3bfc9c5531852be4a7d88427e872e98 Mon Sep 17 00:00:00 2001 From: squfrans Date: Wed, 13 May 2015 12:12:04 +0200 Subject: [PATCH 0513/1103] sync with reality --- celery/app/builtins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/builtins.py b/celery/app/builtins.py index 645611dfd..d1d341af2 100644 --- a/celery/app/builtins.py +++ b/celery/app/builtins.py @@ -23,7 +23,7 @@ def add_backend_cleanup_task(app): backend. If the configured backend requires periodic cleanup this task is also - automatically configured to run every day at midnight (requires + automatically configured to run every day at 4am (requires :program:`celery beat` to be running). """ From fbcff414dc8bcd85890823df2908971892d6073b Mon Sep 17 00:00:00 2001 From: Tom S Date: Fri, 1 May 2015 11:24:56 +0100 Subject: [PATCH 0514/1103] documentation update - CELERY_RESULT_SERIALIZER --- docs/faq.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 86ae18396..84598faa8 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -446,14 +446,17 @@ It is essential that you protect against unauthorized access to your broker, databases and other services transmitting pickled data. -For the task messages you can set the :setting:`CELERY_TASK_SERIALIZER` -setting to "json" or "yaml" instead of pickle. There is -currently no alternative solution for task results (but writing a -custom result backend using JSON is a simple task) - Note that this is not just something you should be aware of with Celery, for example also Django uses pickle for its cache client. +For the task messages you can set the :setting:`CELERY_TASK_SERIALIZER` +setting to "json" or "yaml" instead of pickle. + +Similarly for task results you can set :setting:`CELERY_RESULT_SERIALIZER`. + +For more details of the formats used and the lookup order when +checking which format to use for a task see :ref:`calling-serializers` + Can messages be encrypted? -------------------------- From 04ce9afc4711a3c8f951aaac809c532ee509568a Mon Sep 17 00:00:00 2001 From: Tom S Date: Tue, 12 May 2015 11:44:22 +0100 Subject: [PATCH 0515/1103] update documentation for CELERY_ROUTES copy from history/changelog-2.0.rst since this had a good explanation. --- docs/configuration.rst | 88 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index e1b032924..005f45c1c 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -849,6 +849,8 @@ Also see :ref:`routing-basics` for more information. The default is a queue/exchange/binding key of ``celery``, with exchange type ``direct``. +See also :setting:`CELERY_ROUTES` + .. setting:: CELERY_ROUTES CELERY_ROUTES @@ -856,7 +858,91 @@ CELERY_ROUTES A list of routers, or a single router used to route tasks to queues. When deciding the final destination of a task the routers are consulted -in order. See :ref:`routers` for more information. +in order. + +A router can be specified as either: + +* A router class instances +* A string which provides the path to a router class +* A dict containing router specification. It will be converted to a :class:`celery.routes.MapRoute` instance. + +Examples: + +.. code-block:: python + + CELERY_ROUTES = {"celery.ping": "default", + "mytasks.add": "cpu-bound", + "video.encode": { + "queue": "video", + "exchange": "media" + "routing_key": "media.video.encode"}} + + CELERY_ROUTES = ("myapp.tasks.Router", {"celery.ping": "default}) + +Where ``myapp.tasks.Router`` could be: + +.. code-block:: python + + class Router(object): + + def route_for_task(self, task, args=None, kwargs=None): + if task == "celery.ping": + return "default" + +``route_for_task`` may return a string or a dict. A string then means +it's a queue name in :setting:`CELERY_QUEUES`, a dict means it's a custom route. + +When sending tasks, the routers are consulted in order. The first +router that doesn't return ``None`` is the route to use. The message options +is then merged with the found route settings, where the routers settings +have priority. + +Example if :func:`~celery.execute.apply_async` has these arguments: + +.. code-block:: python + + Task.apply_async(immediate=False, exchange="video", + routing_key="video.compress") + +and a router returns: + +.. code-block:: python + + {"immediate": True, "exchange": "urgent"} + +the final message options will be: + +.. code-block:: python + + immediate=True, exchange="urgent", routing_key="video.compress" + +(and any default message options defined in the +:class:`~celery.task.base.Task` class) + +Values defined in :setting:`CELERY_ROUTES` have precedence over values defined in +:setting:`CELERY_QUEUES` when merging the two. + +With the follow settings: + +.. code-block:: python + + CELERY_QUEUES = {"cpubound": {"exchange": "cpubound", + "routing_key": "cpubound"}} + + CELERY_ROUTES = {"tasks.add": {"queue": "cpubound", + "routing_key": "tasks.add", + "serializer": "json"}} + +The final routing options for ``tasks.add`` will become: + +.. code-block:: python + + {"exchange": "cpubound", + "routing_key": "tasks.add", + "serializer": "json"} + +See :ref:`routers` for more examples. + .. setting:: CELERY_QUEUE_HA_POLICY From c0c1965a18ae4b07bdfdf7d6ddc3dcf8b83132d0 Mon Sep 17 00:00:00 2001 From: Tom S Date: Tue, 12 May 2015 11:46:04 +0100 Subject: [PATCH 0516/1103] update documentation for Task.backend make it clear that it does not default to the string CELERY_RESULT_BACKEND but a class instance which is instantiated from that string --- docs/userguide/tasks.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index bd23625bf..97825f082 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -657,8 +657,9 @@ General .. attribute:: Task.backend - The result store backend to use for this task. Defaults to the - :setting:`CELERY_RESULT_BACKEND` setting. + The result store backend to use for this task. An instance of one of the + backend classes in `celery.backends`. Defaults to `app.backend` which is + defined by the :setting:`CELERY_RESULT_BACKEND` setting. .. attribute:: Task.acks_late From a73f3b92dda22a522b506059ef212baeee1553ef Mon Sep 17 00:00:00 2001 From: squfrans Date: Fri, 22 May 2015 12:08:57 +0200 Subject: [PATCH 0517/1103] adding myself, #2617 --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index bd30be9ac..4ba95a0c3 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -187,3 +187,4 @@ Ilya Georgievsky, 2015/03/31 Fatih Sucu, 2015/04/17 James Pulec, 2015/04/19 Alexander Lebedev, 2015/04/25 +Frantisek Holop, 2015/05/21 From 7acceae388a6338bcdeb60cd77168f69cd8c0ca7 Mon Sep 17 00:00:00 2001 From: Alex Rattray Date: Sun, 24 May 2015 10:56:50 +0530 Subject: [PATCH 0518/1103] Clickable maintainers --- README.rst | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index f4ae4629c..5924d47d4 100644 --- a/README.rst +++ b/README.rst @@ -401,13 +401,19 @@ http://wiki.github.com/celery/celery/ Maintainers =========== -- @ask (primary maintainer) -- @thedrow -- @chrisgogreen -- @PMickael -- @malinoff +- `@ask`_ (primary maintainer) +- `@thedrow`_ +- `@chrisgogreen`_ +- `@PMickael`_ +- `@malinoff`_ - And you? We really need more: https://github.com/celery/celery/issues/2534 +.. _`@ask`: http://github.com/ask +.. _`@thedrow`: http://github.com/thedrow +.. _`@chrisgogreen`: http://github.com/chrisgogreen +.. _`@PMickael`: http://github.com/PMickael +.. _`@malinoff`: http://github.com/malinoff + .. _contributing-short: From a7c168d6df82bcd060e02e647a916d61627959dc Mon Sep 17 00:00:00 2001 From: PMickael Date: Wed, 27 May 2015 21:14:29 +0200 Subject: [PATCH 0519/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 4ba95a0c3..977cd22d5 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -188,3 +188,4 @@ Fatih Sucu, 2015/04/17 James Pulec, 2015/04/19 Alexander Lebedev, 2015/04/25 Frantisek Holop, 2015/05/21 +Feanil Patel, 2015/05/21 From a2fcd666c7c6363f819682144976e7e707a4c874 Mon Sep 17 00:00:00 2001 From: Kirill Pavlov Date: Sat, 30 May 2015 11:11:17 +0800 Subject: [PATCH 0520/1103] Fix typo: rason -> rEason. --- docs/userguide/workers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index e2e7a007b..8fbc6a096 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -318,7 +318,7 @@ the `terminate` option is set. a task is stuck. It's not for terminating the task, it's for terminating the process that is executing the task, and that process may have already started processing another task at the point - when the signal is sent, so for this rason you must never call this + when the signal is sent, so for this reason you must never call this programatically. If `terminate` is set the worker child process processing the task From 06f4a89fd00c5934b4505d3db8fac865ae5e3013 Mon Sep 17 00:00:00 2001 From: Andrei Fokau Date: Wed, 3 Jun 2015 10:17:43 +0200 Subject: [PATCH 0521/1103] Add Python 3.4 to supported versions --- docs/getting-started/introduction.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/introduction.rst b/docs/getting-started/introduction.rst index ca8b480e0..05bb72632 100644 --- a/docs/getting-started/introduction.rst +++ b/docs/getting-started/introduction.rst @@ -39,7 +39,7 @@ What do I need? .. sidebar:: Version Requirements :subtitle: Celery version 3.0 runs on - - Python ❨2.5, 2.6, 2.7, 3.2, 3.3❩ + - Python ❨2.5, 2.6, 2.7, 3.2, 3.3, 3.4❩ - PyPy ❨1.8, 1.9❩ - Jython ❨2.5, 2.7❩. From 7d5a062280578a61cc36e8946be2634cb14adbca Mon Sep 17 00:00:00 2001 From: Colin McIntosh Date: Fri, 5 Jun 2015 13:16:01 -0400 Subject: [PATCH 0522/1103] Added a check for syncing the schedule even if nothing is in the schedule. --- celery/beat.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/celery/beat.py b/celery/beat.py index 21d1316c6..8ba112127 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -504,6 +504,8 @@ def start(self, embedded_process=False): debug('beat: Waking up %s.', humanize_seconds(interval, prefix='in ')) time.sleep(interval) + if self.scheduler.should_sync(): + self.scheduler._do_sync() except (KeyboardInterrupt, SystemExit): self._is_shutdown.set() finally: From bfae26b5eb727fb057a8da74ad0806c29577858b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 8 Jun 2015 10:59:15 -0700 Subject: [PATCH 0523/1103] Common exception base class: CeleryException/CeleryWarning. Closes #2643 --- celery/exceptions.py | 143 ++++++++++++++++++++++++------------------- 1 file changed, 79 insertions(+), 64 deletions(-) diff --git a/celery/exceptions.py b/celery/exceptions.py index ab6501941..96f1bda13 100644 --- a/celery/exceptions.py +++ b/celery/exceptions.py @@ -16,22 +16,33 @@ SoftTimeLimitExceeded, TimeLimitExceeded, WorkerLostError, Terminated, ) -__all__ = ['SecurityError', 'Ignore', 'QueueNotFound', - 'WorkerShutdown', 'WorkerTerminate', - 'ImproperlyConfigured', 'NotRegistered', 'AlreadyRegistered', - 'TimeoutError', 'MaxRetriesExceededError', 'Retry', - 'TaskRevokedError', 'NotConfigured', 'AlwaysEagerIgnored', - 'InvalidTaskError', 'ChordError', 'CPendingDeprecationWarning', - 'CDeprecationWarning', 'FixupWarning', 'DuplicateNodenameWarning', - 'SoftTimeLimitExceeded', 'TimeLimitExceeded', 'WorkerLostError', - 'Terminated'] +__all__ = [ + 'CeleryError', 'CeleryWarning', 'TaskPredicate', + 'SecurityError', 'Ignore', 'QueueNotFound', + 'WorkerShutdown', 'WorkerTerminate', + 'ImproperlyConfigured', 'NotRegistered', 'AlreadyRegistered', + 'TimeoutError', 'MaxRetriesExceededError', 'Retry', + 'TaskRevokedError', 'NotConfigured', 'AlwaysEagerIgnored', + 'InvalidTaskError', 'ChordError', 'CPendingDeprecationWarning', + 'CDeprecationWarning', 'FixupWarning', 'DuplicateNodenameWarning', + 'SoftTimeLimitExceeded', 'TimeLimitExceeded', 'WorkerLostError', + 'Terminated', +] UNREGISTERED_FMT = """\ Task of kind {0} is not registered, please make sure it's imported.\ """ -class SecurityError(Exception): +class CeleryError(Exception): + pass + + +class CeleryWarning(UserWarning): + pass + + +class SecurityError(CeleryError): """Security related exceptions. Handle with care. @@ -39,11 +50,55 @@ class SecurityError(Exception): """ -class Ignore(Exception): +class TaskPredicate(CeleryError): + pass + + +class Retry(TaskPredicate): + """The task is to be retried later.""" + + #: Optional message describing context of retry. + message = None + + #: Exception (if any) that caused the retry to happen. + exc = None + + #: Time of retry (ETA), either :class:`numbers.Real` or + #: :class:`~datetime.datetime`. + when = None + + def __init__(self, message=None, exc=None, when=None, **kwargs): + from kombu.utils.encoding import safe_repr + self.message = message + if isinstance(exc, string_t): + self.exc, self.excs = None, exc + else: + self.exc, self.excs = exc, safe_repr(exc) if exc else None + self.when = when + Exception.__init__(self, exc, when, **kwargs) + + def humanize(self): + if isinstance(self.when, numbers.Real): + return 'in {0.when}s'.format(self) + return 'at {0.when}'.format(self) + + def __str__(self): + if self.message: + return self.message + if self.excs: + return 'Retry {0}: {1}'.format(self.humanize(), self.excs) + return 'Retry {0}'.format(self.humanize()) + + def __reduce__(self): + return self.__class__, (self.message, self.excs, self.when) +RetryTaskError = Retry # XXX compat + + +class Ignore(TaskPredicate): """A task can raise this to ignore doing state updates.""" -class Reject(Exception): +class Reject(TaskPredicate): """A task can raise this if it wants to reject/requeue the message.""" def __init__(self, reason=None, requeue=False): @@ -72,86 +127,46 @@ class ImproperlyConfigured(ImportError): """Celery is somehow improperly configured.""" -class NotRegistered(KeyError): +class NotRegistered(KeyError, CeleryError): """The task is not registered.""" def __repr__(self): return UNREGISTERED_FMT.format(self) -class AlreadyRegistered(Exception): +class AlreadyRegistered(CeleryError): """The task is already registered.""" -class TimeoutError(Exception): +class TimeoutError(CeleryError): """The operation timed out.""" -class MaxRetriesExceededError(Exception): +class MaxRetriesExceededError(CeleryError): """The tasks max restart limit has been exceeded.""" -class Retry(Exception): - """The task is to be retried later.""" - - #: Optional message describing context of retry. - message = None - - #: Exception (if any) that caused the retry to happen. - exc = None - - #: Time of retry (ETA), either :class:`numbers.Real` or - #: :class:`~datetime.datetime`. - when = None - - def __init__(self, message=None, exc=None, when=None, **kwargs): - from kombu.utils.encoding import safe_repr - self.message = message - if isinstance(exc, string_t): - self.exc, self.excs = None, exc - else: - self.exc, self.excs = exc, safe_repr(exc) if exc else None - self.when = when - Exception.__init__(self, exc, when, **kwargs) - - def humanize(self): - if isinstance(self.when, numbers.Real): - return 'in {0.when}s'.format(self) - return 'at {0.when}'.format(self) - - def __str__(self): - if self.message: - return self.message - if self.excs: - return 'Retry {0}: {1}'.format(self.humanize(), self.excs) - return 'Retry {0}'.format(self.humanize()) - - def __reduce__(self): - return self.__class__, (self.message, self.excs, self.when) -RetryTaskError = Retry # XXX compat - - -class TaskRevokedError(Exception): +class TaskRevokedError(CeleryError): """The task has been revoked, so no result available.""" -class NotConfigured(UserWarning): +class NotConfigured(CeleryWarning): """Celery has not been configured, as no config module has been found.""" -class AlwaysEagerIgnored(UserWarning): +class AlwaysEagerIgnored(CeleryWarning): """send_task ignores CELERY_ALWAYS_EAGER option""" -class InvalidTaskError(Exception): +class InvalidTaskError(CeleryError): """The task has invalid data or is not properly constructed.""" -class IncompleteStream(Exception): +class IncompleteStream(CeleryError): """Found the end of a stream of data, but the data is not yet complete.""" -class ChordError(Exception): +class ChordError(CeleryError): """A task part of the chord raised an exception.""" @@ -163,9 +178,9 @@ class CDeprecationWarning(DeprecationWarning): pass -class FixupWarning(UserWarning): +class FixupWarning(CeleryWarning): pass -class DuplicateNodenameWarning(UserWarning): +class DuplicateNodenameWarning(CeleryWarning): """Multiple workers are using the same nodename.""" From dffb61c4d99f1ce5817be267104e9810e88391ee Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 8 Jun 2015 17:09:15 -0700 Subject: [PATCH 0524/1103] Attempts to fix tests --- celery/app/task.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index 2ca92e094..aa45e71fb 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -471,16 +471,14 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, if self.__self__ is not None: args = args if isinstance(args, tuple) else tuple(args or ()) args = (self.__self__, ) + args - shadow = shadow or self.shadow_name(args, kwargs, final_options) - final_options = self._get_exec_options() - if options: - final_options = dict(final_options, **options) + preopts = self._get_exec_options() + options = dict(preopts, **options) if options else preopts return app.send_task( self.name, args, kwargs, task_id=task_id, producer=producer, link=link, link_error=link_error, result_cls=self.AsyncResult, - shadow=shadow, - **final_options + shadow=shadow or self.shadow_name(args, kwargs, options), + **options ) def shadow_name(self, args, kwargs, options): From 4c14eeabfe1c4609493d4c0b2cfc4c3f430f51ba Mon Sep 17 00:00:00 2001 From: allenling Date: Tue, 9 Jun 2015 16:53:51 +0800 Subject: [PATCH 0525/1103] explicit to invoke Settings.__setitem__ to set Settings.change --- celery/app/base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/celery/app/base.py b/celery/app/base.py index 2f40a509b..959087eab 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -33,7 +33,7 @@ _announce_app_finalized, ) from celery.exceptions import AlwaysEagerIgnored, ImproperlyConfigured -from celery.five import values +from celery.five import items, values from celery.loaders import get_loader_cls from celery.local import PromiseProxy, maybe_evaluate from celery.utils import gen_task_name @@ -508,6 +508,10 @@ def _load_config(self): while pending_beat: pargs, pkwargs = pending_beat.popleft() self._add_periodic_task(*pargs, **pkwargs) + # Settings.__setitem__ method, set Settings.change + if self._preconf: + for key, value in items(self._preconf): + setattr(s, key, value) self.on_after_configure.send(sender=self, source=s) return s From cfc9450e19b74614561c64554e4d071027894b5d Mon Sep 17 00:00:00 2001 From: Vladimir Gorbunov Date: Fri, 12 Jun 2015 19:22:25 +0300 Subject: [PATCH 0526/1103] Add CELERY_EMAIL_CHARSET option This option allows setting charset for outgoing celery emails. --- celery/app/base.py | 1 + celery/app/defaults.py | 1 + celery/loaders/base.py | 5 +++-- docs/configuration.rst | 8 ++++++++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 2f40a509b..6fe575ac2 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -465,6 +465,7 @@ def mail_admins(self, subject, body, fail_silently=False): timeout=conf.EMAIL_TIMEOUT, use_ssl=conf.EMAIL_USE_SSL, use_tls=conf.EMAIL_USE_TLS, + charset=conf.EMAIL_CHARSET, ) def select_queues(self, queues=None): diff --git a/celery/app/defaults.py b/celery/app/defaults.py index ca819eb46..d217032b6 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -220,6 +220,7 @@ def __repr__(self): 'TIMEOUT': Option(2, type='float'), 'USE_SSL': Option(False, type='bool'), 'USE_TLS': Option(False, type='bool'), + 'CHARSET': Option('us-ascii'), }, 'SERVER_EMAIL': Option('celery@localhost'), 'ADMINS': Option((), type='tuple'), diff --git a/celery/loaders/base.py b/celery/loaders/base.py index d8e99736c..b1a1f6366 100644 --- a/celery/loaders/base.py +++ b/celery/loaders/base.py @@ -224,10 +224,11 @@ def getarg(arg): def mail_admins(self, subject, body, fail_silently=False, sender=None, to=None, host=None, port=None, user=None, password=None, timeout=None, - use_ssl=False, use_tls=False): + use_ssl=False, use_tls=False, charset='us-ascii'): message = self.mail.Message(sender=sender, to=to, subject=safe_str(subject), - body=safe_str(body)) + body=safe_str(body), + charset=charset) mailer = self.mail.Mailer(host=host, port=port, user=user, password=password, timeout=timeout, use_ssl=use_ssl, diff --git a/docs/configuration.rst b/docs/configuration.rst index 005f45c1c..14a1e45ce 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1633,6 +1633,14 @@ to the SMTP server when sending emails. The default is 2 seconds. +EMAIL_CHARSET +~~~~~~~~~~~~~ + +Charset for outgoing emails. Default is "us-ascii". + +.. setting:: EMAIL_CHARSET + + .. _conf-example-error-mail-config: Example E-Mail configuration From b815f57de34feb796c844e30b12a86ae7509c39c Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 14 Jun 2015 12:45:22 +0300 Subject: [PATCH 0527/1103] Mentioned that CELERY_EMAIL_CHARSET was added on version 3.2.0. --- docs/configuration.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuration.rst b/docs/configuration.rst index 14a1e45ce..0f821f398 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1635,6 +1635,7 @@ The default is 2 seconds. EMAIL_CHARSET ~~~~~~~~~~~~~ +.. versionadded:: 3.2.0 Charset for outgoing emails. Default is "us-ascii". From 3e80590cec7de700eb8f565f14bd6750eda70e10 Mon Sep 17 00:00:00 2001 From: Nat Williams Date: Fri, 12 Jun 2015 12:06:12 -0500 Subject: [PATCH 0528/1103] consumer should respect BROKER_CONNECTION_RETRY --- celery/tests/worker/test_consumer.py | 7 +++++++ celery/worker/consumer.py | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/celery/tests/worker/test_consumer.py b/celery/tests/worker/test_consumer.py index db2d47eff..f3b36435c 100644 --- a/celery/tests/worker/test_consumer.py +++ b/celery/tests/worker/test_consumer.py @@ -115,6 +115,13 @@ def se(*args, **kwargs): c.start() sleep.assert_called_with(1) + def test_no_retry_raises_error(self): + self.app.conf.BROKER_CONNECTION_RETRY = False + c = self.get_consumer() + c.blueprint.start.side_effect = socket.error() + with self.assertRaises(socket.error): + c.start() + def _closer(self, c): def se(*args, **kwargs): c.blueprint.state = CLOSE diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 7bf4576ca..356617772 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -277,6 +277,10 @@ def start(self): try: blueprint.start(self) except self.connection_errors as exc: + # If we're not retrying connections, no need to catch + # connection errors + if not self.app.conf.BROKER_CONNECTION_RETRY: + raise if isinstance(exc, OSError) and exc.errno == errno.EMFILE: raise # Too many open files maybe_shutdown() From d62af7009e02cf51650ef04b9895df6a72703843 Mon Sep 17 00:00:00 2001 From: Michael Duane Mooring Date: Thu, 18 Jun 2015 18:46:48 -0400 Subject: [PATCH 0529/1103] django-celery link to it's docs This needs to be linked. The current ReadTheDocs don't tell of all the other things that need to happen to get django-celery working like `import djcelery djcelery.setup_loader()` in `settings.py` for example. --- docs/django/first-steps-with-django.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/django/first-steps-with-django.rst b/docs/django/first-steps-with-django.rst index ac33d7da2..ed259cd40 100644 --- a/docs/django/first-steps-with-django.rst +++ b/docs/django/first-steps-with-django.rst @@ -136,7 +136,7 @@ concrete app instance: Using the Django ORM/Cache as a result backend. ----------------------------------------------- -The ``django-celery`` library defines result backends that +The [``django-celery``](https://github.com/celery/django-celery) library defines result backends that uses the Django ORM and Django Cache frameworks. To use this with your project you need to follow these four steps: From f8bf7dd0180244757a01e584b6c840b2dd362048 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Fri, 19 Jun 2015 02:57:39 +0300 Subject: [PATCH 0530/1103] Always make timezones aware even if UTC is disabled. Fixes #943. --- celery/schedules.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/celery/schedules.py b/celery/schedules.py index 4b3ffeaa1..51074d76e 100644 --- a/celery/schedules.py +++ b/celery/schedules.py @@ -134,9 +134,7 @@ def is_due(self, last_run_at): return schedstate(is_due=False, next=remaining_s) def maybe_make_aware(self, dt): - if self.utc_enabled: - return maybe_make_aware(dt, self.tz) - return dt + return maybe_make_aware(dt, self.tz) def __repr__(self): return ''.format(self) From d500925158125eef11cad5cd880cab8954534fc3 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Fri, 19 Jun 2015 23:55:33 +0300 Subject: [PATCH 0531/1103] Fixed the tests. --- celery/tests/app/test_beat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/tests/app/test_beat.py b/celery/tests/app/test_beat.py index 40b8c8589..0718e2a77 100644 --- a/celery/tests/app/test_beat.py +++ b/celery/tests/app/test_beat.py @@ -521,7 +521,7 @@ def test_maybe_make_aware(self): self.assertTrue(d.tzinfo) x.utc_enabled = False d2 = x.maybe_make_aware(datetime.utcnow()) - self.assertIsNone(d2.tzinfo) + self.assertTrue(d2.tzinfo) def test_to_local(self): x = schedule(10, app=self.app) From 377178c76a0a2df9708a98560e99e3d80de11636 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 19 Jun 2015 15:46:44 -0700 Subject: [PATCH 0532/1103] Fixes pypy3 CI --- requirements/test-pypy3.txt | 1 + tox.ini | 1 + 2 files changed, 2 insertions(+) create mode 100644 requirements/test-pypy3.txt diff --git a/requirements/test-pypy3.txt b/requirements/test-pypy3.txt new file mode 100644 index 000000000..932a8957f --- /dev/null +++ b/requirements/test-pypy3.txt @@ -0,0 +1 @@ +mock diff --git a/tox.ini b/tox.ini index 4977e8e4b..c8c6851eb 100644 --- a/tox.ini +++ b/tox.ini @@ -55,6 +55,7 @@ commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} basepython = pypy3 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt + -r{toxinidir}/requirements/test-pypy3.txt -r{toxinidir}/requirements/test-ci.txt -r{toxinidir}/requirements/dev.txt setenv = C_DEBUG_TEST = 1 From d87c5700d0d89984b6c80ed7e2dc16dfd91a1b7a Mon Sep 17 00:00:00 2001 From: Mayflower Date: Sat, 20 Jun 2015 11:54:55 +0800 Subject: [PATCH 0533/1103] Add another tornado-celery --- README.rst | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/README.rst b/README.rst index 5924d47d4..3391e16be 100644 --- a/README.rst +++ b/README.rst @@ -166,19 +166,19 @@ Framework Integration Celery is easy to integrate with web frameworks, some of which even have integration packages: - +--------------------+------------------------+ - | `Django`_ | not needed | - +--------------------+------------------------+ - | `Pyramid`_ | `pyramid_celery`_ | - +--------------------+------------------------+ - | `Pylons`_ | `celery-pylons`_ | - +--------------------+------------------------+ - | `Flask`_ | not needed | - +--------------------+------------------------+ - | `web2py`_ | `web2py-celery`_ | - +--------------------+------------------------+ - | `Tornado`_ | `tornado-celery`_ | - +--------------------+------------------------+ + +--------------------+----------------------------------------------------+ + | `Django`_ | not needed | + +--------------------+----------------------------------------------------+ + | `Pyramid`_ | `pyramid_celery`_ | + +--------------------+----------------------------------------------------+ + | `Pylons`_ | `celery-pylons`_ | + +--------------------+----------------------------------------------------+ + | `Flask`_ | not needed | + +--------------------+----------------------------------------------------+ + | `web2py`_ | `web2py-celery`_ | + +--------------------+----------------------------------------------------+ + | `Tornado`_ | `tornado-celery`_ | `another tornado-celery`_ | + +--------------------+----------------------------------------------------+ The integration packages are not strictly necessary, but they can make development easier, and sometimes they add important hooks like closing @@ -196,6 +196,7 @@ database connections at ``fork``. .. _`web2py-celery`: http://code.google.com/p/web2py-celery/ .. _`Tornado`: http://www.tornadoweb.org/ .. _`tornado-celery`: http://github.com/mher/tornado-celery/ +.. _`another tornado-celery`: https://github.com/mayflaver/tornado-celery .. _celery-documentation: From a84265f8bb6b9316b5e29e3153fc1a337c61ca22 Mon Sep 17 00:00:00 2001 From: Smirl Date: Tue, 16 Jun 2015 11:20:53 +0100 Subject: [PATCH 0534/1103] #2654 - couchbase - changing the key_t to str_t Couchbase python API needs to have str type not bytes for the keys. We use kombu.utils.encoding.str_t to make it compatible with Python 2 and 3 --- celery/backends/couchbase.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/celery/backends/couchbase.py b/celery/backends/couchbase.py index 9381fcfc6..d94960ed3 100644 --- a/celery/backends/couchbase.py +++ b/celery/backends/couchbase.py @@ -17,6 +17,7 @@ except ImportError: Couchbase = Connection = NotFoundError = None # noqa +from kombu.utils.encoding import str_t from kombu.utils.url import _parse_url from celery.exceptions import ImproperlyConfigured @@ -38,6 +39,9 @@ class CouchBaseBackend(KeyValueStoreBackend): timeout = 2.5 transcoder = None # supports_autoexpire = False + + # Use str as couchbase key not bytes + key_t = str_t def __init__(self, url=None, *args, **kwargs): """Initialize CouchBase backend instance. From 8049f5b35e232f826584809725c5f0bb61b041c2 Mon Sep 17 00:00:00 2001 From: Alex Williams Date: Sun, 21 Jun 2015 22:14:23 +0100 Subject: [PATCH 0535/1103] #2654 - couchbase - add tests for get_key_for methods --- celery/tests/backends/test_couchbase.py | 64 ++++++++++++++++++++----- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/celery/tests/backends/test_couchbase.py b/celery/tests/backends/test_couchbase.py index 3dc6aadd0..94f72f5c4 100644 --- a/celery/tests/backends/test_couchbase.py +++ b/celery/tests/backends/test_couchbase.py @@ -1,5 +1,9 @@ +"""Tests for the CouchBaseBackend.""" + from __future__ import absolute_import +from kombu.utils.encoding import str_t + from celery.backends import couchbase as module from celery.backends.couchbase import CouchBaseBackend from celery.exceptions import ImproperlyConfigured @@ -18,32 +22,42 @@ class test_CouchBaseBackend(AppCase): + """CouchBaseBackend TestCase.""" + def setup(self): + """Skip the test if couchbase cannot be imported.""" if couchbase is None: raise SkipTest('couchbase is not installed.') self.backend = CouchBaseBackend(app=self.app) def test_init_no_couchbase(self): - """test init no couchbase raises""" - prev, module.couchbase = module.couchbase, None + """ + Test init no couchbase raises. + + If celery.backends.couchbase cannot import the couchbase client, it + sets the couchbase.Couchbase to None and then handles this in the + CouchBaseBackend __init__ method. + """ + prev, module.Couchbase = module.Couchbase, None try: with self.assertRaises(ImproperlyConfigured): CouchBaseBackend(app=self.app) finally: - module.couchbase = prev + module.Couchbase = prev def test_init_no_settings(self): - """test init no settings""" + """Test init no settings.""" self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = [] with self.assertRaises(ImproperlyConfigured): CouchBaseBackend(app=self.app) def test_init_settings_is_None(self): - """Test init settings is None""" + """Test init settings is None.""" self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = None CouchBaseBackend(app=self.app) def test_get_connection_connection_exists(self): + """Test _get_connection works.""" with patch('couchbase.connection.Connection') as mock_Connection: self.backend._connection = sentinel._connection @@ -53,12 +67,13 @@ def test_get_connection_connection_exists(self): self.assertFalse(mock_Connection.called) def test_get(self): - """test_get + """ + Test get method. CouchBaseBackend.get should return and take two params db conn to couchbase is mocked. - TODO Should test on key not exists + TODO Should test on key not exists """ self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = {} x = CouchBaseBackend(app=self.app) @@ -70,11 +85,11 @@ def test_get(self): x._connection.get.assert_called_once_with('1f3fab') def test_set(self): - """test_set + """ + Test set method. CouchBaseBackend.set should return None and take two params db conn to couchbase is mocked. - """ self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = None x = CouchBaseBackend(app=self.app) @@ -84,12 +99,13 @@ def test_set(self): self.assertIsNone(x.set(sentinel.key, sentinel.value)) def test_delete(self): - """test_delete + """ + Test delete method. CouchBaseBackend.delete should return and take two params db conn to couchbase is mocked. - TODO Should test on key not exists + TODO Should test on key not exists. """ self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = {} x = CouchBaseBackend(app=self.app) @@ -101,9 +117,10 @@ def test_delete(self): x._connection.delete.assert_called_once_with('1f3fab') def test_config_params(self): - """test_config_params + """ + Test config params are correct. - celery.conf.CELERY_COUCHBASE_BACKEND_SETTINGS is properly set + celery.conf.CELERY_COUCHBASE_BACKEND_SETTINGS is properly set. """ self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = { 'bucket': 'mycoolbucket', @@ -120,12 +137,14 @@ def test_config_params(self): self.assertEqual(x.port, 1234) def test_backend_by_url(self, url='couchbase://myhost/mycoolbucket'): + """Test that a CouchBaseBackend is loaded from the couchbase url.""" from celery.backends.couchbase import CouchBaseBackend backend, url_ = backends.get_backend_by_url(url, self.app.loader) self.assertIs(backend, CouchBaseBackend) self.assertEqual(url_, url) def test_backend_params_by_url(self): + """Test config params are correct from config url.""" url = 'couchbase://johndoe:mysecret@myhost:123/mycoolbucket' with self.Celery(backend=url) as app: x = app.backend @@ -134,3 +153,22 @@ def test_backend_params_by_url(self): self.assertEqual(x.username, 'johndoe') self.assertEqual(x.password, 'mysecret') self.assertEqual(x.port, 123) + + def test_correct_key_types(self): + """ + Test that the key is the correct type for the couchbase python API. + + We check that get_key_for_task, get_key_for_chord, and + get_key_for_group always returns a python string. Need to use str_t + for cross Python reasons. + """ + keys = [ + self.backend.get_key_for_task('task_id', bytes('key')), + self.backend.get_key_for_chord('group_id', bytes('key')), + self.backend.get_key_for_group('group_id', bytes('key')), + self.backend.get_key_for_task('task_id', 'key'), + self.backend.get_key_for_chord('group_id', 'key'), + self.backend.get_key_for_group('group_id', 'key'), + ] + for key in keys: + self.assertIsInstance(key, str_t) From c7201e2a42f3339415d9a0741e1352a54fdbcfe9 Mon Sep 17 00:00:00 2001 From: Mher Movsisyan Date: Sun, 21 Jun 2015 18:20:18 -0700 Subject: [PATCH 0536/1103] Fix punctuation --- docs/userguide/monitoring.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index 7633f5179..5ba493b5e 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -188,16 +188,16 @@ Features - Real-time monitoring using Celery Events - - Task progress and history. + - Task progress and history - Ability to show task details (arguments, start time, runtime, and more) - Graphs and statistics - Remote Control - - View worker status and statistics. - - Shutdown and restart worker instances. - - Control worker pool size and autoscale settings. - - View and modify the queues a worker instance consumes from. + - View worker status and statistics + - Shutdown and restart worker instances + - Control worker pool size and autoscale settings + - View and modify the queues a worker instance consumes from - View currently running tasks - View scheduled tasks (ETA/countdown) - View reserved and revoked tasks From f556b6cbcdba64c126d3edd993dd422705ea17b1 Mon Sep 17 00:00:00 2001 From: Dieter Adriaenssens Date: Mon, 22 Jun 2015 19:47:33 +0200 Subject: [PATCH 0537/1103] fix grammar --- docs/userguide/workers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index 8fbc6a096..75cdf72f6 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -887,7 +887,7 @@ You can get a list of tasks waiting to be scheduled by using Dump of reserved tasks ---------------------- -Reserved tasks are tasks that has been received, but is still waiting to be +Reserved tasks are tasks that have been received, but are still waiting to be executed. You can get a list of these using From 1a41806f66a9081a8176178ed74bf3b408269447 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 22 Jun 2015 13:18:28 -0700 Subject: [PATCH 0538/1103] Advocate use of the rpc:// backend over amqp --- docs/configuration.rst | 106 ++++++++++++------ .../first-steps-with-celery.rst | 8 +- docs/userguide/tasks.rst | 36 ++---- 3 files changed, 82 insertions(+), 68 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 0f821f398..6ed8e206e 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -189,10 +189,18 @@ The backend used to store task results (tombstones). Disabled by default. Can be one of the following: +* rpc + Send results back as AMQP messages + See :ref:`conf-rpc-result-backend`. + * database Use a relational database supported by `SQLAlchemy`_. See :ref:`conf-database-result-backend`. +* redis + Use `Redis`_ to store the results. + See :ref:`conf-redis-result-backend`. + * cache Use `memcached`_ to store the results. See :ref:`conf-cache-result-backend`. @@ -201,14 +209,6 @@ Can be one of the following: Use `MongoDB`_ to store the results. See :ref:`conf-mongodb-result-backend`. -* redis - Use `Redis`_ to store the results. - See :ref:`conf-redis-result-backend`. - -* amqp - Send results back as AMQP messages - See :ref:`conf-amqp-result-backend`. - * cassandra Use `Cassandra`_ to store the results. See :ref:`conf-cassandra-result-backend`. @@ -225,6 +225,10 @@ Can be one of the following: Use `CouchDB`_ to store the results. See :ref:`conf-couchdb-result-backend`. +* amqp + Older AMQP backend (badly) emulating a database-based backend. + See :ref:`conf-amqp-result-backend`. + .. warning: While the AMQP result backend is very efficient, you must make sure @@ -341,35 +345,12 @@ you to customize the table names: 'group': 'myapp_groupmeta', } -.. _conf-amqp-result-backend: - -AMQP backend settings ---------------------- - -.. note:: +.. _conf-rpc-result-backend: - The AMQP backend requires RabbitMQ 1.1.0 or higher to automatically - expire results. If you are running an older version of RabbitMQ - you should disable result expiration like this: - - CELERY_TASK_RESULT_EXPIRES = None - -.. setting:: CELERY_RESULT_EXCHANGE - -CELERY_RESULT_EXCHANGE -~~~~~~~~~~~~~~~~~~~~~~ - -Name of the exchange to publish results in. Default is `celeryresults`. - -.. setting:: CELERY_RESULT_EXCHANGE_TYPE - -CELERY_RESULT_EXCHANGE_TYPE -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The exchange type of the result exchange. Default is to use a `direct` -exchange. +RPC backend settings +-------------------- -.. setting:: CELERY_RESULT_PERSISTENT +.. _conf-amqp-result-backend: CELERY_RESULT_PERSISTENT ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -383,8 +364,9 @@ Example configuration .. code-block:: python - CELERY_RESULT_BACKEND = 'amqp' - CELERY_TASK_RESULT_EXPIRES = 18000 # 5 hours. + CELERY_RESULT_BACKEND = 'rpc://' + CELERY_RESULT_PERSISTENT = False + .. _conf-cache-result-backend: @@ -821,6 +803,56 @@ The URL is formed out of the following parts: The default container the CouchDB server is writing to. Defaults to ``default``. +AMQP backend settings +--------------------- + +.. admonition:: Do not use in production. + + This is the old AMQP result backend that creates one queue per task, + if you want to send results back as message please consider using the + RPC backend instead, or if you need the results to be persistent + use a result backend designed for that purpose (e.g. Redis, or a database). + +.. note:: + + The AMQP backend requires RabbitMQ 1.1.0 or higher to automatically + expire results. If you are running an older version of RabbitMQ + you should disable result expiration like this: + + CELERY_TASK_RESULT_EXPIRES = None + +.. setting:: CELERY_RESULT_EXCHANGE + +CELERY_RESULT_EXCHANGE +~~~~~~~~~~~~~~~~~~~~~~ + +Name of the exchange to publish results in. Default is `celeryresults`. + +.. setting:: CELERY_RESULT_EXCHANGE_TYPE + +CELERY_RESULT_EXCHANGE_TYPE +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The exchange type of the result exchange. Default is to use a `direct` +exchange. + +.. setting:: CELERY_RESULT_PERSISTENT + +CELERY_RESULT_PERSISTENT +~~~~~~~~~~~~~~~~~~~~~~~~ + +If set to :const:`True`, result messages will be persistent. This means the +messages will not be lost after a broker restart. The default is for the +results to be transient. + +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + CELERY_RESULT_BACKEND = 'amqp' + CELERY_TASK_RESULT_EXPIRES = 18000 # 5 hours. + .. _conf-messaging: diff --git a/docs/getting-started/first-steps-with-celery.rst b/docs/getting-started/first-steps-with-celery.rst index d02097ac8..91d3e60ab 100644 --- a/docs/getting-started/first-steps-with-celery.rst +++ b/docs/getting-started/first-steps-with-celery.rst @@ -223,12 +223,12 @@ built-in result backends to choose from: `SQLAlchemy`_/`Django`_ ORM, .. _`SQLAlchemy`: http://www.sqlalchemy.org/ .. _`Django`: http://djangoproject.com -For this example you will use the `amqp` result backend, which sends states -as messages. The backend is specified via the ``backend`` argument to +For this example you will use the `rpc` result backend, which sends states +back as transient messages. The backend is specified via the ``backend`` argument to :class:`@Celery`, (or via the :setting:`CELERY_RESULT_BACKEND` setting if you choose to use a configuration module):: - app = Celery('tasks', backend='amqp', broker='amqp://') + app = Celery('tasks', backend='rpc://', broker='amqp://') Or if you want to use Redis as the result backend, but still use RabbitMQ as the message broker (a popular combination):: @@ -333,7 +333,7 @@ current directory or on the Python path, it could look like this: .. code-block:: python BROKER_URL = 'amqp://' - CELERY_RESULT_BACKEND = 'amqp://' + CELERY_RESULT_BACKEND = 'rpc://' CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 97825f082..fe40668ac 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -733,48 +733,31 @@ Result Backends If you want to keep track of tasks or need the return values, then Celery must store or send the states somewhere so that they can be retrieved later. There are several built-in result backends to choose from: SQLAlchemy/Django ORM, -Memcached, RabbitMQ (amqp), MongoDB, and Redis -- or you can define your own. +Memcached, RabbitMQ/QPid (rpc), MongoDB, and Redis -- or you can define your own. No backend works well for every use case. You should read about the strengths and weaknesses of each backend, and choose the most appropriate for your needs. - .. seealso:: :ref:`conf-result-backend` -RabbitMQ Result Backend -~~~~~~~~~~~~~~~~~~~~~~~ +RPC Result Backend (RabbitMQ/QPid) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The RabbitMQ result backend (amqp) is special as it does not actually *store* +The RPC result backend (`rpc://`) is special as it does not actually *store* the states, but rather sends them as messages. This is an important difference as it -means that a result *can only be retrieved once*; If you have two processes -waiting for the same result, one of the processes will never receive the -result! +means that a result *can only be retrieved once*, and *only by the client +that initiated the task*. Two different processes can not wait for the same result. Even with that limitation, it is an excellent choice if you need to receive state changes in real-time. Using messaging means the client does not have to poll for new states. -There are several other pitfalls you should be aware of when using the -RabbitMQ result backend: - -* Every new task creates a new queue on the server, with thousands of tasks - the broker may be overloaded with queues and this will affect performance in - negative ways. If you're using RabbitMQ then each queue will be a separate - Erlang process, so if you're planning to keep many results simultaneously you - may have to increase the Erlang process limit, and the maximum number of file - descriptors your OS allows. - -* Old results will be cleaned automatically, based on the - :setting:`CELERY_TASK_RESULT_EXPIRES` setting. By default this is set to - expire after 1 day: if you have a very busy cluster you should lower - this value. - -For a list of options supported by the RabbitMQ result backend, please see -:ref:`conf-amqp-result-backend`. - +The messages are transient (non-persistent) by default, so the results will +disappear if the broker restarts. You can configure the result backend to send +persistent messages using the :setting:`CELERY_RESULT_PERSISTENT` setting. Database Result Backend ~~~~~~~~~~~~~~~~~~~~~~~ @@ -794,7 +777,6 @@ limitations. transaction is committed. It is recommended that you change to the `READ-COMMITTED` isolation level. - .. _task-builtin-states: Built-in States From b4667a2561003f8135bb7934242cef0bbe1c8a32 Mon Sep 17 00:00:00 2001 From: Raghuram Srinivasan Date: Wed, 24 Jun 2015 10:35:29 -0700 Subject: [PATCH 0539/1103] Might fix worker hanging for redis --- celery/backends/redis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 236ac3871..26909a1b3 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -131,6 +131,7 @@ def _params_from_url(self, url, defaults): # Query parameters override other parameters connparams.update(query) + connparams.update(socket_timeout=5) return connparams def get(self, key): From 75ab5c3656c5fd04e6d86506cd4995a363813edd Mon Sep 17 00:00:00 2001 From: Raghuram Srinivasan Date: Wed, 24 Jun 2015 18:30:53 -0700 Subject: [PATCH 0540/1103] Better way for setting it through CELERY_REDIS_SOCKET_TIMEOUT --- celery/backends/redis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 26909a1b3..6592a1c0c 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -85,6 +85,7 @@ def _get(key): 'port': _get('PORT') or 6379, 'db': _get('DB') or 0, 'password': _get('PASSWORD'), + 'socket_timeout': _get('SOCKET_TIMEOUT'), 'max_connections': self.max_connections, } if url: @@ -131,7 +132,6 @@ def _params_from_url(self, url, defaults): # Query parameters override other parameters connparams.update(query) - connparams.update(socket_timeout=5) return connparams def get(self, key): From cab679be858cf1cbdbe1b484395d6544589fe8ea Mon Sep 17 00:00:00 2001 From: TakesxiSximada Date: Mon, 29 Jun 2015 18:07:16 +0900 Subject: [PATCH 0541/1103] fixes docstring typo --- celery/backends/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 50dec0c0a..c96698d32 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -118,7 +118,7 @@ def mark_as_done(self, task_id, result, request=None): status=states.SUCCESS, request=request) def mark_as_failure(self, task_id, exc, traceback=None, request=None): - """Mark task as executed with failure. Stores the execption.""" + """Mark task as executed with failure. Stores the exception.""" return self.store_result(task_id, exc, status=states.FAILURE, traceback=traceback, request=request) From 8fc32a58240c74fe9a2d993152f369935ba13722 Mon Sep 17 00:00:00 2001 From: PMickael Date: Fri, 3 Jul 2015 10:34:55 +0200 Subject: [PATCH 0542/1103] [Re-Fix] Protocol shadow exception when self.__self__ is None (Erase with merge #dffb61c) --- celery/app/task.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/app/task.py b/celery/app/task.py index aa45e71fb..44f2ab004 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -471,13 +471,14 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, if self.__self__ is not None: args = args if isinstance(args, tuple) else tuple(args or ()) args = (self.__self__, ) + args + shadow = shadow or self.shadow_name(args, kwargs, final_options) preopts = self._get_exec_options() options = dict(preopts, **options) if options else preopts return app.send_task( self.name, args, kwargs, task_id=task_id, producer=producer, link=link, link_error=link_error, result_cls=self.AsyncResult, - shadow=shadow or self.shadow_name(args, kwargs, options), + shadow=shadow, **options ) From fe3838cb006de3606fd38d03ac168f09dc5b0507 Mon Sep 17 00:00:00 2001 From: Philip Garnero Date: Wed, 8 Jul 2015 18:32:54 +0200 Subject: [PATCH 0543/1103] fix some weird behavior with scaling do not update keepalive when scaling down --- celery/tests/worker/test_autoscale.py | 6 +++--- celery/worker/autoscale.py | 21 ++++++++++----------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/celery/tests/worker/test_autoscale.py b/celery/tests/worker/test_autoscale.py index 45ea488cc..21226ab6d 100644 --- a/celery/tests/worker/test_autoscale.py +++ b/celery/tests/worker/test_autoscale.py @@ -107,7 +107,7 @@ def test_body(self): state.reserved_requests.clear() x.body() self.assertEqual(x.pool.num_processes, 10) - x._last_action = monotonic() - 10000 + x._last_scale_up = monotonic() - 10000 x.body() self.assertEqual(x.pool.num_processes, 3) self.assertTrue(worker.consumer._update_prefetch_count.called) @@ -141,7 +141,7 @@ def test_shrink_raises_ValueError(self, debug): worker = Mock(name='worker') x = autoscale.Autoscaler(self.pool, 10, 3, worker=worker) x.scale_up(3) - x._last_action = monotonic() - 10000 + x._last_scale_up = monotonic() - 10000 x.pool.shrink_raises_ValueError = True x.scale_down(1) self.assertTrue(debug.call_count) @@ -156,7 +156,7 @@ def test_update_and_force(self): self.assertEqual(x.processes, 5) x.force_scale_down(3) self.assertEqual(x.processes, 2) - x.update(3, None) + x.update(None, 3) self.assertEqual(x.processes, 3) x.force_scale_down(1000) self.assertEqual(x.min_concurrency, 0) diff --git a/celery/worker/autoscale.py b/celery/worker/autoscale.py index 265feda49..06036b246 100644 --- a/celery/worker/autoscale.py +++ b/celery/worker/autoscale.py @@ -71,7 +71,7 @@ def __init__(self, pool, max_concurrency, self.max_concurrency = max_concurrency self.min_concurrency = min_concurrency self.keepalive = keepalive - self._last_action = None + self._last_scale_up = None self.worker = worker assert self.keepalive, 'cannot scale down too fast.' @@ -87,8 +87,9 @@ def _maybe_scale(self, req=None): if cur > procs: self.scale_up(cur - procs) return True - elif cur < procs: - self.scale_down((procs - cur) - self.min_concurrency) + cur = max(self.qty, self.min_concurrency) + if cur < procs: + self.scale_down(procs - cur) return True def maybe_scale(self, req=None): @@ -98,12 +99,12 @@ def maybe_scale(self, req=None): def update(self, max=None, min=None): with self.mutex: if max is not None: - if max < self.max_concurrency: + if max < self.processes: self._shrink(self.processes - max) self.max_concurrency = max if min is not None: - if min > self.min_concurrency: - self._grow(min - self.min_concurrency) + if min > self.processes: + self._grow(min - self.processes) self.min_concurrency = min return self.max_concurrency, self.min_concurrency @@ -112,7 +113,6 @@ def force_scale_up(self, n): new = self.processes + n if new > self.max_concurrency: self.max_concurrency = new - self.min_concurrency += 1 self._grow(n) def force_scale_down(self, n): @@ -123,13 +123,12 @@ def force_scale_down(self, n): self._shrink(min(n, self.processes)) def scale_up(self, n): - self._last_action = monotonic() + self._last_scale_up = monotonic() return self._grow(n) def scale_down(self, n): - if n and self._last_action and ( - monotonic() - self._last_action > self.keepalive): - self._last_action = monotonic() + if self._last_scale_up and ( + monotonic() - self._last_scale_up > self.keepalive): return self._shrink(n) def _grow(self, n): From 759842aab336c8696056b4daa6f6e029281f5b9e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 10 Jul 2015 15:43:00 -0700 Subject: [PATCH 0544/1103] Redis backend get_many now supports on_message --- celery/backends/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/celery/backends/base.py b/celery/backends/base.py index 50dec0c0a..8c64683cd 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -461,6 +461,7 @@ def _mget_to_results(self, values, keys): } def get_many(self, task_ids, timeout=None, interval=0.5, no_ack=True, + on_message=None, READY_STATES=states.READY_STATES): interval = 0.5 if interval is None else interval ids = task_ids if isinstance(task_ids, set) else set(task_ids) @@ -485,6 +486,8 @@ def get_many(self, task_ids, timeout=None, interval=0.5, no_ack=True, cache.update(r) ids.difference_update({bytes_to_str(v) for v in r}) for key, value in items(r): + if on_message is not None: + on_message(value) yield bytes_to_str(key), value if timeout and iterations * interval >= timeout: raise TimeoutError('Operation timed out ({0})'.format(timeout)) From 1e3fcaa969de6ad32b52a3ed8e74281e5e5360e6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 10 Jul 2015 15:45:13 -0700 Subject: [PATCH 0545/1103] [Canvas] Support special case of group(A.s() | group(B.s() | C.S())) --- celery/canvas.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 36e985c08..545eb7fb2 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -21,6 +21,7 @@ from kombu.utils import cached_property, fxrange, reprcall, uuid from celery._state import current_app, get_current_worker_task +from celery.result import GroupResult from celery.utils.functional import ( maybe_list, is_list, regen, chunks as _chunks, @@ -368,6 +369,7 @@ def __init__(self, *tasks, **options): self, 'celery.chain', (), {'tasks': tasks}, **options ) self.subtask_type = 'chain' + self._frozen = None def __call__(self, *args, **kwargs): if self.tasks: @@ -387,10 +389,14 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, app = app or self.app args = (tuple(args) + tuple(self.args) if args and not self.immutable else self.args) - tasks, results = self.prepare_steps( - args, self.tasks, root_id, link_error, app, - task_id, group_id, chord, - ) + + try: + tasks, results = self._frozen + except (AttributeError, ValueError): + tasks, results = self.prepare_steps( + args, self.tasks, root_id, link_error, app, + task_id, group_id, chord, + ) if results: # make sure we can do a link() and link_error() on a chain object. if link: @@ -398,6 +404,12 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, tasks[0].apply_async(**options) return results[-1] + def freeze(self, _id=None, group_id=None, chord=None, root_id=None): + _, results = self._frozen = self.prepare_steps( + (), self.tasks, root_id, None, self.app, _id, group_id, chord, + ) + return results[-1] + def prepare_steps(self, args, tasks, root_id=None, link_error=None, app=None, last_task_id=None, group_id=None, chord_body=None, @@ -665,6 +677,16 @@ def apply_async(self, args=(), kwargs=None, add_to_parent=True, result = self.app.GroupResult( group_id, list(self._apply_tasks(tasks, producer, app, **options)), ) + + # - Special case of group(A.s() | group(B.s(), C.s())) + # That is, group with single item that is a chain but the + # last task in that chain is a group. + # + # We cannot actually support arbitrary GroupResults in chains, + # but this special case we can. + if len(result) == 1 and isinstance(result[0], GroupResult): + result = result[0] + parent_task = get_current_worker_task() if add_to_parent and parent_task: parent_task.add_trail(result) From 72b16ac7c43c9cbd56d2cc9d87ba5552d159ef1e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 10 Jul 2015 18:34:41 -0700 Subject: [PATCH 0546/1103] Apparently (x,) is legal now, so no need for (x, ) --- celery/app/amqp.py | 2 +- celery/app/annotations.py | 2 +- celery/app/base.py | 6 ++-- celery/app/routes.py | 2 +- celery/app/task.py | 7 ++-- celery/app/trace.py | 6 ++-- celery/apps/worker.py | 2 +- celery/backends/base.py | 4 +-- celery/backends/mongodb.py | 2 +- celery/backends/redis.py | 2 +- celery/bin/logtool.py | 2 +- celery/bin/multi.py | 6 ++-- celery/bootsteps.py | 4 +-- celery/canvas.py | 10 +++--- celery/concurrency/asynpool.py | 10 +++--- celery/concurrency/gevent.py | 2 +- celery/contrib/batches.py | 2 +- celery/events/cursesmon.py | 2 +- celery/events/snapshot.py | 2 +- celery/five.py | 6 ++-- celery/fixups/django.py | 2 +- celery/local.py | 4 +-- celery/platforms.py | 2 +- celery/schedules.py | 2 +- celery/security/serialization.py | 4 +-- celery/security/utils.py | 2 +- celery/task/base.py | 2 +- celery/tests/app/test_annotations.py | 2 +- celery/tests/app/test_app.py | 2 +- celery/tests/app/test_loaders.py | 2 +- celery/tests/bin/test_base.py | 6 ++-- celery/tests/bin/test_celeryevdump.py | 2 +- celery/tests/case.py | 6 ++-- celery/tests/compat_modules/test_compat.py | 2 +- celery/tests/compat_modules/test_sets.py | 16 ++++----- celery/tests/concurrency/test_concurrency.py | 4 +-- celery/tests/concurrency/test_pool.py | 2 +- celery/tests/concurrency/test_prefork.py | 8 ++--- celery/tests/events/test_state.py | 4 +-- celery/tests/fixups/test_django.py | 4 +-- celery/tests/security/test_security.py | 4 +-- celery/tests/tasks/test_canvas.py | 10 +++--- celery/tests/tasks/test_chord.py | 2 +- celery/tests/tasks/test_trace.py | 10 +++--- celery/tests/utils/test_datastructures.py | 2 +- celery/tests/utils/test_imports.py | 2 +- celery/tests/utils/test_pickle.py | 2 +- celery/tests/utils/test_timer2.py | 4 +-- celery/tests/utils/test_utils.py | 2 +- celery/tests/worker/test_bootsteps.py | 4 +-- celery/tests/worker/test_consumer.py | 4 +-- celery/tests/worker/test_control.py | 6 ++-- celery/tests/worker/test_hub.py | 6 ++-- celery/tests/worker/test_loops.py | 4 +-- celery/tests/worker/test_request.py | 6 ++-- celery/tests/worker/test_worker.py | 16 ++++----- celery/utils/functional.py | 4 +-- celery/utils/serialization.py | 2 +- celery/worker/__init__.py | 2 +- celery/worker/autoreload.py | 2 +- celery/worker/autoscale.py | 2 +- celery/worker/components.py | 6 ++-- celery/worker/consumer.py | 18 +++++------ celery/worker/heartbeat.py | 2 +- celery/worker/request.py | 2 +- celery/worker/strategy.py | 2 +- docs/configuration.rst | 2 +- docs/getting-started/brokers/django.rst | 2 +- docs/history/changelog-2.5.rst | 6 ++-- docs/history/changelog-3.1.rst | 2 +- docs/internals/deprecation.rst | 2 +- docs/userguide/application.rst | 2 +- docs/userguide/calling.rst | 2 +- docs/userguide/canvas.rst | 4 +-- docs/userguide/extending.rst | 34 ++++++++++---------- docs/userguide/monitoring.rst | 4 +-- docs/userguide/routing.rst | 16 +++++---- docs/whatsnew-2.5.rst | 2 +- docs/whatsnew-3.0.rst | 6 ++-- examples/eventlet/README.rst | 2 +- examples/gevent/celeryconfig.py | 2 +- funtests/suite/config.py | 2 +- setup.py | 6 ++-- 83 files changed, 191 insertions(+), 190 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 09320be04..136f5db53 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -238,7 +238,7 @@ def Queues(self, queues, create_missing=None, ha_policy=None, if not queues and conf.CELERY_DEFAULT_QUEUE: queues = (Queue(conf.CELERY_DEFAULT_QUEUE, exchange=self.default_exchange, - routing_key=conf.CELERY_DEFAULT_ROUTING_KEY), ) + routing_key=conf.CELERY_DEFAULT_ROUTING_KEY),) autoexchange = (self.autoexchange if autoexchange is None else autoexchange) return self.queues_cls( diff --git a/celery/app/annotations.py b/celery/app/annotations.py index 62ee2e72e..6535aa81b 100644 --- a/celery/app/annotations.py +++ b/celery/app/annotations.py @@ -50,7 +50,7 @@ def expand_annotation(annotation): if annotations is None: return () elif not isinstance(annotations, (list, tuple)): - annotations = (annotations, ) + annotations = (annotations,) return [expand_annotation(anno) for anno in annotations] diff --git a/celery/app/base.py b/celery/app/base.py index 6fe575ac2..cff4f8d1b 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -253,7 +253,7 @@ def _create_task_cls(fun): ret = self._task_from_fun(fun, **opts) else: # return a proxy object that evaluates on first use - ret = PromiseProxy(self._task_from_fun, (fun, ), opts, + ret = PromiseProxy(self._task_from_fun, (fun,), opts, __doc__=fun.__doc__) self._pending.append(ret) if _filt: @@ -280,7 +280,7 @@ def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): if name not in self._tasks: run = fun if bind else staticmethod(fun) - task = type(fun.__name__, (base, ), dict({ + task = type(fun.__name__, (base,), dict({ 'app': self, 'name': name, 'run': run, @@ -583,7 +583,7 @@ def __reduce__(self): if not keep_reduce: attrs['__reduce__'] = __reduce__ - return type(name or Class.__name__, (Class, ), attrs) + return type(name or Class.__name__, (Class,), attrs) def _rgetattr(self, path): return attrgetter(path)(self) diff --git a/celery/app/routes.py b/celery/app/routes.py index c3952b10d..0fa384103 100644 --- a/celery/app/routes.py +++ b/celery/app/routes.py @@ -92,5 +92,5 @@ def expand_route(route): if routes is None: return () if not isinstance(routes, (list, tuple)): - routes = (routes, ) + routes = (routes,) return [expand_route(route) for route in routes] diff --git a/celery/app/task.py b/celery/app/task.py index 44f2ab004..f56027c91 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -470,15 +470,14 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, # add 'self' if this is a "task_method". if self.__self__ is not None: args = args if isinstance(args, tuple) else tuple(args or ()) - args = (self.__self__, ) + args - shadow = shadow or self.shadow_name(args, kwargs, final_options) + args = (self.__self__,) + args preopts = self._get_exec_options() options = dict(preopts, **options) if options else preopts return app.send_task( self.name, args, kwargs, task_id=task_id, producer=producer, link=link, link_error=link_error, result_cls=self.AsyncResult, - shadow=shadow, + shadow=shadow or self.shadow_name(args, kwargs, options), **options ) @@ -658,7 +657,7 @@ def apply(self, args=None, kwargs=None, args = args or () # add 'self' if this is a bound method. if self.__self__ is not None: - args = (self.__self__, ) + tuple(args) + args = (self.__self__,) + tuple(args) kwargs = kwargs or {} task_id = options.get('task_id') or uuid() retries = options.get('retries', 0) diff --git a/celery/app/trace.py b/celery/app/trace.py index 8afc1988d..5b588b881 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -310,7 +310,7 @@ def on_error(request, exc, uuid, state=FAILURE, call_errbacks=True): group( [signature(errback, app=app) for errback in request.errbacks or []], app=app, - ).apply_async((uuid, )) + ).apply_async((uuid,)) return I, R, I.state, I.retval def trace_task(uuid, args, kwargs, request=None): @@ -392,9 +392,9 @@ def trace_task(uuid, args, kwargs, request=None): else: sigs.append(sig) for group_ in groups: - group.apply_async((retval, )) + group.apply_async((retval,)) if sigs: - group(sigs).apply_async((retval, )) + group(sigs).apply_async((retval,)) else: signature(callbacks[0], app=app).delay(retval) if publish_result: diff --git a/celery/apps/worker.py b/celery/apps/worker.py index e5a12548d..a9436b8fa 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -175,7 +175,7 @@ def on_start(self): def on_consumer_ready(self, consumer): signals.worker_ready.send(sender=consumer) - print('{0} ready.'.format(safe_str(self.hostname), )) + print('{0} ready.'.format(safe_str(self.hostname),)) def setup_logging(self, colorize=None): if colorize is None and self.no_color is not None: diff --git a/celery/backends/base.py b/celery/backends/base.py index c47fc54b1..e3201e437 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -131,7 +131,7 @@ def chord_error_from_stack(self, callback, exc=None): [app.signature(errback) for errback in callback.options.get('link_error') or []], app=app, - ).apply_async((callback.id, )) + ).apply_async((callback.id,)) except Exception as eb_exc: return backend.fail_from_current_stack(callback.id, exc=eb_exc) else: @@ -352,7 +352,7 @@ def fallback_chord_unlock(self, group_id, body, result=None, countdown=1, **kwargs): kwargs['result'] = [r.as_tuple() for r in result] self.app.tasks['celery.chord_unlock'].apply_async( - (group_id, body, ), kwargs, countdown=countdown, + (group_id, body,), kwargs, countdown=countdown, ) def apply_chord(self, header, partial_args, group_id, body, diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 926ef454b..2e48fb3df 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -126,7 +126,7 @@ def __init__(self, app=None, url=None, **kwargs): self.options.update(config) def _prepare_client_options(self): - if pymongo.version_tuple >= (3, ): + if pymongo.version_tuple >= (3,): return {'maxPoolSize': self.max_pool_size} else: # pragma: no cover return {'max_pool_size': self.max_pool_size, diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 6592a1c0c..beefdbb11 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -264,7 +264,7 @@ def client(self): def __reduce__(self, args=(), kwargs={}): return super(RedisBackend, self).__reduce__( - (self.url, ), {'expires': self.expires}, + (self.url,), {'expires': self.expires}, ) @deprecated_property(3.2, 3.3) diff --git a/celery/bin/logtool.py b/celery/bin/logtool.py index 872f64ec9..7e1fffa94 100644 --- a/celery/bin/logtool.py +++ b/celery/bin/logtool.py @@ -162,7 +162,7 @@ def incomplete(self, files): audit = Audit() audit.run(files) for task_id in audit.incomplete_tasks(): - self.error('Did not complete: %r' % (task_id, )) + self.error('Did not complete: %r' % (task_id,)) def debug(self, files): Audit(on_debug=self.out).run(files) diff --git a/celery/bin/multi.py b/celery/bin/multi.py index d0ea4a668..7429619df 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -160,7 +160,7 @@ def main(): def celery_exe(*args): - return ' '.join((CELERY_EXE, ) + args) + return ' '.join((CELERY_EXE,) + args) class MultiTool(object): @@ -494,11 +494,11 @@ def multi_args(p, cmd='celery worker', append='', prefix='', suffix=''): if ns_name.isdigit(): ns_index = int(ns_name) - 1 if ns_index < 0: - raise KeyError('Indexes start at 1 got: %r' % (ns_name, )) + raise KeyError('Indexes start at 1 got: %r' % (ns_name,)) try: p.namespaces[names[ns_index]].update(ns_opts) except IndexError: - raise KeyError('No node at index %r' % (ns_name, )) + raise KeyError('No node at index %r' % (ns_name,)) for name in names: hostname = suffix diff --git a/celery/bootsteps.py b/celery/bootsteps.py index 4471a4cb3..fa9c71b14 100644 --- a/celery/bootsteps.py +++ b/celery/bootsteps.py @@ -21,7 +21,7 @@ try: from greenlet import GreenletExit - IGNORE_ERRORS = (GreenletExit, ) + IGNORE_ERRORS = (GreenletExit,) except ImportError: # pragma: no cover IGNORE_ERRORS = () @@ -393,7 +393,7 @@ def include(self, parent): class ConsumerStep(StartStopStep): - requires = ('celery.worker.consumer:Connection', ) + requires = ('celery.worker.consumer:Connection',) consumers = None def get_consumers(self, channel): diff --git a/celery/canvas.py b/celery/canvas.py index 545eb7fb2..3aafd52a8 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -280,12 +280,12 @@ def __or__(self, other): if isinstance(other, group): other = maybe_unroll_group(other) if not isinstance(self, chain) and isinstance(other, chain): - return chain((self, ) + other.tasks, app=self._app) + return chain((self,) + other.tasks, app=self._app) elif isinstance(other, chain): return chain(*self.tasks + other.tasks, app=self._app) elif isinstance(other, Signature): if isinstance(self, chain): - return chain(*self.tasks + (other, ), app=self._app) + return chain(*self.tasks + (other,), app=self._app) return chain(self, other, app=self._app) return NotImplemented @@ -299,7 +299,7 @@ def __invert__(self): def __reduce__(self): # for serialization, the task type is lazily loaded, # and not stored in the dict itself. - return signature, (dict(self), ) + return signature, (dict(self),) def __json__(self): return dict(self) @@ -484,7 +484,7 @@ def apply(self, args=(), kwargs={}, **options): last, fargs = None, args for task in self.tasks: res = task.clone(fargs).apply( - last and (last.get(), ), **dict(self.options, **options)) + last and (last.get(),), **dict(self.options, **options)) res.parent, last, fargs = last, res, None return last @@ -835,7 +835,7 @@ def apply(self, args=(), kwargs={}, propagate=True, body=None, **options): tasks = (self.tasks.clone() if isinstance(self.tasks, group) else group(self.tasks)) return body.apply( - args=(tasks.apply().get(propagate=propagate), ), + args=(tasks.apply().get(propagate=propagate),), ) def _traverse_tasks(self, tasks, value=None): diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index b1cb64751..c4829c950 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -196,7 +196,7 @@ def on_loop_start(self, pid): # our version sends a WORKER_UP message when the process is ready # to accept work, this will tell the parent that the inqueue fd # is writable. - self.outq.put((WORKER_UP, (pid, ))) + self.outq.put((WORKER_UP, (pid,))) class ResultHandler(_pool.ResultHandler): @@ -644,8 +644,8 @@ def _create_write_handlers(self, hub, revoked_tasks = worker_state.revoked getpid = os.getpid - precalc = {ACK: self._create_payload(ACK, (0, )), - NACK: self._create_payload(NACK, (0, ))} + precalc = {ACK: self._create_payload(ACK, (0,)), + NACK: self._create_payload(NACK, (0,))} def _put_back(job, _time=time.time): # puts back at the end of the queue @@ -854,7 +854,7 @@ def send_ack(response, pid, job, fd, WRITE=WRITE, ERR=ERR): cor = _write_ack(fd, msg, callback=callback) mark_write_gen_as_active(cor) mark_write_fd_as_active(fd) - callback.args = (cor, ) + callback.args = (cor,) add_writer(fd, cor) self.send_ack = send_ack @@ -1225,7 +1225,7 @@ def _set_result_sentinel(cls, _outqueue, _pool): def _help_stuff_finish_args(self): # Pool._help_stuff_finished is a classmethod so we have to use this # trick to modify the arguments passed to it. - return (self._pool, ) + return (self._pool,) @classmethod def _help_stuff_finish(cls, pool): diff --git a/celery/concurrency/gevent.py b/celery/concurrency/gevent.py index 0574178c9..ba39c8f8b 100644 --- a/celery/concurrency/gevent.py +++ b/celery/concurrency/gevent.py @@ -30,7 +30,7 @@ def apply_timeout(target, args=(), kwargs={}, callback=None, with Timeout(timeout): return apply_target(target, args, kwargs, callback, accept_callback, pid, - propagate=(Timeout, ), **rest) + propagate=(Timeout,), **rest) except Timeout: return timeout_callback(False, timeout) diff --git a/celery/contrib/batches.py b/celery/contrib/batches.py index e3d2e86c5..a476387d1 100644 --- a/celery/contrib/batches.py +++ b/celery/contrib/batches.py @@ -230,7 +230,7 @@ def task_message_handler(message, body, ack, reject, callbacks, **kw): def flush(self, requests): return self.apply_buffer(requests, ([SimpleRequest.from_request(r) - for r in requests], )) + for r in requests],)) def _do_flush(self): logger.debug('Batches: Wake-up to flush buffer...') diff --git a/celery/events/cursesmon.py b/celery/events/cursesmon.py index 796565fc4..4f34a66de 100644 --- a/celery/events/cursesmon.py +++ b/celery/events/cursesmon.py @@ -236,7 +236,7 @@ def readline(self, x, y): if ch != -1: if ch in (10, curses.KEY_ENTER): # enter break - if ch in (27, ): + if ch in (27,): buffer = str() break buffer += chr(ch) diff --git a/celery/events/snapshot.py b/celery/events/snapshot.py index a202a70f3..1888636ef 100644 --- a/celery/events/snapshot.py +++ b/celery/events/snapshot.py @@ -29,7 +29,7 @@ class Polaroid(object): timer = None - shutter_signal = Signal(providing_args=('state', )) + shutter_signal = Signal(providing_args=('state',)) cleanup_signal = Signal() clear_after = False diff --git a/celery/five.py b/celery/five.py index 732ccde97..6c5d9b007 100644 --- a/celery/five.py +++ b/celery/five.py @@ -160,7 +160,7 @@ def __dir__(self): return list(set(self.__all__) | DEFAULT_ATTRS) def __reduce__(self): - return import_module, (self.__name__, ) + return import_module, (self.__name__,) def create_module(name, attrs, cls_attrs=None, pkg=None, @@ -174,7 +174,7 @@ def create_module(name, attrs, cls_attrs=None, pkg=None, attr_name: (prepare_attr(attr) if prepare_attr else attr) for attr_name, attr in items(attrs) } - module = sys.modules[fqdn] = type(modname, (base, ), cls_attrs)(name) + module = sys.modules[fqdn] = type(modname, (base,), cls_attrs)(name) module.__dict__.update(attrs) return module @@ -206,7 +206,7 @@ def get_compat_module(pkg, name): def prepare(attr): if isinstance(attr, string_t): - return Proxy(getappattr, (attr, )) + return Proxy(getappattr, (attr,)) return attr attrs = COMPAT_MODULES[pkg.__name__][name] diff --git a/celery/fixups/django.py b/celery/fixups/django.py index 66b76f4db..60b836290 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -143,7 +143,7 @@ def __init__(self, app): except (ImportError, AttributeError): self._close_old_connections = None self.database_errors = ( - (DatabaseError, ) + + (DatabaseError,) + _my_database_errors + _pg_database_errors + _lite_database_errors + diff --git a/celery/local.py b/celery/local.py index 1a10c2d8c..70f7df72d 100644 --- a/celery/local.py +++ b/celery/local.py @@ -39,7 +39,7 @@ def __new__(cls, getter): def __get__(self, obj, cls=None): return self.__getter(obj) if obj is not None else self - return type(name, (type_, ), { + return type(name, (type_,), { '__new__': __new__, '__get__': __get__, }) @@ -212,7 +212,7 @@ class PromiseProxy(Proxy): """ - __slots__ = ('__pending__', ) + __slots__ = ('__pending__',) def _get_current_object(self): try: diff --git a/celery/platforms.py b/celery/platforms.py index 194c2b9bd..a665e7f48 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -693,7 +693,7 @@ def ignore_errno(*errnos, **kwargs): :keyword types: A tuple of exceptions to ignore (when the errno matches), defaults to :exc:`Exception`. """ - types = kwargs.get('types') or (Exception, ) + types = kwargs.get('types') or (Exception,) errnos = [get_errno_name(errno) for errno in errnos] try: yield diff --git a/celery/schedules.py b/celery/schedules.py index 4b3ffeaa1..406b8ff64 100644 --- a/celery/schedules.py +++ b/celery/schedules.py @@ -251,7 +251,7 @@ def _parse_part(self, part): m = regex.match(part) if m: return handler(m.groups()) - return self._expand_range((part, )) + return self._expand_range((part,)) def _expand_range(self, toks): fr = self._expand_number(toks[0]) diff --git a/celery/security/serialization.py b/celery/security/serialization.py index 278bfb9e9..3b0458974 100644 --- a/celery/security/serialization.py +++ b/celery/security/serialization.py @@ -33,7 +33,7 @@ def serialize(self, data): """serialize data structure into string""" assert self._key is not None assert self._cert is not None - with reraise_errors('Unable to serialize: {0!r}', (Exception, )): + with reraise_errors('Unable to serialize: {0!r}', (Exception,)): content_type, content_encoding, body = dumps( bytes_to_str(data), serializer=self._serializer) # What we sign is the serialized body, not the body itself. @@ -48,7 +48,7 @@ def serialize(self, data): def deserialize(self, data): """deserialize data structure from string""" assert self._cert_store is not None - with reraise_errors('Unable to deserialize: {0!r}', (Exception, )): + with reraise_errors('Unable to deserialize: {0!r}', (Exception,)): payload = self._unpack(data) signature, signer, body = (payload['signature'], payload['signer'], diff --git a/celery/security/utils.py b/celery/security/utils.py index d184d0b4c..7683afc59 100644 --- a/celery/security/utils.py +++ b/celery/security/utils.py @@ -26,7 +26,7 @@ @contextmanager def reraise_errors(msg='{0!r}', errors=None): assert crypto is not None - errors = (crypto.Error, ) if errors is None else errors + errors = (crypto.Error,) if errors is None else errors try: yield except errors as exc: diff --git a/celery/task/base.py b/celery/task/base.py index 27f72408b..31a45544c 100644 --- a/celery/task/base.py +++ b/celery/task/base.py @@ -40,7 +40,7 @@ def __hash__(self): return hash(self.name) def __repr__(self): - return '' % (self.name, ) + return '' % (self.name,) def __call__(self, app): return self.cons(app) diff --git a/celery/tests/app/test_annotations.py b/celery/tests/app/test_annotations.py index 559f5cb01..1b4f6afd8 100644 --- a/celery/tests/app/test_annotations.py +++ b/celery/tests/app/test_annotations.py @@ -48,7 +48,7 @@ def test_dict_to_MapAnnotation(self): def test_returns_list(self): self.assertListEqual(prepare(1), [1]) self.assertListEqual(prepare([1]), [1]) - self.assertListEqual(prepare((1, )), [1]) + self.assertListEqual(prepare((1,)), [1]) self.assertEqual(prepare(None), ()) def test_evalutes_qualnames(self): diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index 413d71857..af4dedc02 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -338,7 +338,7 @@ def aawsX(x, y): with self.assertRaises(TypeError): aawsX.apply_async(()) with self.assertRaises(TypeError): - aawsX.apply_async((2, )) + aawsX.apply_async((2,)) with patch('celery.app.amqp.AMQP.create_task_message') as create: with patch('celery.app.amqp.AMQP.send_task_message') as send: diff --git a/celery/tests/app/test_loaders.py b/celery/tests/app/test_loaders.py index bc39f6108..c98582933 100644 --- a/celery/tests/app/test_loaders.py +++ b/celery/tests/app/test_loaders.py @@ -238,7 +238,7 @@ def setup(self): self.loader = AppLoader(app=self.app) def test_on_worker_init(self): - self.app.conf.CELERY_IMPORTS = ('subprocess', ) + self.app.conf.CELERY_IMPORTS = ('subprocess',) sys.modules.pop('subprocess', None) self.loader.init_worker() self.assertIn('subprocess', sys.modules) diff --git a/celery/tests/bin/test_base.py b/celery/tests/bin/test_base.py index 61d56fe0d..36de997cb 100644 --- a/celery/tests/bin/test_base.py +++ b/celery/tests/bin/test_base.py @@ -123,7 +123,7 @@ def run(a, b, c): c.run = run with self.assertRaises(c.UsageError): - c.verify_args((1, )) + c.verify_args((1,)) c.verify_args((1, 2, 3)) def test_run_interface(self): @@ -186,7 +186,7 @@ def test_with_custom_broker(self): def test_with_custom_app(self): cmd = MockCommand(app=self.app) app = '.'.join([__name__, 'APP']) - cmd.setup_app_from_commandline(['--app=%s' % (app, ), + cmd.setup_app_from_commandline(['--app=%s' % (app,), '--loglevel=INFO']) self.assertIs(cmd.app, APP) cmd.setup_app_from_commandline(['-A', app, @@ -311,7 +311,7 @@ def after(*args, **kwargs): def test_parse_preload_options_shortopt(self): cmd = Command() - cmd.preload_options = (Option('-s', action='store', dest='silent'), ) + cmd.preload_options = (Option('-s', action='store', dest='silent'),) acc = cmd.parse_preload_options(['-s', 'yes']) self.assertEqual(acc.get('silent'), 'yes') diff --git a/celery/tests/bin/test_celeryevdump.py b/celery/tests/bin/test_celeryevdump.py index 09cdc4d1f..9eb7d52bc 100644 --- a/celery/tests/bin/test_celeryevdump.py +++ b/celery/tests/bin/test_celeryevdump.py @@ -58,7 +58,7 @@ def se(*_a, **_k): Conn = app.connection.return_value = Mock(name='conn') conn = Conn.clone.return_value = Mock(name='cloned_conn') - conn.connection_errors = (KeyError, ) + conn.connection_errors = (KeyError,) conn.channel_errors = () evdump(app) diff --git a/celery/tests/case.py b/celery/tests/case.py index ad9951afa..ad94d3b57 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -204,7 +204,7 @@ def __inner(*args, **kwargs): try: importlib.import_module(module) except ImportError: - raise SkipTest('Does not have %s' % (module, )) + raise SkipTest('Does not have %s' % (module,)) return fun(*args, **kwargs) @@ -362,11 +362,11 @@ def assertItemsEqual(self, expected_seq, actual_seq, msg=None): errors = [] if missing: errors.append( - 'Expected, but missing:\n %s' % (safe_repr(missing), ) + 'Expected, but missing:\n %s' % (safe_repr(missing),) ) if unexpected: errors.append( - 'Unexpected, but present:\n %s' % (safe_repr(unexpected), ) + 'Unexpected, but present:\n %s' % (safe_repr(unexpected),) ) if errors: standardMsg = '\n'.join(errors) diff --git a/celery/tests/compat_modules/test_compat.py b/celery/tests/compat_modules/test_compat.py index aa7be5dd4..ee9c5cb26 100644 --- a/celery/tests/compat_modules/test_compat.py +++ b/celery/tests/compat_modules/test_compat.py @@ -29,7 +29,7 @@ def now(self): def test_must_have_run_every(self): with self.assertRaises(NotImplementedError): - type('Foo', (PeriodicTask, ), {'__module__': __name__}) + type('Foo', (PeriodicTask,), {'__module__': __name__}) def test_remaining_estimate(self): s = self.my_periodic.run_every diff --git a/celery/tests/compat_modules/test_sets.py b/celery/tests/compat_modules/test_sets.py index 4176143d5..710adae76 100644 --- a/celery/tests/compat_modules/test_sets.py +++ b/celery/tests/compat_modules/test_sets.py @@ -95,7 +95,7 @@ def test_apply_async(self): def test_delay_argmerge(self): s = self.MockTask.subtask( - (2, ), {'cache': True}, {'routing_key': 'CPU-bound'}, + (2,), {'cache': True}, {'routing_key': 'CPU-bound'}, ) args, kwargs, options = s.delay(10, cache=False, other='foo') self.assertTupleEqual(args, (10, 2)) @@ -104,9 +104,9 @@ def test_delay_argmerge(self): def test_apply_async_argmerge(self): s = self.MockTask.subtask( - (2, ), {'cache': True}, {'routing_key': 'CPU-bound'}, + (2,), {'cache': True}, {'routing_key': 'CPU-bound'}, ) - args, kwargs, options = s.apply_async((10, ), + args, kwargs, options = s.apply_async((10,), {'cache': False, 'other': 'foo'}, routing_key='IO-bound', exchange='fast') @@ -118,9 +118,9 @@ def test_apply_async_argmerge(self): def test_apply_argmerge(self): s = self.MockTask.subtask( - (2, ), {'cache': True}, {'routing_key': 'CPU-bound'}, + (2,), {'cache': True}, {'routing_key': 'CPU-bound'}, ) - args, kwargs, options = s.apply((10, ), + args, kwargs, options = s.apply((10,), {'cache': False, 'other': 'foo'}, routing_key='IO-bound', exchange='fast') @@ -133,19 +133,19 @@ def test_apply_argmerge(self): def test_is_JSON_serializable(self): s = self.MockTask.subtask( - (2, ), {'cache': True}, {'routing_key': 'CPU-bound'}, + (2,), {'cache': True}, {'routing_key': 'CPU-bound'}, ) # tuples are not preserved, but this doesn't matter. s.args = list(s.args) self.assertEqual(s, self.subtask(json.loads(json.dumps(s)))) def test_repr(self): - s = self.MockTask.subtask((2, ), {'cache': True}) + s = self.MockTask.subtask((2,), {'cache': True}) self.assertIn('2', repr(s)) self.assertIn('cache=True', repr(s)) def test_reduce(self): - s = self.MockTask.subtask((2, ), {'cache': True}) + s = self.MockTask.subtask((2,), {'cache': True}) cls, args = s.__reduce__() self.assertDictEqual(dict(cls(*args)), dict(s)) diff --git a/celery/tests/concurrency/test_concurrency.py b/celery/tests/concurrency/test_concurrency.py index 293887741..dd845de1f 100644 --- a/celery/tests/concurrency/test_concurrency.py +++ b/celery/tests/concurrency/test_concurrency.py @@ -29,7 +29,7 @@ def callback(*args): accept_callback=gen_callback('accept_callback')) self.assertDictContainsSubset( - {'target': (1, (8, 16)), 'callback': (2, (42, ))}, + {'target': (1, (8, 16)), 'callback': (2, (42,))}, scratch, ) pa1 = scratch['accept_callback'] @@ -45,7 +45,7 @@ def callback(*args): accept_callback=None) self.assertDictEqual(scratch, {'target': (3, (8, 16)), - 'callback': (4, (42, ))}) + 'callback': (4, (42,))}) def test_does_not_debug(self): x = BasePool(10) diff --git a/celery/tests/concurrency/test_pool.py b/celery/tests/concurrency/test_pool.py index d1b314b52..4930dc89f 100644 --- a/celery/tests/concurrency/test_pool.py +++ b/celery/tests/concurrency/test_pool.py @@ -66,7 +66,7 @@ def mycallback(ret_value): self.assertIsInstance(scratchpad[1]['ret_value'], ExceptionInfo) self.assertEqual(scratchpad[1]['ret_value'].exception.args, - ('FOO EXCEPTION', )) + ('FOO EXCEPTION',)) self.assertEqual(res3.get(), 400) time.sleep(0.5) diff --git a/celery/tests/concurrency/test_prefork.py b/celery/tests/concurrency/test_prefork.py index 47081ce26..b48629c9d 100644 --- a/celery/tests/concurrency/test_prefork.py +++ b/celery/tests/concurrency/test_prefork.py @@ -112,7 +112,7 @@ class ExeMockPool(MockPool): def apply_async(self, target, args=(), kwargs={}, callback=noop): from threading import Timer res = target(*args, **kwargs) - Timer(0.1, callback, (res, )).start() + Timer(0.1, callback, (res,)).start() return MockResult(res, next(self._current_proc)) @@ -227,7 +227,7 @@ def se2(*args): def test_promise(self): fun = Mock() - x = asynpool.promise(fun, (1, ), {'foo': 1}) + x = asynpool.promise(fun, (1,), {'foo': 1}) x() self.assertTrue(x.ready) fun.assert_called_with(1, foo=1) @@ -235,7 +235,7 @@ def test_promise(self): def test_Worker(self): w = asynpool.Worker(Mock(), Mock()) w.on_loop_start(1234) - w.outq.put.assert_called_with((asynpool.WORKER_UP, (1234, ))) + w.outq.put.assert_called_with((asynpool.WORKER_UP, (1234,))) class test_ResultHandler(PoolCase): @@ -287,7 +287,7 @@ def test_start(self): def test_apply_async(self): pool = TaskPool(10) pool.start() - pool.apply_async(lambda x: x, (2, ), {}) + pool.apply_async(lambda x: x, (2,), {}) def test_grow_shrink(self): pool = TaskPool(10) diff --git a/celery/tests/events/test_state.py b/celery/tests/events/test_state.py index 6ed41dad4..e12ae77c9 100644 --- a/celery/tests/events/test_state.py +++ b/celery/tests/events/test_state.py @@ -253,8 +253,8 @@ def test_info(self): self.assertEqual(sorted(list(task._info_fields)), sorted(task.info().keys())) - self.assertEqual(sorted(list(task._info_fields + ('received', ))), - sorted(task.info(extra=('received', )))) + self.assertEqual(sorted(list(task._info_fields + ('received',))), + sorted(task.info(extra=('received',)))) self.assertEqual(sorted(['args', 'kwargs']), sorted(task.info(['args', 'kwargs']).keys())) diff --git a/celery/tests/fixups/test_django.py b/celery/tests/fixups/test_django.py index 94b755eee..c2dffd41c 100644 --- a/celery/tests/fixups/test_django.py +++ b/celery/tests/fixups/test_django.py @@ -114,7 +114,7 @@ def test_install(self): self.app.conf = {'CELERY_DB_REUSE_MAX': None} self.app.loader = Mock() with self.fixup_context(self.app) as (f, _, _): - with patch_many('celery.fixups.django.signals') as (sigs, ): + with patch_many('celery.fixups.django.signals') as (sigs,): f.install() sigs.beat_embedded_init.connect.assert_called_with( f.close_database, @@ -207,7 +207,7 @@ def test__close_database(self): with self.fixup_context(self.app) as (f, _, _): conns = [Mock(), Mock(), Mock()] conns[1].close.side_effect = KeyError('already closed') - f.database_errors = (KeyError, ) + f.database_errors = (KeyError,) f._db.connections = Mock() # ConnectionHandler f._db.connections.all.side_effect = lambda: conns diff --git a/celery/tests/security/test_security.py b/celery/tests/security/test_security.py index 227c65a5d..07d594d0a 100644 --- a/celery/tests/security/test_security.py +++ b/celery/tests/security/test_security.py @@ -103,8 +103,8 @@ def import_hook(name, *args, **kwargs): def test_reraise_errors(self): with self.assertRaises(SecurityError): - with reraise_errors(errors=(KeyError, )): + with reraise_errors(errors=(KeyError,)): raise KeyError('foo') with self.assertRaises(KeyError): - with reraise_errors(errors=(ValueError, )): + with reraise_errors(errors=(ValueError,)): raise KeyError('bar') diff --git a/celery/tests/tasks/test_canvas.py b/celery/tests/tasks/test_canvas.py index 393cda69b..365f11a64 100644 --- a/celery/tests/tasks/test_canvas.py +++ b/celery/tests/tasks/test_canvas.py @@ -17,7 +17,7 @@ from celery.tests.case import AppCase, Mock SIG = Signature({'task': 'TASK', - 'args': ('A1', ), + 'args': ('A1',), 'kwargs': {'K1': 'V1'}, 'options': {'task_id': 'TASK_ID'}, 'subtask_type': ''}) @@ -54,7 +54,7 @@ def test_getitem_property_class(self): def test_getitem_property(self): self.assertEqual(SIG.task, 'TASK') - self.assertEqual(SIG.args, ('A1', )) + self.assertEqual(SIG.args, ('A1',)) self.assertEqual(SIG.kwargs, {'K1': 'V1'}) self.assertEqual(SIG.options, {'task_id': 'TASK_ID'}) self.assertEqual(SIG.subtask_type, '') @@ -69,7 +69,7 @@ def test_link_on_scalar(self): def test_replace(self): x = Signature('TASK', ('A'), {}) - self.assertTupleEqual(x.replace(args=('B', )).args, ('B', )) + self.assertTupleEqual(x.replace(args=('B',)).args, ('B',)) self.assertDictEqual( x.replace(kwargs={'FOO': 'BAR'}).kwargs, {'FOO': 'BAR'}, @@ -130,7 +130,7 @@ def test_INVERT(self): def test_merge_immutable(self): x = self.add.si(2, 2, foo=1) - args, kwargs, options = x._merge((4, ), {'bar': 2}, {'task_id': 3}) + args, kwargs, options = x._merge((4,), {'bar': 2}, {'task_id': 3}) self.assertTupleEqual(args, (2, 2)) self.assertDictEqual(kwargs, {'foo': 1}) self.assertDictEqual(options, {'task_id': 3}) @@ -247,7 +247,7 @@ def test_from_dict_no_args__with_args(self): x = dict(self.add.s(2, 2) | self.add.s(4)) x['args'] = None self.assertIsInstance(chain.from_dict(x), chain) - x['args'] = (2, ) + x['args'] = (2,) self.assertIsInstance(chain.from_dict(x), chain) def test_accepts_generator_argument(self): diff --git a/celery/tests/tasks/test_chord.py b/celery/tests/tasks/test_chord.py index df06bdc4f..e09211f00 100644 --- a/celery/tests/tasks/test_chord.py +++ b/celery/tests/tasks/test_chord.py @@ -72,7 +72,7 @@ class AlwaysReady(TSR): with self._chord_context(AlwaysReady) as (cb, retry, _): cb.type.apply_async.assert_called_with( - ([2, 4, 8, 6], ), {}, task_id=cb.id, + ([2, 4, 8, 6],), {}, task_id=cb.id, ) # did not retry self.assertFalse(retry.call_count) diff --git a/celery/tests/tasks/test_trace.py b/celery/tests/tasks/test_trace.py index 3149206fe..0714acc2e 100644 --- a/celery/tests/tasks/test_trace.py +++ b/celery/tests/tasks/test_trace.py @@ -130,23 +130,23 @@ def ignored(): def test_trace_SystemExit(self): with self.assertRaises(SystemExit): - self.trace(self.raises, (SystemExit(), ), {}) + self.trace(self.raises, (SystemExit(),), {}) def test_trace_Retry(self): exc = Retry('foo', 'bar') - _, info = self.trace(self.raises, (exc, ), {}) + _, info = self.trace(self.raises, (exc,), {}) self.assertEqual(info.state, states.RETRY) self.assertIs(info.retval, exc) def test_trace_exception(self): exc = KeyError('foo') - _, info = self.trace(self.raises, (exc, ), {}) + _, info = self.trace(self.raises, (exc,), {}) self.assertEqual(info.state, states.FAILURE) self.assertIs(info.retval, exc) def test_trace_exception_propagate(self): with self.assertRaises(KeyError): - self.trace(self.raises, (KeyError('foo'), ), {}, propagate=True) + self.trace(self.raises, (KeyError('foo'),), {}, propagate=True) @patch('celery.app.trace.build_tracer') @patch('celery.app.trace.report_internal_error') @@ -167,7 +167,7 @@ def xtask(): class test_TraceInfo(TraceCase): class TI(TraceInfo): - __slots__ = TraceInfo.__slots__ + ('__dict__', ) + __slots__ = TraceInfo.__slots__ + ('__dict__',) def test_handle_error_state(self): x = self.TI(states.FAILURE) diff --git a/celery/tests/utils/test_datastructures.py b/celery/tests/utils/test_datastructures.py index e9ee0f7d8..bb148c653 100644 --- a/celery/tests/utils/test_datastructures.py +++ b/celery/tests/utils/test_datastructures.py @@ -158,7 +158,7 @@ def test_exception_info(self): self.assertEqual(str(einfo), einfo.traceback) self.assertIsInstance(einfo.exception, LookupError) self.assertTupleEqual( - einfo.exception.args, ('The quick brown fox jumps...', ), + einfo.exception.args, ('The quick brown fox jumps...',), ) self.assertTrue(einfo.traceback) diff --git a/celery/tests/utils/test_imports.py b/celery/tests/utils/test_imports.py index e7d88bc09..d714451f9 100644 --- a/celery/tests/utils/test_imports.py +++ b/celery/tests/utils/test_imports.py @@ -21,7 +21,7 @@ def test_find_module(self): find_module('foo.bar.baz', imp=imp) def test_qualname(self): - Class = type('Fox', (object, ), {'__module__': 'quick.brown'}) + Class = type('Fox', (object,), {'__module__': 'quick.brown'}) self.assertEqual(qualname(Class), 'quick.brown.Fox') self.assertEqual(qualname(Class()), 'quick.brown.Fox') diff --git a/celery/tests/utils/test_pickle.py b/celery/tests/utils/test_pickle.py index 6b65bb3c5..59ce6b8e7 100644 --- a/celery/tests/utils/test_pickle.py +++ b/celery/tests/utils/test_pickle.py @@ -29,7 +29,7 @@ def test_pickle_regular_exception(self): exception = unpickled.get('exception') self.assertTrue(exception) self.assertIsInstance(exception, RegularException) - self.assertTupleEqual(exception.args, ('RegularException raised', )) + self.assertTupleEqual(exception.args, ('RegularException raised',)) def test_pickle_arg_override_exception(self): diff --git a/celery/tests/utils/test_timer2.py b/celery/tests/utils/test_timer2.py index cb18c2123..582e54366 100644 --- a/celery/tests/utils/test_timer2.py +++ b/celery/tests/utils/test_timer2.py @@ -23,12 +23,12 @@ def timed(x, y, moo='foo'): self.assertTupleEqual(scratch[0], (4, 4, 'baz')) def test_cancel(self): - tref = timer2.Entry(lambda x: x, (1, ), {}) + tref = timer2.Entry(lambda x: x, (1,), {}) tref.cancel() self.assertTrue(tref.cancelled) def test_repr(self): - tref = timer2.Entry(lambda x: x(1, ), {}) + tref = timer2.Entry(lambda x: x(1,), {}) self.assertTrue(repr(tref)) diff --git a/celery/tests/utils/test_utils.py b/celery/tests/utils/test_utils.py index 2837ad636..f9244dcbc 100644 --- a/celery/tests/utils/test_utils.py +++ b/celery/tests/utils/test_utils.py @@ -87,7 +87,7 @@ def test_chunks(self): class test_utils(Case): def test_is_iterable(self): - for a in 'f', ['f'], ('f', ), {'f': 'f'}: + for a in 'f', ['f'], ('f',), {'f': 'f'}: self.assertTrue(is_iterable(a)) for b in object(), 1: self.assertFalse(is_iterable(b)) diff --git a/celery/tests/worker/test_bootsteps.py b/celery/tests/worker/test_bootsteps.py index 522d263b3..f35f66919 100644 --- a/celery/tests/worker/test_bootsteps.py +++ b/celery/tests/worker/test_bootsteps.py @@ -238,7 +238,7 @@ def test_send_all_with_None_steps(self): blueprint.send_all(parent, 'close', 'Closing', reverse=False) def test_join_raises_IGNORE_ERRORS(self): - prev, bootsteps.IGNORE_ERRORS = bootsteps.IGNORE_ERRORS, (KeyError, ) + prev, bootsteps.IGNORE_ERRORS = bootsteps.IGNORE_ERRORS, (KeyError,) try: blueprint = self.Blueprint(app=self.app) blueprint.shutdown_complete = Mock() @@ -278,7 +278,7 @@ class b2s2(bootsteps.Step): def test_topsort_raises_KeyError(self): class Step(bootsteps.Step): - requires = ('xyxxx.fsdasewe.Unknown', ) + requires = ('xyxxx.fsdasewe.Unknown',) b = self.Blueprint([Step], app=self.app) b.steps = b.claim_steps() diff --git a/celery/tests/worker/test_consumer.py b/celery/tests/worker/test_consumer.py index f3b36435c..59ee8edc6 100644 --- a/celery/tests/worker/test_consumer.py +++ b/celery/tests/worker/test_consumer.py @@ -37,7 +37,7 @@ def get_consumer(self, no_hub=False, **kwargs): consumer.blueprint = Mock() consumer._restart_state = Mock() consumer.connection = _amqp_connection() - consumer.connection_errors = (socket.error, OSError, ) + consumer.connection_errors = (socket.error, OSError,) return consumer def test_taskbuckets_defaultdict(self): @@ -88,7 +88,7 @@ def test_limit_task(self): self.assertEqual(c._limit_order, limit_order + 1) bucket.can_consume.assert_called_with(4) c.timer.call_after.assert_called_with( - 3.33, c._limit_move_to_pool, (request, ), + 3.33, c._limit_move_to_pool, (request,), priority=c._limit_order, ) bucket.expected_time.assert_called_with(4) diff --git a/celery/tests/worker/test_control.py b/celery/tests/worker/test_control.py index 340ade75b..b9df3fefe 100644 --- a/celery/tests/worker/test_control.py +++ b/celery/tests/worker/test_control.py @@ -198,7 +198,7 @@ def test_heartbeat(self): panel = self.create_panel(consumer=consumer) consumer.event_dispatcher.enabled = True panel.handle('heartbeat') - self.assertIn(('worker-heartbeat', ), + self.assertIn(('worker-heartbeat',), consumer.event_dispatcher.send.call_args) def test_time_limit(self): @@ -347,10 +347,10 @@ def test_dump_schedule(self): self.assertFalse(panel.handle('dump_schedule')) r = Request(TaskMessage(self.mytask.name, 'CAFEBABE'), app=self.app) consumer.timer.schedule.enter_at( - consumer.timer.Entry(lambda x: x, (r, )), + consumer.timer.Entry(lambda x: x, (r,)), datetime.now() + timedelta(seconds=10)) consumer.timer.schedule.enter_at( - consumer.timer.Entry(lambda x: x, (object(), )), + consumer.timer.Entry(lambda x: x, (object(),)), datetime.now() + timedelta(seconds=10)) self.assertTrue(panel.handle('dump_schedule')) diff --git a/celery/tests/worker/test_hub.py b/celery/tests/worker/test_hub.py index 4e9e4906e..3909e9a2e 100644 --- a/celery/tests/worker/test_hub.py +++ b/celery/tests/worker/test_hub.py @@ -192,7 +192,7 @@ def test_fire_timers_raises(self): hub.timer = Mock() hub.scheduler = iter([(0, eback)]) with self.assertRaises(KeyError): - hub.fire_timers(propagate=(KeyError, )) + hub.fire_timers(propagate=(KeyError,)) eback.side_effect = ValueError('foo') hub.scheduler = iter([(0, eback)]) @@ -258,8 +258,8 @@ def test_add_remove_readers(self): call(11, hub.READ | hub.ERR), ], any_order=True) - self.assertEqual(hub.readers[10], (read_A, (10, ))) - self.assertEqual(hub.readers[11], (read_B, (11, ))) + self.assertEqual(hub.readers[10], (read_A, (10,))) + self.assertEqual(hub.readers[11], (read_B, (11,))) hub.remove(10) self.assertNotIn(10, hub.readers) diff --git a/celery/tests/worker/test_loops.py b/celery/tests/worker/test_loops.py index 4473eb47e..aa92f66d1 100644 --- a/celery/tests/worker/test_loops.py +++ b/celery/tests/worker/test_loops.py @@ -42,7 +42,7 @@ def __init__(self, app, heartbeat=None, on_task_message=None): ) self.consumer.callbacks = [] self.obj.strategies = {} - self.connection.connection_errors = (socket.error, ) + self.connection.connection_errors = (socket.error,) self.hub.readers = {} self.hub.writers = {} self.hub.consolidate = set() @@ -217,7 +217,7 @@ def test_updates_qos(self): x.hub.on_tick.add(x.closer(mod=2)) asynloop(*x.args) x.qos.update.assert_called_with() - x.hub.fire_timers.assert_called_with(propagate=(socket.error, )) + x.hub.fire_timers.assert_called_with(propagate=(socket.error,)) def test_poll_empty(self): x = X(self.app) diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index 392c6d509..b642199ce 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -72,7 +72,7 @@ def mro(cls): A.x = 10 self.assertEqual(mro_lookup(C, 'x'), A) - self.assertIsNone(mro_lookup(C, 'x', stop=(A, ))) + self.assertIsNone(mro_lookup(C, 'x', stop=(A,))) B.x = 10 self.assertEqual(mro_lookup(C, 'x'), B) C.x = 10 @@ -183,7 +183,7 @@ def test_execute_jail_failure(self): self.app, uuid(), self.mytask_raising.name, [4], {}, ) self.assertIsInstance(ret, ExceptionInfo) - self.assertTupleEqual(ret.exception.args, (4, )) + self.assertTupleEqual(ret.exception.args, (4,)) def test_execute_ignore_result(self): @@ -234,7 +234,7 @@ def get_request(self, sig, Request=Request, **kwargs): on_reject=Mock(name='on_reject'), eventer=Mock(name='eventer'), app=self.app, - connection_errors=(socket.error, ), + connection_errors=(socket.error,), task=sig.type, **kwargs ) diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index ebf4425c6..f42f2b1b1 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -39,7 +39,7 @@ def MockStep(step=None): step = Mock() if step is None else step step.blueprint = Mock() step.blueprint.name = 'MockNS' - step.name = 'MockStep(%s)' % (id(step), ) + step.name = 'MockStep(%s)' % (id(step),) return step @@ -333,7 +333,7 @@ def loop(self, *args, **kwargs): send_events=False, pool=BasePool(), app=self.app) l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() - l.channel_errors = (KeyError, ) + l.channel_errors = (KeyError,) with self.assertRaises(KeyError): l.start() l.timer.stop() @@ -354,7 +354,7 @@ def loop(self, *args, **kwargs): l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() - l.connection_errors = (KeyError, ) + l.connection_errors = (KeyError,) self.assertRaises(SyntaxError, l.start) l.timer.stop() @@ -424,8 +424,8 @@ def drain_events(self, **kwargs): def test_ignore_errors(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) - l.connection_errors = (AttributeError, KeyError, ) - l.channel_errors = (SyntaxError, ) + l.connection_errors = (AttributeError, KeyError,) + l.channel_errors = (SyntaxError,) ignore_errors(l, Mock(side_effect=AttributeError('foo'))) ignore_errors(l, Mock(side_effect=KeyError('foo'))) ignore_errors(l, Mock(side_effect=SyntaxError('foo'))) @@ -547,7 +547,7 @@ def test_receieve_message_ack_raises(self, logger, warn): l.event_dispatcher = mock_event_dispatcher() l.update_strategies() - l.connection_errors = (socket.error, ) + l.connection_errors = (socket.error,) m.reject = Mock() m.reject.side_effect = socket.error('foo') callback = self._get_on_message(l) @@ -631,7 +631,7 @@ def test_reset_pidbox_node(self): chan = con.node.channel = Mock() l.connection = Mock() chan.close.side_effect = socket.error('foo') - l.connection_errors = (socket.error, ) + l.connection_errors = (socket.error,) con.reset() chan.close.assert_called_with() @@ -716,7 +716,7 @@ def close(self): def test_connect_errback(self, sleep, connect): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) from kombu.transport.memory import Transport - Transport.connection_errors = (ChannelError, ) + Transport.connection_errors = (ChannelError,) def effect(): if connect.call_count > 1: diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 63242bdbb..b345e283b 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -158,7 +158,7 @@ def _M(*args, **kwargs): if keyfun: key = keyfun(args, kwargs) else: - key = args + (KEYWORD_MARK, ) + tuple(sorted(kwargs.items())) + key = args + (KEYWORD_MARK,) + tuple(sorted(kwargs.items())) try: with mutex: value = cache[key] @@ -314,7 +314,7 @@ def __init__(self, it): self.__it = it def __reduce__(self): - return list, (self.data, ) + return list, (self.data,) def __length_hint__(self): return self.__it.__length_hint__() diff --git a/celery/utils/serialization.py b/celery/utils/serialization.py index 9861dd6cf..598e058a4 100644 --- a/celery/utils/serialization.py +++ b/celery/utils/serialization.py @@ -34,7 +34,7 @@ def subclass_exception(name, parent, module): # noqa - return type(name, (parent, ), {'__module__': module}) + return type(name, (parent,), {'__module__': module}) def find_pickleable_exception(exc, loads=pickle.loads, diff --git a/celery/worker/__init__.py b/celery/worker/__init__.py index 6f7cccc83..24dc777fe 100644 --- a/celery/worker/__init__.py +++ b/celery/worker/__init__.py @@ -218,7 +218,7 @@ def start(self): def register_with_event_loop(self, hub): self.blueprint.send_all( - self, 'register_with_event_loop', args=(hub, ), + self, 'register_with_event_loop', args=(hub,), description='hub.register', ) diff --git a/celery/worker/autoreload.py b/celery/worker/autoreload.py index 03dcc8efd..3613e2004 100644 --- a/celery/worker/autoreload.py +++ b/celery/worker/autoreload.py @@ -46,7 +46,7 @@ class WorkerComponent(bootsteps.StartStopStep): label = 'Autoreloader' conditional = True - requires = (Pool, ) + requires = (Pool,) def __init__(self, w, autoreload=None, **kwargs): self.enabled = w.autoreload = autoreload diff --git a/celery/worker/autoscale.py b/celery/worker/autoscale.py index 265feda49..e8ebe0d25 100644 --- a/celery/worker/autoscale.py +++ b/celery/worker/autoscale.py @@ -39,7 +39,7 @@ class WorkerComponent(bootsteps.StartStopStep): label = 'Autoscaler' conditional = True - requires = (Pool, ) + requires = (Pool,) def __init__(self, w, **kwargs): self.enabled = w.autoscale diff --git a/celery/worker/components.py b/celery/worker/components.py index bb02f4e9e..4b5ae0371 100644 --- a/celery/worker/components.py +++ b/celery/worker/components.py @@ -60,7 +60,7 @@ def on_timer_tick(self, delay): class Hub(bootsteps.StartStopStep): - requires = (Timer, ) + requires = (Timer,) def __init__(self, w, **kwargs): w.hub = None @@ -100,7 +100,7 @@ class Queues(bootsteps.Step): """This bootstep initializes the internal queues used by the worker.""" label = 'Queues (intra)' - requires = (Hub, ) + requires = (Hub,) def create(self, w): w.process_task = w._process_task @@ -123,7 +123,7 @@ class Pool(bootsteps.StartStopStep): * min_concurrency """ - requires = (Queues, ) + requires = (Queues,) def __init__(self, w, autoscale=None, autoreload=None, no_execv=False, optimization=None, **kwargs): diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 356617772..8077f954c 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -262,7 +262,7 @@ def _limit_task(self, request, bucket, tokens): hold = bucket.expected_time(tokens) pri = self._limit_order = (self._limit_order + 1) % 10 self.timer.call_after( - hold, self._limit_move_to_pool, (request, ), + hold, self._limit_move_to_pool, (request,), priority=pri, ) else: @@ -300,7 +300,7 @@ def start(self): def register_with_event_loop(self, hub): self.blueprint.send_all( - self, 'register_with_event_loop', args=(hub, ), + self, 'register_with_event_loop', args=(hub,), description='Hub.register', ) @@ -522,7 +522,7 @@ def info(self, c, params='N/A'): class Events(bootsteps.StartStopStep): - requires = (Connection, ) + requires = (Connection,) def __init__(self, c, send_events=None, **kwargs): self.send_events = True @@ -563,7 +563,7 @@ def shutdown(self, c): class Heart(bootsteps.StartStopStep): - requires = (Events, ) + requires = (Events,) def __init__(self, c, without_heartbeat=False, heartbeat_interval=None, **kwargs): @@ -584,7 +584,7 @@ def stop(self, c): class Mingle(bootsteps.StartStopStep): label = 'Mingle' - requires = (Events, ) + requires = (Events,) compatible_transports = {'amqp', 'redis'} def __init__(self, c, without_mingle=False, **kwargs): @@ -617,7 +617,7 @@ def start(self, c): class Tasks(bootsteps.StartStopStep): - requires = (Mingle, ) + requires = (Mingle,) def __init__(self, c, **kwargs): c.task_consumer = c.qos = None @@ -664,7 +664,7 @@ def info(self, c): class Agent(bootsteps.StartStopStep): conditional = True - requires = (Connection, ) + requires = (Connection,) def __init__(self, c, **kwargs): self.agent_cls = self.enabled = c.app.conf.CELERYD_AGENT @@ -675,7 +675,7 @@ def create(self, c): class Control(bootsteps.StartStopStep): - requires = (Tasks, ) + requires = (Tasks,) def __init__(self, c, **kwargs): self.is_green = c.pool is not None and c.pool.is_green @@ -690,7 +690,7 @@ def include_if(self, c): class Gossip(bootsteps.ConsumerStep): label = 'Gossip' - requires = (Mingle, ) + requires = (Mingle,) _cons_stamp_fields = itemgetter( 'id', 'clock', 'hostname', 'pid', 'topic', 'action', 'cver', ) diff --git a/celery/worker/heartbeat.py b/celery/worker/heartbeat.py index cf46ab0c8..fe2550541 100644 --- a/celery/worker/heartbeat.py +++ b/celery/worker/heartbeat.py @@ -47,7 +47,7 @@ def start(self): if self.eventer.enabled: self._send('worker-online') self.tref = self.timer.call_repeatedly( - self.interval, self._send, ('worker-heartbeat', ), + self.interval, self._send, ('worker-heartbeat',), ) def stop(self): diff --git a/celery/worker/request.py b/celery/worker/request.py index 194358045..0388a0970 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -328,7 +328,7 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): task_ready(self) if isinstance(exc_info.exception, MemoryError): - raise MemoryError('Process got: %s' % (exc_info.exception, )) + raise MemoryError('Process got: %s' % (exc_info.exception,)) elif isinstance(exc_info.exception, Reject): return self.reject(requeue=exc_info.exception.requeue) elif isinstance(exc_info.exception, Ignore): diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index 68115c06d..ac8f2ad50 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -113,7 +113,7 @@ def task_message_handler(message, body, ack, reject, callbacks, req.acknowledge() else: consumer.qos.increment_eventually() - call_at(eta, apply_eta_task, (req, ), priority=6) + call_at(eta, apply_eta_task, (req,), priority=6) else: if rate_limits_enabled: bucket = get_bucket(task.name) diff --git a/docs/configuration.rst b/docs/configuration.rst index 6ed8e206e..614418ae0 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -27,7 +27,7 @@ It should contain all you need to run a basic Celery set-up. BROKER_URL = 'amqp://guest:guest@localhost:5672//' # List of modules to import when celery starts. - CELERY_IMPORTS = ('myapp.tasks', ) + CELERY_IMPORTS = ('myapp.tasks',) ## Using the database to store task state and results. CELERY_RESULT_BACKEND = 'db+sqlite:///results.db' diff --git a/docs/getting-started/brokers/django.rst b/docs/getting-started/brokers/django.rst index d4358d710..b36f40687 100644 --- a/docs/getting-started/brokers/django.rst +++ b/docs/getting-started/brokers/django.rst @@ -30,7 +30,7 @@ configuration values. #. Add :mod:`kombu.transport.django` to `INSTALLED_APPS`:: - INSTALLED_APPS = ('kombu.transport.django', ) + INSTALLED_APPS = ('kombu.transport.django',) #. Sync your database schema: diff --git a/docs/history/changelog-2.5.rst b/docs/history/changelog-2.5.rst index fa395a2c7..133ee8742 100644 --- a/docs/history/changelog-2.5.rst +++ b/docs/history/changelog-2.5.rst @@ -76,7 +76,7 @@ News @task_sent.connect def on_task_sent(**kwargs): - print("sent task: %r" % (kwargs, )) + print("sent task: %r" % (kwargs,)) - Invalid task messages are now rejected instead of acked. @@ -96,8 +96,8 @@ News .. code-block:: python - >>> s = add.subtask((5, )) - >>> new = s.clone(args=(10, ), countdown=5}) + >>> s = add.subtask((5,)) + >>> new = s.clone(args=(10,), countdown=5}) >>> new.args (10, 5) diff --git a/docs/history/changelog-3.1.rst b/docs/history/changelog-3.1.rst index f03e98869..6e748025d 100644 --- a/docs/history/changelog-3.1.rst +++ b/docs/history/changelog-3.1.rst @@ -422,7 +422,7 @@ News exceptions. - **Worker**: No longer sends task error emails for expected errors (in - ``@task(throws=(..., )))``. + ``@task(throws=(...,)))``. - **Canvas**: Fixed problem with exception deserialization when using the JSON serializer (Issue #1987). diff --git a/docs/internals/deprecation.rst b/docs/internals/deprecation.rst index 687c5ed0c..ef68be949 100644 --- a/docs/internals/deprecation.rst +++ b/docs/internals/deprecation.rst @@ -122,7 +122,7 @@ for example:: @task() def add(x, y, task_id=None): - print("My task id is %r" % (task_id, )) + print("My task id is %r" % (task_id,)) should be rewritten into:: diff --git a/docs/userguide/application.rst b/docs/userguide/application.rst index c29d4e16b..5c080ffbe 100644 --- a/docs/userguide/application.rst +++ b/docs/userguide/application.rst @@ -430,7 +430,7 @@ chain breaks: >>> from celery.execute import apply_async - >>> apply_async(hello, ('world!', )) + >>> apply_async(hello, ('world!',)) or you could also create a ``Task`` class to set certain options, or override other behavior diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index bdaf94abb..36cefe9aa 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -41,7 +41,7 @@ The API defines a standard set of execution options, as well as three methods: - ``T.delay(arg, kwarg=value)`` always a shortcut to ``.apply_async``. - - ``T.apply_async((arg, ), {'kwarg': value})`` + - ``T.apply_async((arg,), {'kwarg': value})`` - ``T.apply_async(countdown=10)`` executes 10 seconds from now. diff --git a/docs/userguide/canvas.rst b/docs/userguide/canvas.rst index 4ba43d842..51adfbdbb 100644 --- a/docs/userguide/canvas.rst +++ b/docs/userguide/canvas.rst @@ -110,7 +110,7 @@ creates partials: >>> partial = add.s(2) # incomplete signature >>> partial.delay(4) # 4 + 2 - >>> partial.apply_async((4, )) # same + >>> partial.apply_async((4,)) # same - Any keyword arguments added will be merged with the kwargs in the signature, with the new keyword arguments taking precedence:: @@ -130,7 +130,7 @@ You can also clone signatures to create derivatives: >>> s = add.s(2) proj.tasks.add(2) - >>> s.clone(args=(4, ), kwargs={'debug': True}) + >>> s.clone(args=(4,), kwargs={'debug': True}) proj.tasks.add(2, 4, debug=True) Immutability diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index 3d64dc0ed..1ed9786f0 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -65,7 +65,7 @@ whenever the connection is established: mechanisms. The first one is the ``callbacks`` argument which accepts a list of callbacks with a ``(body, message)`` signature, the second one is the ``on_message`` argument which takes a single - callback with a ``(message, )`` signature. The latter will not + callback with a ``(message,)`` signature. The latter will not automatically decode and deserialize the payload which is useful in many cases: @@ -146,7 +146,7 @@ Attributes .. code-block:: python class WorkerStep(bootsteps.StartStopStep): - requires = ('celery.worker.components:Hub', ) + requires = ('celery.worker.components:Hub',) .. attribute:: pool @@ -158,7 +158,7 @@ Attributes .. code-block:: python class WorkerStep(bootsteps.StartStopStep): - requires = ('celery.worker.components:Pool', ) + requires = ('celery.worker.components:Pool',) .. attribute:: timer @@ -169,7 +169,7 @@ Attributes .. code-block:: python class WorkerStep(bootsteps.StartStopStep): - requires = ('celery.worker.components:Timer', ) + requires = ('celery.worker.components:Timer',) .. attribute:: statedb @@ -183,7 +183,7 @@ Attributes .. code-block:: python class WorkerStep(bootsteps.StartStopStep): - requires = ('celery.worker.components:Statedb', ) + requires = ('celery.worker.components:Statedb',) .. attribute:: autoscaler @@ -197,7 +197,7 @@ Attributes .. code-block:: python class WorkerStep(bootsteps.StartStopStep): - requires = ('celery.worker.autoscaler:Autoscaler', ) + requires = ('celery.worker.autoscaler:Autoscaler',) .. attribute:: autoreloader @@ -210,7 +210,7 @@ Attributes .. code-block:: python class WorkerStep(bootsteps.StartStopStep): - requires = ('celery.worker.autoreloader:Autoreloader', ) + requires = ('celery.worker.autoreloader:Autoreloader',) An example Worker bootstep could be: @@ -219,7 +219,7 @@ An example Worker bootstep could be: from celery import bootsteps class ExampleWorkerStep(bootsteps.StartStopStep): - requires = ('Pool', ) + requires = ('Pool',) def __init__(self, worker, **kwargs): print('Called when the WorkController instance is constructed') @@ -252,7 +252,7 @@ Another example could use the timer to wake up at regular intervals: class DeadlockDetection(bootsteps.StartStopStep): - requires = ('Timer', ) + requires = ('Timer',) def __init__(self, worker, deadlock_timeout=3600): self.timeout = deadlock_timeout @@ -262,7 +262,7 @@ Another example could use the timer to wake up at regular intervals: def start(self, worker): # run every 30 seconds. self.tref = worker.timer.call_repeatedly( - 30.0, self.detect, (worker, ), priority=10, + 30.0, self.detect, (worker,), priority=10, ) def stop(self, worker): @@ -321,7 +321,7 @@ Attributes .. code-block:: python class WorkerStep(bootsteps.StartStopStep): - requires = ('celery.worker:Hub', ) + requires = ('celery.worker:Hub',) .. attribute:: connection @@ -334,7 +334,7 @@ Attributes .. code-block:: python class Step(bootsteps.StartStopStep): - requires = ('celery.worker.consumer:Connection', ) + requires = ('celery.worker.consumer:Connection',) .. attribute:: event_dispatcher @@ -345,7 +345,7 @@ Attributes .. code-block:: python class Step(bootsteps.StartStopStep): - requires = ('celery.worker.consumer:Events', ) + requires = ('celery.worker.consumer:Events',) .. attribute:: gossip @@ -357,7 +357,7 @@ Attributes .. code-block:: python class Step(bootsteps.StartStopStep): - requires = ('celery.worker.consumer:Events', ) + requires = ('celery.worker.consumer:Events',) .. attribute:: pool @@ -378,7 +378,7 @@ Attributes .. code-block:: python class Step(bootsteps.StartStopStep): - requires = ('celery.worker.consumer:Heart', ) + requires = ('celery.worker.consumer:Heart',) .. attribute:: task_consumer @@ -389,7 +389,7 @@ Attributes .. code-block:: python class Step(bootsteps.StartStopStep): - requires = ('celery.worker.consumer:Heart', ) + requires = ('celery.worker.consumer:Heart',) .. attribute:: strategies @@ -409,7 +409,7 @@ Attributes .. code-block:: python class Step(bootsteps.StartStopStep): - requires = ('celery.worker.consumer:Heart', ) + requires = ('celery.worker.consumer:Heart',) .. attribute:: task_buckets diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index 5ba493b5e..2618ab897 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -584,7 +584,7 @@ Combining these you can easily process events in real-time: task = state.tasks.get(event['uuid']) print('TASK FAILED: %s[%s] %s' % ( - task.name, task.uuid, task.info(), )) + task.name, task.uuid, task.info(),)) with app.connection() as connection: recv = app.events.Receiver(connection, handlers={ @@ -620,7 +620,7 @@ You can listen to specific events by specifying the handlers: task = state.tasks.get(event['uuid']) print('TASK FAILED: %s[%s] %s' % ( - task.name, task.uuid, task.info(), )) + task.name, task.uuid, task.info(),)) with app.connection() as connection: recv = app.events.Receiver(connection, handlers={ diff --git a/docs/userguide/routing.rst b/docs/userguide/routing.rst index 0656a8515..8b0705436 100644 --- a/docs/userguide/routing.rst +++ b/docs/userguide/routing.rst @@ -535,11 +535,11 @@ becomes --> You install router classes by adding them to the :setting:`CELERY_ROUTES` setting:: - CELERY_ROUTES = (MyRouter(), ) + CELERY_ROUTES = (MyRouter(),) Router classes can also be added by name:: - CELERY_ROUTES = ('myapp.routers.MyRouter', ) + CELERY_ROUTES = ('myapp.routers.MyRouter',) For simple task name -> route mappings like the router example above, @@ -548,10 +548,12 @@ same behavior: .. code-block:: python - CELERY_ROUTES = ({'myapp.tasks.compress_video': { - 'queue': 'video', - 'routing_key': 'video.compress' - }}, ) + CELERY_ROUTES = ( + {'myapp.tasks.compress_video': { + 'queue': 'video', + 'routing_key': 'video.compress', + }}, + ) The routers will then be traversed in order, it will stop at the first router returning a true value, and use that as the final route for the task. @@ -567,7 +569,7 @@ copies of tasks to all workers connected to it: from kombu.common import Broadcast - CELERY_QUEUES = (Broadcast('broadcast_tasks'), ) + CELERY_QUEUES = (Broadcast('broadcast_tasks'),) CELERY_ROUTES = {'tasks.reload_cache': {'queue': 'broadcast_tasks'}} diff --git a/docs/whatsnew-2.5.rst b/docs/whatsnew-2.5.rst index 08dc3135f..ec3d2e721 100644 --- a/docs/whatsnew-2.5.rst +++ b/docs/whatsnew-2.5.rst @@ -288,7 +288,7 @@ You can change methods too, for example the ``on_failure`` handler: .. code-block:: python def my_on_failure(self, exc, task_id, args, kwargs, einfo): - print('Oh no! Task failed: %r' % (exc, )) + print('Oh no! Task failed: %r' % (exc,)) CELERY_ANNOTATIONS = {'*': {'on_failure': my_on_failure}} diff --git a/docs/whatsnew-3.0.rst b/docs/whatsnew-3.0.rst index abadd7182..24dd072f9 100644 --- a/docs/whatsnew-3.0.rst +++ b/docs/whatsnew-3.0.rst @@ -315,7 +315,7 @@ Tasks can now have callbacks and errbacks, and dependencies are recorded # (2 + 2) * 8 / 2 >>> res = chain(add.subtask((2, 2)), - mul.subtask((8, )), + mul.subtask((8,)), div.subtask((2,))).apply_async() >>> res.get() == 16 @@ -633,7 +633,7 @@ without also initializing the app environment:: abstract = True def __call__(self, *args, **kwargs): - print('CALLING %r' % (self, )) + print('CALLING %r' % (self,)) return self.run(*args, **kwargs) >>> DebugTask @@ -742,7 +742,7 @@ In Other News @wraps(fun) def _inner(*args, **kwargs): - print('ARGS: %r' % (args, )) + print('ARGS: %r' % (args,)) return _inner CELERY_ANNOTATIONS = { diff --git a/examples/eventlet/README.rst b/examples/eventlet/README.rst index 6bf00e9fa..eb64b7081 100644 --- a/examples/eventlet/README.rst +++ b/examples/eventlet/README.rst @@ -46,7 +46,7 @@ To open several URLs at once you can do:: >>> result = group(urlopen.s(url) ... for url in LIST_OF_URLS).apply_async() >>> for incoming_result in result.iter_native(): - ... print(incoming_result, ) + ... print(incoming_result) * `webcrawler.crawl` diff --git a/examples/gevent/celeryconfig.py b/examples/gevent/celeryconfig.py index c7d94783f..e3714f277 100644 --- a/examples/gevent/celeryconfig.py +++ b/examples/gevent/celeryconfig.py @@ -10,4 +10,4 @@ CELERY_RESULT_BACKEND = 'amqp' CELERY_TASK_RESULT_EXPIRES = 30 * 60 -CELERY_IMPORTS = ('tasks', ) +CELERY_IMPORTS = ('tasks',) diff --git a/funtests/suite/config.py b/funtests/suite/config.py index 741df4b40..8060126b7 100644 --- a/funtests/suite/config.py +++ b/funtests/suite/config.py @@ -12,7 +12,7 @@ CELERYD_LOG_COLOR = False -CELERY_IMPORTS = ('celery.tests.functional.tasks', ) +CELERY_IMPORTS = ('celery.tests.functional.tasks',) @atexit.register diff --git a/setup.py b/setup.py index 2c28e4cfe..136318076 100644 --- a/setup.py +++ b/setup.py @@ -84,16 +84,16 @@ def add_default(m): attr_name, attr_value = m.groups() - return ((attr_name, rq(attr_value)), ) + return ((attr_name, rq(attr_value)),) def add_version(m): v = list(map(rq, m.groups()[0].split(', '))) - return (('VERSION', '.'.join(v[0:3]) + ''.join(v[3:])), ) + return (('VERSION', '.'.join(v[0:3]) + ''.join(v[3:])),) def add_doc(m): - return (('doc', m.groups()[0]), ) + return (('doc', m.groups()[0]),) pats = {re_meta: add_default, re_vers: add_version, From 3574d014caa5b3f14c7b484b5011cc904ce8d38b Mon Sep 17 00:00:00 2001 From: Steven Parker Date: Sat, 11 Jul 2015 04:16:15 -0700 Subject: [PATCH 0547/1103] Fixing simple misplaced parenthesis. --- docs/userguide/canvas.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/userguide/canvas.rst b/docs/userguide/canvas.rst index 51adfbdbb..59d19c951 100644 --- a/docs/userguide/canvas.rst +++ b/docs/userguide/canvas.rst @@ -911,11 +911,11 @@ while calling ``.apply_async`` will create a dedicated task so that the individual tasks are applied in a worker instead:: - >>> add.chunks(zip(range(100), range(100), 10)).apply_async() + >>> add.chunks(zip(range(100), range(100)), 10).apply_async() You can also convert chunks to a group:: - >>> group = add.chunks(zip(range(100), range(100), 10)).group() + >>> group = add.chunks(zip(range(100), range(100)), 10).group() and with the group skew the countdown of each task by increments of one:: From 8470bbb439db6b783f32db00afb235748cc1320d Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Sat, 30 Aug 2014 13:32:01 -0700 Subject: [PATCH 0548/1103] Fix issue #2225 Creating a chord no longer results in "TypeError: group object got multiple values for keyword argument 'task_id'". Chords now complete without hanging. --- celery/app/amqp.py | 1 + celery/backends/base.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 136f5db53..640442b8c 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -371,6 +371,7 @@ def as_task_v1(self, task_id, name, args=None, kwargs=None, 'id': task_id, 'args': args, 'kwargs': kwargs, + 'group': group_id, 'retries': retries, 'eta': eta, 'expires': expires, diff --git a/celery/backends/base.py b/celery/backends/base.py index e3201e437..6502c08f0 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -534,7 +534,11 @@ def _restore_group(self, group_id): def _apply_chord_incr(self, header, partial_args, group_id, body, result=None, options={}, **kwargs): self.save_group(group_id, self.app.GroupResult(group_id, result)) - return header(*partial_args, task_id=group_id, **options or {}) + + fixed_options = dict((k,v) for k,v in options.items() if k != 'task_id') + + return header(*partial_args, task_id=group_id, **fixed_options or {}) + def on_chord_part_return(self, task, state, result, propagate=None): if not self.implements_incr: From e6ae13bd281e9a12a02ec051733f21a7a0a0a9c1 Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Sun, 31 Aug 2014 21:22:40 -0700 Subject: [PATCH 0549/1103] Fix issue mentioned in https://github.com/celery/celery/issues/1671 See the comment from @lance-burton on June 20, 2014. A nested group in an expression such as: c = (group(add.s(1,1),add.s(2,2)) | add.s(1) | add.s(1) | group(mul.s(1),mul.s(2))) res = c.apply_async().get() Causes an "AttributeError: 'dict' object has no attribute 'type'". --- celery/canvas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index 3aafd52a8..eefca82af 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -602,7 +602,7 @@ def _maybe_group(tasks): elif isinstance(tasks, Signature): tasks = [tasks] else: - tasks = regen(tasks) + tasks = map(signature, regen(tasks)) return tasks From 8e3a2e88e905a517e3969f23ad200a5b258fc535 Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Sun, 31 Aug 2014 21:57:53 -0700 Subject: [PATCH 0550/1103] Fix additional issue #2225 Earlier commit with the same title missed one of the cases causing the duplicate task_id argument error (i.e., when using AMQP). This commit addresses the issue. --- celery/backends/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 6502c08f0..e561ce722 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -357,8 +357,8 @@ def fallback_chord_unlock(self, group_id, body, result=None, def apply_chord(self, header, partial_args, group_id, body, options={}, **kwargs): - options['task_id'] = group_id - result = header(*partial_args, **options or {}) + fixed_options = dict((k,v) for k,v in options.items() if k!='task_id') + result = header(*partial_args, task_id=group_id, **fixed_options or {}) self.fallback_chord_unlock(group_id, body, **kwargs) return result From af368f50f08ce817a0c1b49b398b5f1485a95013 Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Mon, 1 Sep 2014 01:01:32 -0700 Subject: [PATCH 0551/1103] Fix issue #2228 Fixes the bug where the wrong result is returned when a chain contains a chord as the penultimate task. https://github.com/celery/celery/issues/2228 --- celery/canvas.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index eefca82af..1e72d7b0c 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -473,9 +473,9 @@ def prepare_steps(self, args, tasks, if link_error: task.set(link_error=link_error) - if not isinstance(prev_task, chord): - results.append(res) - tasks.append(task) + tasks.append(task) + results.append(res) + prev_task, prev_res = task, res return tasks, results From 10d263765386b68930d41161ddc33de203cb9ef2 Mon Sep 17 00:00:00 2001 From: PMickael Date: Mon, 13 Jul 2015 00:31:03 +0200 Subject: [PATCH 0552/1103] Erase by Merge 72b16ac, shadow name exception --- celery/app/task.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/app/task.py b/celery/app/task.py index f56027c91..9d0991dc5 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -471,13 +471,14 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, if self.__self__ is not None: args = args if isinstance(args, tuple) else tuple(args or ()) args = (self.__self__,) + args + shadow = shadow or self.shadow_name(args, kwargs, options) preopts = self._get_exec_options() options = dict(preopts, **options) if options else preopts return app.send_task( self.name, args, kwargs, task_id=task_id, producer=producer, link=link, link_error=link_error, result_cls=self.AsyncResult, - shadow=shadow or self.shadow_name(args, kwargs, options), + shadow=shadow, **options ) From 3d00cc63c52401ae7002d29ec221a6286140af83 Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Sat, 30 Aug 2014 13:32:01 -0700 Subject: [PATCH 0553/1103] Fix issue #2225 Creating a chord no longer results in "TypeError: group object got multiple values for keyword argument 'task_id'". Chords now complete without hanging. --- celery/app/amqp.py | 1 + celery/backends/base.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 136f5db53..640442b8c 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -371,6 +371,7 @@ def as_task_v1(self, task_id, name, args=None, kwargs=None, 'id': task_id, 'args': args, 'kwargs': kwargs, + 'group': group_id, 'retries': retries, 'eta': eta, 'expires': expires, diff --git a/celery/backends/base.py b/celery/backends/base.py index e3201e437..6502c08f0 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -534,7 +534,11 @@ def _restore_group(self, group_id): def _apply_chord_incr(self, header, partial_args, group_id, body, result=None, options={}, **kwargs): self.save_group(group_id, self.app.GroupResult(group_id, result)) - return header(*partial_args, task_id=group_id, **options or {}) + + fixed_options = dict((k,v) for k,v in options.items() if k != 'task_id') + + return header(*partial_args, task_id=group_id, **fixed_options or {}) + def on_chord_part_return(self, task, state, result, propagate=None): if not self.implements_incr: From 05f84b34b1c7dc6ec1024b12cf32e266736375bc Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Sun, 31 Aug 2014 21:22:40 -0700 Subject: [PATCH 0554/1103] Fix issue mentioned in https://github.com/celery/celery/issues/1671 See the comment from @lance-burton on June 20, 2014. A nested group in an expression such as: c = (group(add.s(1,1),add.s(2,2)) | add.s(1) | add.s(1) | group(mul.s(1),mul.s(2))) res = c.apply_async().get() Causes an "AttributeError: 'dict' object has no attribute 'type'". --- celery/canvas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index 3aafd52a8..eefca82af 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -602,7 +602,7 @@ def _maybe_group(tasks): elif isinstance(tasks, Signature): tasks = [tasks] else: - tasks = regen(tasks) + tasks = map(signature, regen(tasks)) return tasks From 16760602cd1beaf029583db30f8283d4ca864fc0 Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Sun, 31 Aug 2014 21:57:53 -0700 Subject: [PATCH 0555/1103] Fix additional issue #2225 Earlier commit with the same title missed one of the cases causing the duplicate task_id argument error (i.e., when using AMQP). This commit addresses the issue. --- celery/backends/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 6502c08f0..e561ce722 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -357,8 +357,8 @@ def fallback_chord_unlock(self, group_id, body, result=None, def apply_chord(self, header, partial_args, group_id, body, options={}, **kwargs): - options['task_id'] = group_id - result = header(*partial_args, **options or {}) + fixed_options = dict((k,v) for k,v in options.items() if k!='task_id') + result = header(*partial_args, task_id=group_id, **fixed_options or {}) self.fallback_chord_unlock(group_id, body, **kwargs) return result From 466babf42e56d67f110a1a5c74e4a4b5ef995a4e Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Mon, 1 Sep 2014 01:01:32 -0700 Subject: [PATCH 0556/1103] Fix issue #2228 Fixes the bug where the wrong result is returned when a chain contains a chord as the penultimate task. https://github.com/celery/celery/issues/2228 --- celery/canvas.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index eefca82af..1e72d7b0c 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -473,9 +473,9 @@ def prepare_steps(self, args, tasks, if link_error: task.set(link_error=link_error) - if not isinstance(prev_task, chord): - results.append(res) - tasks.append(task) + tasks.append(task) + results.append(res) + prev_task, prev_res = task, res return tasks, results From ad61921d4865ccb2a8cce046f5b9f60d684902f5 Mon Sep 17 00:00:00 2001 From: Aaron McMillin Date: Mon, 13 Jul 2015 11:11:57 -0400 Subject: [PATCH 0557/1103] Update whatsnew-3.2.rst from https://github.com/celery/celery/commit/07ecd08a8621affde3b8ed15d118164cb26e334d commit message. --- docs/whatsnew-3.2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/whatsnew-3.2.rst b/docs/whatsnew-3.2.rst index d75b6e9a8..c7effaef1 100644 --- a/docs/whatsnew-3.2.rst +++ b/docs/whatsnew-3.2.rst @@ -126,7 +126,7 @@ Task.replace A new builtin task (`celery.accumulate` was added for this purpose) - Closes #81 + Closes #817 Optimized Beat implementation From bc964c4bd4755c36c90c7c93c3ef0956928f2016 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 13 Jul 2015 14:01:15 -0700 Subject: [PATCH 0558/1103] flakes --- celery/backends/couchbase.py | 2 +- celery/canvas.py | 2 +- funtests/suite/test_leak.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/celery/backends/couchbase.py b/celery/backends/couchbase.py index d94960ed3..793a69d88 100644 --- a/celery/backends/couchbase.py +++ b/celery/backends/couchbase.py @@ -39,7 +39,7 @@ class CouchBaseBackend(KeyValueStoreBackend): timeout = 2.5 transcoder = None # supports_autoexpire = False - + # Use str as couchbase key not bytes key_t = str_t diff --git a/celery/canvas.py b/celery/canvas.py index 3aafd52a8..9075d8776 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -392,7 +392,7 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, try: tasks, results = self._frozen - except (AttributeError, ValueError): + except (AttributeError, ValueError, TypeError): tasks, results = self.prepare_steps( args, self.tasks, root_id, link_error, app, task_id, group_id, chord, diff --git a/funtests/suite/test_leak.py b/funtests/suite/test_leak.py index b19c23f41..98ea07a54 100644 --- a/funtests/suite/test_leak.py +++ b/funtests/suite/test_leak.py @@ -127,5 +127,6 @@ def task2(): finally: self.app.conf.BROKER_POOL_LIMIT = pool_limit + if __name__ == '__main__': unittest.main() From 5b025713018ad0b86619c84f221f8f54ea2c711d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 13 Jul 2015 14:14:57 -0700 Subject: [PATCH 0559/1103] Attempt to fix tests on Travis --- celery/canvas.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 9075d8776..3e272f445 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -390,13 +390,14 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, args = (tuple(args) + tuple(self.args) if args and not self.immutable else self.args) - try: - tasks, results = self._frozen - except (AttributeError, ValueError, TypeError): + if self._frozen: + tasks, result = self._frozen + else: tasks, results = self.prepare_steps( args, self.tasks, root_id, link_error, app, task_id, group_id, chord, ) + if results: # make sure we can do a link() and link_error() on a chain object. if link: From 7566d2dbe7483d0dd784da95597bfc33b652ceb5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 13 Jul 2015 14:17:30 -0700 Subject: [PATCH 0560/1103] Fixes typo in docstring for Issue #817 --- celery/app/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/task.py b/celery/app/task.py index 9d0991dc5..d94f15702 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -741,7 +741,7 @@ def replace(self, sig): :param sig: :class:`@signature` Note: This will raise :exc:`~@Ignore`, so the best practice - is to always use ``raise self.replace_in_chord(...)`` to convey + is to always use ``raise self.replace(...)`` to convey to the reader that the task will not continue after being replaced. :param: Signature of new task. From a8621d687c5ef9707edc1f6cbac4ba73eec725b0 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 14 Jul 2015 09:34:08 +0300 Subject: [PATCH 0561/1103] Fixed typo in assertion. --- celery/tests/utils/test_mail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/tests/utils/test_mail.py b/celery/tests/utils/test_mail.py index 4006fb0b5..e4fc9650d 100644 --- a/celery/tests/utils/test_mail.py +++ b/celery/tests/utils/test_mail.py @@ -46,7 +46,7 @@ def test_send(self, SMTP): mailer = Mailer(use_ssl=False, use_tls=False) mailer._send(msg) - client.sendmail.assert_called_With(msg.sender, msg.to, str(msg)) + client.sendmail.assert_called_with(msg.sender, msg.to, str(msg)) client.quit.side_effect = SSLError() mailer._send(msg) From 4fd22bb88aeef1385ce9d057f46cedfac07b569a Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 14 Jul 2015 09:43:43 +0300 Subject: [PATCH 0562/1103] Added pip caching and moved the build to the new infrastructure. --- .travis.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index eae9ac385..365248d2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,8 @@ language: python +sudo: false +cache: + directories: + - $HOME/.cache/pip python: 2.7 env: global: @@ -8,11 +12,6 @@ env: - TOXENV=3.3 - TOXENV=3.4 - TOXENV=pypy -before_install: - - | - python --version - uname -a - lsb_release -a install: - pip install tox script: From 8496a51ed9374089d3fddd76ca13bbeb3dbdeacf Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 14 Jul 2015 10:24:00 +0300 Subject: [PATCH 0563/1103] Use dict comprehension instead of trasforming a generator into a dict. --- celery/backends/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index e561ce722..9bed68850 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -535,7 +535,7 @@ def _apply_chord_incr(self, header, partial_args, group_id, body, result=None, options={}, **kwargs): self.save_group(group_id, self.app.GroupResult(group_id, result)) - fixed_options = dict((k,v) for k,v in options.items() if k != 'task_id') + fixed_options = {k: v for k,v in options.items() if k != 'task_id'} return header(*partial_args, task_id=group_id, **fixed_options or {}) From b242f1bffa839e27f53677720cca81174637e0f7 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 14 Jul 2015 10:25:31 +0300 Subject: [PATCH 0564/1103] Use dict comprehension instead of trasforming a generator into a dict. --- celery/backends/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 9bed68850..781206b7f 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -357,7 +357,7 @@ def fallback_chord_unlock(self, group_id, body, result=None, def apply_chord(self, header, partial_args, group_id, body, options={}, **kwargs): - fixed_options = dict((k,v) for k,v in options.items() if k!='task_id') + fixed_options = {k: v for k,v in options.items() if k!='task_id'} result = header(*partial_args, task_id=group_id, **fixed_options or {}) self.fallback_chord_unlock(group_id, body, **kwargs) return result From a50bfd8f340cd4ce525c8a4ac56d3d6f3ee86939 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 14 Jul 2015 16:13:47 -0700 Subject: [PATCH 0565/1103] Redis: new_join does not need to support CHORD_PROPAGATES --- celery/backends/redis.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index beefdbb11..fb1eaba6d 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -196,11 +196,8 @@ def _new_chord_apply(self, header, partial_args, group_id, body, options['task_id'] = group_id return header(*partial_args, **options or {}) - def _new_chord_return(self, task, state, result, propagate=None, - PROPAGATE_STATES=states.PROPAGATE_STATES): + def _new_chord_return(self, task, state, result, propagate=None): app = self.app - if propagate is None: - propagate = self.app.conf.CELERY_CHORD_PROPAGATES request = task.request tid, gid = request.id, request.group if not gid or not tid: From c0f492205bde9fae30841239dc5dc5d6b2e2a5ce Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 17 Jul 2015 12:45:41 -0700 Subject: [PATCH 0566/1103] Fixes typo "unbound error: results" --- celery/canvas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index 3b7ff98de..8a4e6b24c 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -391,7 +391,7 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, if args and not self.immutable else self.args) if self._frozen: - tasks, result = self._frozen + tasks, results = self._frozen else: tasks, results = self.prepare_steps( args, self.tasks, root_id, link_error, app, From 00551933e04fa421bc81f5e610e86d7482690f8a Mon Sep 17 00:00:00 2001 From: Aaron McMillin Date: Fri, 17 Jul 2015 00:42:05 -0400 Subject: [PATCH 0567/1103] If this chain was in a group, the args from the group are already on self --- celery/canvas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index 8a4e6b24c..a3ba2df25 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -407,7 +407,7 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, def freeze(self, _id=None, group_id=None, chord=None, root_id=None): _, results = self._frozen = self.prepare_steps( - (), self.tasks, root_id, None, self.app, _id, group_id, chord, + self.args, self.tasks, root_id, None, self.app, _id, group_id, chord, ) return results[-1] From bf944e4e767d34207afd12ec83a06a3bfc825036 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 17 Jul 2015 13:47:48 -0700 Subject: [PATCH 0568/1103] Fixes bug with incorrect id set when subtask is a chain --- celery/canvas.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index a3ba2df25..cbb950bc6 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -408,13 +408,14 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, def freeze(self, _id=None, group_id=None, chord=None, root_id=None): _, results = self._frozen = self.prepare_steps( self.args, self.tasks, root_id, None, self.app, _id, group_id, chord, + clone=False, ) return results[-1] def prepare_steps(self, args, tasks, root_id=None, link_error=None, app=None, last_task_id=None, group_id=None, chord_body=None, - from_dict=Signature.from_dict): + clone=True, from_dict=Signature.from_dict): app = app or self.app steps = deque(tasks) next_step = prev_task = prev_res = None @@ -429,7 +430,8 @@ def prepare_steps(self, args, tasks, task = maybe_unroll_group(task) # first task gets partial args from chain - task = task.clone(args) if not i else task.clone() + if clone: + task = task.clone(args) if not i else task.clone() if isinstance(task, chain): # splice the chain @@ -655,7 +657,7 @@ def _apply_tasks(self, tasks, producer=None, app=None, for sig, res in tasks: sig.apply_async(producer=producer, add_to_parent=False, **options) - yield res + yield res # <-- r.parent, etc set in the frozen result. def _freeze_gid(self, options): # remove task_id and use that as the group_id, From bef6847b679bb876adda3aeeb068e56e3e94c1e0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 17 Jul 2015 13:58:10 -0700 Subject: [PATCH 0569/1103] Also pass partial args for frozen chain --- celery/canvas.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/celery/canvas.py b/celery/canvas.py index cbb950bc6..2f216f4de 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -432,6 +432,8 @@ def prepare_steps(self, args, tasks, # first task gets partial args from chain if clone: task = task.clone(args) if not i else task.clone() + elif not i: + task.args = tuple(args) + tuple(task.args) if isinstance(task, chain): # splice the chain From 61aca5ff6a2f9cc1ac9721dafa941051e16e5553 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 24 Jul 2015 15:13:08 -0700 Subject: [PATCH 0570/1103] Task: Retry/signature_from_request should include headers. Closes #2706 --- celery/app/task.py | 3 ++- celery/tests/tasks/test_tasks.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/celery/app/task.py b/celery/app/task.py index d94f15702..920232529 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -520,9 +520,10 @@ def signature_from_request(self, request=None, args=None, kwargs=None, 'soft_time_limit': limit_soft, 'time_limit': limit_hard, 'reply_to': request.reply_to, + 'headers': request.headers, } options.update( - {'queue': queue} if queue else (request.delivery_info or {}) + {'queue': queue} if queue else (request.delivery_info or {}), ) return self.signature( args, kwargs, options, type=self, **extra_options diff --git a/celery/tests/tasks/test_tasks.py b/celery/tests/tasks/test_tasks.py index dca6d2cf1..d135f13e3 100644 --- a/celery/tests/tasks/test_tasks.py +++ b/celery/tests/tasks/test_tasks.py @@ -124,6 +124,22 @@ def test_retry_no_args(self): self.retry_task_noargs.apply(propagate=True).get() self.assertEqual(self.retry_task_noargs.iterations, 4) + def test_signature_from_request__passes_headers(self): + self.retry_task.push_request() + self.retry_task.request.headers = {'custom': 10.1} + sig = self.retry_task.signature_from_request() + self.assertEqual(sig.options['headers']['custom'], 10.1) + + def test_signature_from_request__delivery_info(self): + self.retry_task.push_request() + self.retry_task.request.delivery_info = { + 'exchange': 'testex', + 'routing_key': 'testrk', + } + sig = self.retry_task.signature_from_request() + self.assertEqual(sig.options['exchange'], 'testex') + self.assertEqual(sig.options['routing_key'], 'testrk') + def test_retry_kwargs_can_be_empty(self): self.retry_task_mockapply.push_request() try: From bd1edfe78c37ebb92ff198421252234e895c6afa Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Mon, 27 Jul 2015 09:13:43 -0400 Subject: [PATCH 0571/1103] Add additional information about the backend_cleanup task --- docs/configuration.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 614418ae0..49ca75877 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1352,7 +1352,8 @@ Time (in seconds, or a :class:`~datetime.timedelta` object) for when after stored task tombstones will be deleted. A built-in periodic task will delete the results after this time -(:class:`celery.task.backend_cleanup`). +(``celery.backend_cleanup``), assuming that ``celery beat`` is +enabled. The task runs daily at 4am. A value of :const:`None` or 0 means results will never expire (depending on backend specifications). From 858e312611f44917c00cfd5e429f833648b5925e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 27 Jul 2015 12:22:20 -0700 Subject: [PATCH 0572/1103] Result: Exception can be None. Closes #2687 --- celery/backends/base.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 781206b7f..9669675f1 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -167,11 +167,12 @@ def prepare_exception(self, exc, serializer=None): def exception_to_python(self, exc): """Convert serialized exception to Python exception.""" - if not isinstance(exc, BaseException): - exc = create_exception_cls( - from_utf8(exc['exc_type']), __name__)(exc['exc_message']) - if self.serializer in EXCEPTION_ABLE_CODECS: - exc = get_pickled_exception(exc) + if exc: + if not isinstance(exc, BaseException): + exc = create_exception_cls( + from_utf8(exc['exc_type']), __name__)(exc['exc_message']) + if self.serializer in EXCEPTION_ABLE_CODECS: + exc = get_pickled_exception(exc) return exc def prepare_value(self, result): From 5d3c555a66e8e7d86d25fba02ce65f4659326a88 Mon Sep 17 00:00:00 2001 From: Marco Buttu Date: Wed, 29 Jul 2015 17:05:08 +0200 Subject: [PATCH 0573/1103] Fixed some typos --- docs/getting-started/first-steps-with-celery.rst | 2 +- docs/userguide/signals.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started/first-steps-with-celery.rst b/docs/getting-started/first-steps-with-celery.rst index 91d3e60ab..fd152df73 100644 --- a/docs/getting-started/first-steps-with-celery.rst +++ b/docs/getting-started/first-steps-with-celery.rst @@ -280,7 +280,7 @@ It has an input and an output, where you must connect the input to a broker and the output to a result backend if so wanted. But if you look closely at the back there's a lid revealing loads of sliders, dials and buttons: this is the configuration. -The default configuration should be good enough for most uses, but there's +The default configuration should be good enough for most uses, but there are many things to tweak so Celery works just the way you want it to. Reading about the options available is a good idea to get familiar with what can be configured. You can read about the options in the diff --git a/docs/userguide/signals.rst b/docs/userguide/signals.rst index bfa2c5b5c..8be7f37c2 100644 --- a/docs/userguide/signals.rst +++ b/docs/userguide/signals.rst @@ -10,7 +10,7 @@ Signals Signals allows decoupled applications to receive notifications when certain actions occur elsewhere in the application. -Celery ships with many signals that you application can hook into +Celery ships with many signals that your application can hook into to augment behavior of certain actions. .. _signal-basics: From fc85b5f407a859380740583616e2370ea126e743 Mon Sep 17 00:00:00 2001 From: "D. Yu" Date: Sat, 8 Aug 2015 11:37:24 +0800 Subject: [PATCH 0574/1103] Hint for CELERYBEAT_SCHEDULE args for 1-item tuple It took me a long time to figure out(around 2 days of trying out different things) why I was getting `celerybeat raised exception : TypeError('argument 2 to map() must support iteration',)` I was testing it out with a Task that only had one argument and mapped `args` to `(3)` instead of `(3,) --- docs/userguide/periodic-tasks.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/userguide/periodic-tasks.rst b/docs/userguide/periodic-tasks.rst index a1546bdf5..cac985280 100644 --- a/docs/userguide/periodic-tasks.rst +++ b/docs/userguide/periodic-tasks.rst @@ -131,6 +131,9 @@ Example: Run the `tasks.add` task every 30 seconds. please see :ref:`celerytut-configuration`. You can either set these options on your app directly or you can keep a separate module for configuration. + + If you want to use a single item tuple for `args`, don't forget + that the constructor is a comma and not a pair of parentheses. Using a :class:`~datetime.timedelta` for the schedule means the task will be sent in 30 second intervals (the first task will be sent 30 seconds From 41cb188783d5ed1c458ad3eba1620329bc9959e7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 12 Aug 2015 16:31:48 -0700 Subject: [PATCH 0575/1103] flakes --- celery/backends/base.py | 5 ++--- celery/canvas.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 9669675f1..c4dffaaa6 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -358,7 +358,7 @@ def fallback_chord_unlock(self, group_id, body, result=None, def apply_chord(self, header, partial_args, group_id, body, options={}, **kwargs): - fixed_options = {k: v for k,v in options.items() if k!='task_id'} + fixed_options = {k: v for k, v in items(options) if k != 'task_id'} result = header(*partial_args, task_id=group_id, **fixed_options or {}) self.fallback_chord_unlock(group_id, body, **kwargs) return result @@ -536,11 +536,10 @@ def _apply_chord_incr(self, header, partial_args, group_id, body, result=None, options={}, **kwargs): self.save_group(group_id, self.app.GroupResult(group_id, result)) - fixed_options = {k: v for k,v in options.items() if k != 'task_id'} + fixed_options = {k: v for k, v in items(options) if k != 'task_id'} return header(*partial_args, task_id=group_id, **fixed_options or {}) - def on_chord_part_return(self, task, state, result, propagate=None): if not self.implements_incr: return diff --git a/celery/canvas.py b/celery/canvas.py index 2f216f4de..719729c36 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -407,8 +407,8 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, def freeze(self, _id=None, group_id=None, chord=None, root_id=None): _, results = self._frozen = self.prepare_steps( - self.args, self.tasks, root_id, None, self.app, _id, group_id, chord, - clone=False, + self.args, self.tasks, root_id, None, + self.app, _id, group_id, chord, clone=False, ) return results[-1] From 44c7452da8f0b83e8c7b2cce169da587852288ec Mon Sep 17 00:00:00 2001 From: Carlos Garcia-Dubus Date: Fri, 14 Aug 2015 16:15:16 -0700 Subject: [PATCH 0576/1103] Change tasks.reload_tasks to tasks.reload_cache 'tasks.reload_tasks' should be 'tasks.reload_cache' on userguide/routing/broadcast docs. --- docs/userguide/routing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/routing.rst b/docs/userguide/routing.rst index 8b0705436..b248e70f2 100644 --- a/docs/userguide/routing.rst +++ b/docs/userguide/routing.rst @@ -573,7 +573,7 @@ copies of tasks to all workers connected to it: CELERY_ROUTES = {'tasks.reload_cache': {'queue': 'broadcast_tasks'}} -Now the ``tasks.reload_tasks`` task will be sent to every +Now the ``tasks.reload_cache`` task will be sent to every worker consuming from this queue. .. admonition:: Broadcast & Results From 4f9e965873f7466e21a20be7c55f0e28d56c0a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ionel=20Cristian=20M=C4=83rie=C8=99?= Date: Thu, 20 Aug 2015 18:02:19 +0300 Subject: [PATCH 0577/1103] The object is not really iterable, just return it. --- celery/canvas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index 719729c36..1f0775315 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -93,7 +93,7 @@ def maybe_unroll_group(g): try: size = g.tasks.__length_hint__() except (AttributeError, TypeError): - pass + return g else: return list(g.tasks)[0] if size == 1 else g else: From cae8bf96c0bc9b7fcfbc46c10b7cacef164daef5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ionel=20Cristian=20M=C4=83rie=C8=99?= Date: Thu, 20 Aug 2015 18:33:18 +0300 Subject: [PATCH 0578/1103] Don't use `map` here - it doesn't make a list on Python 3. --- celery/canvas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index 1f0775315..a2edd3817 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -607,7 +607,7 @@ def _maybe_group(tasks): elif isinstance(tasks, Signature): tasks = [tasks] else: - tasks = map(signature, regen(tasks)) + tasks = [signature(t) for t in regen(tasks)] return tasks From 7d71d241f5881788ea20550d7679e64c173b8724 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ionel=20Cristian=20M=C4=83rie=C8=99?= Date: Thu, 20 Aug 2015 18:38:44 +0300 Subject: [PATCH 0579/1103] Don't fail if m doesn't have a __class__ attr. --- celery/tests/case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/tests/case.py b/celery/tests/case.py index ad94d3b57..26f495e15 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -232,7 +232,7 @@ def _is_magic_module(m): # will load _tkinter and other shit when touched. # pyflakes refuses to accept 'noqa' for this isinstance. - cls, modtype = m.__class__, types.ModuleType + cls, modtype = getattr(m, '__class__', None), types.ModuleType return (cls is not modtype and ( '__getattr__' in vars(m.__class__) or '__getattribute__' in vars(m.__class__))) From d7648985b3db53fc48a24f2b131a06483152241d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ionel=20Cristian=20M=C4=83rie=C8=99?= Date: Thu, 20 Aug 2015 18:49:27 +0300 Subject: [PATCH 0580/1103] Improve the magic module check. --- celery/tests/case.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/celery/tests/case.py b/celery/tests/case.py index 26f495e15..accc6a1f2 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -232,10 +232,15 @@ def _is_magic_module(m): # will load _tkinter and other shit when touched. # pyflakes refuses to accept 'noqa' for this isinstance. - cls, modtype = getattr(m, '__class__', None), types.ModuleType - return (cls is not modtype and ( - '__getattr__' in vars(m.__class__) or - '__getattribute__' in vars(m.__class__))) + cls, modtype = type(m), types.ModuleType + try: + variables = vars(cls) + except TypeError: + return True + else: + return (cls is not modtype and ( + '__getattr__' in variables or + '__getattribute__' in variables)) class _AssertWarnsContext(_AssertRaisesBaseContext): From 02fa051ff6b9c4706acc41358ec7f92e76f72c67 Mon Sep 17 00:00:00 2001 From: Steve Peak Date: Fri, 21 Aug 2015 14:28:12 -0400 Subject: [PATCH 0581/1103] switch to Codecov for coverage reporting --- .travis.yml | 2 +- README.rst | 4 ++-- requirements/test-ci.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 365248d2c..7695827dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ install: script: - tox -v -- -v after_success: - - .tox/$TRAVIS_PYTHON_VERSION/bin/coveralls + - .tox/$TRAVIS_PYTHON_VERSION/bin/codecov -e TOXENV notifications: irc: channels: diff --git a/README.rst b/README.rst index 3391e16be..5c3b3d76e 100644 --- a/README.rst +++ b/README.rst @@ -450,5 +450,5 @@ file in the top distribution directory for the full license text. .. |build-status| image:: https://travis-ci.org/celery/celery.svg?branch=master :target: https://travis-ci.org/celery/celery -.. |coverage-status| image:: https://coveralls.io/repos/celery/celery/badge.svg - :target: https://coveralls.io/r/celery/celery +.. |coverage-status| image:: https://codecov.io/gh/celery/celery/badge.svg + :target: https://codecov.io/gh/celery/celery diff --git a/requirements/test-ci.txt b/requirements/test-ci.txt index 8385252ae..52789ebe7 100644 --- a/requirements/test-ci.txt +++ b/requirements/test-ci.txt @@ -1,5 +1,5 @@ coverage>=3.0 -coveralls +codecov redis #riak >=2.0 #pymongo From 07510bd2bb774d0e4e0b4097108cb4eac2e032a6 Mon Sep 17 00:00:00 2001 From: Steve Peak Date: Fri, 21 Aug 2015 14:55:04 -0400 Subject: [PATCH 0582/1103] call coveage xml because its in tox env --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 7695827dd..b1d246e0c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ install: script: - tox -v -- -v after_success: + - .tox/$TRAVIS_PYTHON_VERSION/bin/coverage xml - .tox/$TRAVIS_PYTHON_VERSION/bin/codecov -e TOXENV notifications: irc: From 76702e828d456d8a268209c934d9976003fca7f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 10:44:15 +0200 Subject: [PATCH 0583/1103] files in place for DataStax cassandra driver --- celery/backends/new_cassandra.py | 173 ++++++++++++++++++ celery/tests/backends/test_new_cassandra.py | 0 docs/includes/installation.txt | 5 +- .../celery.backends.new_cassandra.rst | 11 ++ requirements/extras/new_cassandra.txt | 1 + setup.py | 1 + 6 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 celery/backends/new_cassandra.py create mode 100644 celery/tests/backends/test_new_cassandra.py create mode 100644 docs/internals/reference/celery.backends.new_cassandra.rst create mode 100644 requirements/extras/new_cassandra.txt diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py new file mode 100644 index 000000000..67c388902 --- /dev/null +++ b/celery/backends/new_cassandra.py @@ -0,0 +1,173 @@ +# -* coding: utf-8 -*- +""" + celery.backends.cassandra + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Apache Cassandra result store backend. + +""" +from __future__ import absolute_import + +try: # pragma: no cover + import cassandra +except ImportError: # pragma: no cover + cassandra = None # noqa + +import time + +from celery import states +from celery.exceptions import ImproperlyConfigured +from celery.five import monotonic +from celery.utils.log import get_logger + +from .base import BaseBackend + +__all__ = ['NewCassandraBackend'] + +logger = get_logger(__name__) + + +class NewCassandraBackend(BaseBackend): + """New Cassandra backend utilizing DataStax's driver + + .. attribute:: servers + + List of Cassandra servers with format: ``hostname:port`` or ``hostname`` + + :raises celery.exceptions.ImproperlyConfigured: if + module :mod:`pycassa` is not available. + + """ + servers = [] + keyspace = None + column_family = None + detailed_mode = False + _retry_timeout = 300 + _retry_wait = 3 + supports_autoexpire = True + + def __init__(self, servers=None, keyspace=None, column_family=None, + cassandra_options=None, detailed_mode=False, port=9042, **kwargs): + """Initialize Cassandra backend. + + Raises :class:`celery.exceptions.ImproperlyConfigured` if + the :setting:`CASSANDRA_SERVERS` setting is not set. + + """ + super(NewCassandraBackend, self).__init__(**kwargs) + + if not cassandra: + raise ImproperlyConfigured( + 'You need to install the cassandra library to use the ' + 'Cassandra backend. See https://github.com/datastax/python-driver') + + conf = self.app.conf + self.servers = (servers or + conf.get('CASSANDRA_SERVERS') or + self.servers) + self.port = (port or + conf.get('CASSANDRA_PORT')) + self.keyspace = (keyspace or + conf.get('CASSANDRA_KEYSPACE') or + self.keyspace) + self.column_family = (column_family or + conf.get('CASSANDRA_COLUMN_FAMILY') or + self.column_family) + self.cassandra_options = dict(conf.get('CASSANDRA_OPTIONS') or {}, + **cassandra_options or {}) + self.detailed_mode = (detailed_mode or + conf.get('CASSANDRA_DETAILED_MODE') or + self.detailed_mode) + read_cons = conf.get('CASSANDRA_READ_CONSISTENCY') or 'LOCAL_QUORUM' + write_cons = conf.get('CASSANDRA_WRITE_CONSISTENCY') or 'LOCAL_QUORUM' + try: + self.read_consistency = getattr(cassandra.ConsistencyLevel, + read_cons) + except AttributeError: + self.read_consistency = cassandra.ConsistencyLevel.LOCAL_QUORUM + try: + self.write_consistency = getattr(cassandra.ConsistencyLevel, + write_cons) + except AttributeError: + self.write_consistency = cassandra.ConsistencyLevel.LOCAL_QUORUM + + if not self.servers or not self.keyspace or not self.column_family: + raise ImproperlyConfigured( + 'Cassandra backend not configured.') + + self._connection = None + + def _get_connection(self): + if self._connection is None: + self._connection = cassandra.Cluster(self.servers, port=self.port) + + def _retry_on_error(self, fun, *args, **kwargs): + ts = monotonic() + self._retry_timeout + while 1: + try: + return fun(*args, **kwargs) + except (cassandra.Unavailable, + cassandra.Timeout, + cassandra.InvalidRequest) as exc: + if monotonic() > ts: + raise + logger.warning('Cassandra error: %r. Retrying...', exc) + time.sleep(self._retry_wait) + + def _store_result(self, task_id, result, status, + traceback=None, request=None, **kwargs): + """Store return value and status of an executed task.""" + + def _do_store(): + self._get_connection() + date_done = self.app.now() + + + + meta = {'status': status, + 'date_done': date_done.strftime('%Y-%m-%dT%H:%M:%SZ'), + 'traceback': self.encode(traceback), + 'result': self.encode(result), + 'children': self.encode( + self.current_task_children(request), + )} + if self.detailed_mode: + cf.insert( + task_id, {date_done: self.encode(meta)}, ttl=self.expires, + ) + else: + cf.insert(task_id, meta, ttl=self.expires) + + return self._retry_on_error(_do_store) + + def _get_task_meta_for(self, task_id): + """Get task metadata for a task by id.""" + + def _do_get(): + cf = self._get_column_family() + try: + if self.detailed_mode: + row = cf.get(task_id, column_reversed=True, column_count=1) + return self.decode(list(row.values())[0]) + else: + obj = cf.get(task_id) + return self.meta_from_decoded({ + 'task_id': task_id, + 'status': obj['status'], + 'result': self.decode(obj['result']), + 'date_done': obj['date_done'], + 'traceback': self.decode(obj['traceback']), + 'children': self.decode(obj['children']), + }) + except (KeyError, pycassa.NotFoundException): + return {'status': states.PENDING, 'result': None} + + return self._retry_on_error(_do_get) + + def __reduce__(self, args=(), kwargs={}): + kwargs.update( + dict(servers=self.servers, + keyspace=self.keyspace, + column_family=self.column_family, + cassandra_options=self.cassandra_options)) + return super(NewCassandraBackend, self).__reduce__(args, kwargs) diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py new file mode 100644 index 000000000..e69de29bb diff --git a/docs/includes/installation.txt b/docs/includes/installation.txt index 2ab46ab35..18c2ab9b6 100644 --- a/docs/includes/installation.txt +++ b/docs/includes/installation.txt @@ -78,7 +78,10 @@ Transports and Backends for using memcached as a result backend. :celery[cassandra]: - for using Apache Cassandra as a result backend. + for using Apache Cassandra as a result backend with pycassa driver. + +:celery[new_cassandra]: + for using Apache Cassandra as a result backend with DataStax driver. :celery[couchdb]: for using CouchDB as a message transport (*experimental*). diff --git a/docs/internals/reference/celery.backends.new_cassandra.rst b/docs/internals/reference/celery.backends.new_cassandra.rst new file mode 100644 index 000000000..e7696fa62 --- /dev/null +++ b/docs/internals/reference/celery.backends.new_cassandra.rst @@ -0,0 +1,11 @@ +================================================ + celery.backends.new_cassandra +================================================ + +.. contents:: + :local: +.. currentmodule:: celery.backends.new_cassandra + +.. automodule:: celery.backends.new_cassandra + :members: + :undoc-members: diff --git a/requirements/extras/new_cassandra.txt b/requirements/extras/new_cassandra.txt new file mode 100644 index 000000000..a94062dad --- /dev/null +++ b/requirements/extras/new_cassandra.txt @@ -0,0 +1 @@ +cassandra-driver \ No newline at end of file diff --git a/setup.py b/setup.py index 136318076..01cc1c427 100644 --- a/setup.py +++ b/setup.py @@ -160,6 +160,7 @@ def reqs(*f): 'eventlet', 'gevent', 'msgpack', 'yaml', 'redis', 'mongodb', 'sqs', 'couchdb', 'riak', 'beanstalk', 'zookeeper', 'zeromq', 'sqlalchemy', 'librabbitmq', 'pyro', 'slmq', + 'new_cassandra', } extras_require = {x: extras(x + '.txt') for x in features} extra['extras_require'] = extras_require From 3989c3e3940369322381a813c6156fc9ebcdf27c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 11:23:14 +0200 Subject: [PATCH 0584/1103] needs preliminary tests --- celery/backends/new_cassandra.py | 82 +++++++++++++++++++------------- docs/configuration.rst | 22 +++++++++ 2 files changed, 72 insertions(+), 32 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index 67c388902..ec272fa55 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -96,10 +96,41 @@ def __init__(self, servers=None, keyspace=None, column_family=None, 'Cassandra backend not configured.') self._connection = None + self._session = None def _get_connection(self): if self._connection is None: self._connection = cassandra.Cluster(self.servers, port=self.port) + self._session = self._connection.connect(self.keyspace) + + self._write_stmt = self._session.prepare('''INSERT INTO '''+ + self.column_family+''' (task_id,status, result,date_done,''' + '''traceback, children) VALUES (?, ?, ?, ?, ?, ?) ''' + '''USING TTL '''+str(self.expires), + consistency_level=self.write_consistency) + + self._make_stmt = self._session.prepare( + '''CREATE TABLE '''+self.column_family+''' ( + task_id text, + status text, + result text, + date_done timestamp, + traceback text, + children text, + PRIMARY KEY ((task_id), date_done) + ) WITH CLUSTERING ORDER BY (date_done DESC) + WITH default_time_to_live = '''+str(self.expires)+';') + + self._read_stmt = self._session.prepare( + '''SELECT task_id, status, result, date_done, traceback, children + FROM '''+self.column_family+''' + WHERE task_id=? LIMIT 1''', + consistency_level=self.read_consistency) + + try: + self._session.execute(self._make_stmt) + except cassandra.AlreadyExists: + pass def _retry_on_error(self, fun, *args, **kwargs): ts = monotonic() + self._retry_timeout @@ -122,46 +153,33 @@ def _do_store(): self._get_connection() date_done = self.app.now() - - - meta = {'status': status, - 'date_done': date_done.strftime('%Y-%m-%dT%H:%M:%SZ'), - 'traceback': self.encode(traceback), - 'result': self.encode(result), - 'children': self.encode( - self.current_task_children(request), - )} - if self.detailed_mode: - cf.insert( - task_id, {date_done: self.encode(meta)}, ttl=self.expires, - ) - else: - cf.insert(task_id, meta, ttl=self.expires) - + self._session.execute(self._write_stmt, ( + task_id, status, result, + self.app.now().strftime('%Y-%m-%dT%H:%M:%SZ'), + traceback, self.encode(self.current_task_children(request)) + )) return self._retry_on_error(_do_store) def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" def _do_get(): - cf = self._get_column_family() - try: - if self.detailed_mode: - row = cf.get(task_id, column_reversed=True, column_count=1) - return self.decode(list(row.values())[0]) - else: - obj = cf.get(task_id) - return self.meta_from_decoded({ - 'task_id': task_id, - 'status': obj['status'], - 'result': self.decode(obj['result']), - 'date_done': obj['date_done'], - 'traceback': self.decode(obj['traceback']), - 'children': self.decode(obj['children']), - }) - except (KeyError, pycassa.NotFoundException): + + res = self._session.execute(self._read_stmt, (task_id, )) + if not res: return {'status': states.PENDING, 'result': None} + task_id, status, result, date_done, traceback, children = res[0] + + return self.meta_from_decoded({ + 'task_id': task_id, + 'status': status, + 'result': self.decode(result), + 'date_done': date_done, + 'traceback': self.decode(traceback), + 'children': self.decode(children), + }) + return self._retry_on_error(_do_get) def __reduce__(self, args=(), kwargs={}): diff --git a/docs/configuration.rst b/docs/configuration.rst index 49ca75877..21f6e99ff 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -213,6 +213,10 @@ Can be one of the following: Use `Cassandra`_ to store the results. See :ref:`conf-cassandra-result-backend`. +* new_cassandra + Use `Cassandra`_ to store the results, using other backend than _cassandra_. + See :ref:`conf-cassandra-result-backend`. + * ironcache Use `IronCache`_ to store the results. See :ref:`conf-ironcache-result-backend`. @@ -544,6 +548,16 @@ Cassandra backend settings $ pip install pycassa + If you are using new_cassandra, :mod:`cassandra-driver` is required instead: + https://pypi.python.org/pypi/cassandra-driver + + To install, use `pip` or `easy_install`: + + .. code-block:: bash + + $ pip install cassandra-driver + + This backend requires the following configuration directives to be set. .. setting:: CASSANDRA_SERVERS @@ -555,6 +569,10 @@ List of ``host:port`` Cassandra servers. e.g.:: CASSANDRA_SERVERS = ['localhost:9160'] +Omit the ``port`` part when using new_cassandra. e.g.:: + + CASSANDRA_SERVERS = ['localhost'] + .. setting:: CASSANDRA_KEYSPACE CASSANDRA_KEYSPACE @@ -601,6 +619,8 @@ use the ``TimeUUID`` type as a comparator:: create column family task_results with comparator = TimeUUIDType; +new_cassandra uses detailed mode by default, and that cannot be disabled. + CASSANDRA_OPTIONS ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -608,6 +628,8 @@ Options to be passed to the `pycassa connection pool`_ (optional). .. _`pycassa connection pool`: http://pycassa.github.com/pycassa/api/pycassa/pool.html +Not used in new_cassandra + Example configuration ~~~~~~~~~~~~~~~~~~~~~ From 8e7d8d4f639f4bf1baacda8396b937008820c235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 15:41:18 +0200 Subject: [PATCH 0585/1103] works for me --- celery/backends/__init__.py | 1 + celery/backends/new_cassandra.py | 174 ++++++++++++++----------------- docs/configuration.rst | 106 ++++++++++++++++--- 3 files changed, 173 insertions(+), 108 deletions(-) diff --git a/celery/backends/__init__.py b/celery/backends/__init__.py index eec585227..afff815c2 100644 --- a/celery/backends/__init__.py +++ b/celery/backends/__init__.py @@ -30,6 +30,7 @@ 'db': 'celery.backends.database:DatabaseBackend', 'database': 'celery.backends.database:DatabaseBackend', 'cassandra': 'celery.backends.cassandra:CassandraBackend', + 'new_cassandra': 'celery.backends.new_cassandra:NewCassandraBackend', 'couchbase': 'celery.backends.couchbase:CouchBaseBackend', 'couchdb': 'celery.backends.couchdb:CouchDBBackend', 'riak': 'celery.backends.riak:RiakBackend', diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index ec272fa55..fe764c9d2 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -1,9 +1,9 @@ # -* coding: utf-8 -*- """ - celery.backends.cassandra + celery.backends.new_cassandra ~~~~~~~~~~~~~~~~~~~~~~~~~ - Apache Cassandra result store backend. + Apache Cassandra result store backend using DataStax driver """ from __future__ import absolute_import @@ -13,11 +13,8 @@ except ImportError: # pragma: no cover cassandra = None # noqa -import time - from celery import states from celery.exceptions import ImproperlyConfigured -from celery.five import monotonic from celery.utils.log import get_logger from .base import BaseBackend @@ -32,22 +29,19 @@ class NewCassandraBackend(BaseBackend): .. attribute:: servers - List of Cassandra servers with format: ``hostname:port`` or ``hostname`` + List of Cassandra servers with format: ``hostname`` :raises celery.exceptions.ImproperlyConfigured: if - module :mod:`pycassa` is not available. + module :mod:`cassandra` is not available. """ servers = [] keyspace = None - column_family = None - detailed_mode = False - _retry_timeout = 300 - _retry_wait = 3 + table = None supports_autoexpire = True - def __init__(self, servers=None, keyspace=None, column_family=None, - cassandra_options=None, detailed_mode=False, port=9042, **kwargs): + def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, + port=9042, **kwargs): """Initialize Cassandra backend. Raises :class:`celery.exceptions.ImproperlyConfigured` if @@ -70,14 +64,16 @@ def __init__(self, servers=None, keyspace=None, column_family=None, self.keyspace = (keyspace or conf.get('CASSANDRA_KEYSPACE') or self.keyspace) - self.column_family = (column_family or - conf.get('CASSANDRA_COLUMN_FAMILY') or - self.column_family) - self.cassandra_options = dict(conf.get('CASSANDRA_OPTIONS') or {}, - **cassandra_options or {}) - self.detailed_mode = (detailed_mode or - conf.get('CASSANDRA_DETAILED_MODE') or - self.detailed_mode) + self.table = (table or + conf.get('CASSANDRA_TABLE') or + self.table) + expires = (entry_ttl or conf.get('CASSANDRA_ENTRY_TTL', None)) + + if expires is not None: + self.cqlexpires = ' USING TTL %s' % (expires, ) + else: + self.cqlexpires = '' + read_cons = conf.get('CASSANDRA_READ_CONSISTENCY') or 'LOCAL_QUORUM' write_cons = conf.get('CASSANDRA_WRITE_CONSISTENCY') or 'LOCAL_QUORUM' try: @@ -91,101 +87,91 @@ def __init__(self, servers=None, keyspace=None, column_family=None, except AttributeError: self.write_consistency = cassandra.ConsistencyLevel.LOCAL_QUORUM - if not self.servers or not self.keyspace or not self.column_family: + if not self.servers or not self.keyspace or not self.table: raise ImproperlyConfigured( 'Cassandra backend not configured.') self._connection = None self._session = None - - def _get_connection(self): + self._write_stmt = None + self._read_stmt = None + + def process_cleanup(self): + if self._connection is not None: + self._session.shutdown() + self._connection = None + self._session = None + + def _get_connection(self, write=False): + # only writers can create the table to get rid of two processes + # creating table at same time and Cassandra choking on that if self._connection is None: - self._connection = cassandra.Cluster(self.servers, port=self.port) + self._connection = cassandra.cluster.Cluster(self.servers, + port=self.port) self._session = self._connection.connect(self.keyspace) - self._write_stmt = self._session.prepare('''INSERT INTO '''+ - self.column_family+''' (task_id,status, result,date_done,''' - '''traceback, children) VALUES (?, ?, ?, ?, ?, ?) ''' - '''USING TTL '''+str(self.expires), - consistency_level=self.write_consistency) - - self._make_stmt = self._session.prepare( - '''CREATE TABLE '''+self.column_family+''' ( - task_id text, - status text, - result text, - date_done timestamp, - traceback text, - children text, - PRIMARY KEY ((task_id), date_done) - ) WITH CLUSTERING ORDER BY (date_done DESC) - WITH default_time_to_live = '''+str(self.expires)+';') - - self._read_stmt = self._session.prepare( - '''SELECT task_id, status, result, date_done, traceback, children - FROM '''+self.column_family+''' - WHERE task_id=? LIMIT 1''', - consistency_level=self.read_consistency) - - try: - self._session.execute(self._make_stmt) - except cassandra.AlreadyExists: - pass - - def _retry_on_error(self, fun, *args, **kwargs): - ts = monotonic() + self._retry_timeout - while 1: - try: - return fun(*args, **kwargs) - except (cassandra.Unavailable, - cassandra.Timeout, - cassandra.InvalidRequest) as exc: - if monotonic() > ts: - raise - logger.warning('Cassandra error: %r. Retrying...', exc) - time.sleep(self._retry_wait) + self._write_stmt = cassandra.query.SimpleStatement( + 'INSERT INTO '+self.table+' (task_id, status, result,''' + ''' date_done, traceback, children) VALUES''' + ' (%s, %s, %s, %s, %s, %s) '+self.cqlexpires+';') + self._write_stmt.consistency_level = self.write_consistency + + self._read_stmt = cassandra.query.SimpleStatement( + '''SELECT status, result, date_done, traceback, children + FROM '''+self.table+''' + WHERE task_id=%s''') + self._read_stmt.consistency_level = self.read_consistency + + if write: + self._make_stmt = cassandra.query.SimpleStatement( + '''CREATE TABLE '''+self.table+''' ( + task_id text, + status text, + result blob, + date_done text, + traceback blob, + children blob, + PRIMARY KEY (task_id) + );''') + self._make_stmt.consistency_level = self.write_consistency + try: + self._session.execute(self._make_stmt) + except cassandra.AlreadyExists: + pass def _store_result(self, task_id, result, status, traceback=None, request=None, **kwargs): """Store return value and status of an executed task.""" + self._get_connection(write=True) - def _do_store(): - self._get_connection() - date_done = self.app.now() - - self._session.execute(self._write_stmt, ( - task_id, status, result, - self.app.now().strftime('%Y-%m-%dT%H:%M:%SZ'), - traceback, self.encode(self.current_task_children(request)) - )) - return self._retry_on_error(_do_store) + self._session.execute(self._write_stmt, ( + task_id, status, buffer(self.encode(result)), + self.app.now().strftime('%Y-%m-%dT%H:%M:%SZ'), + buffer(self.encode(traceback)), buffer(self.encode(self.current_task_children(request))) + )) def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" + self._get_connection() - def _do_get(): - - res = self._session.execute(self._read_stmt, (task_id, )) - if not res: - return {'status': states.PENDING, 'result': None} - - task_id, status, result, date_done, traceback, children = res[0] + res = self._session.execute(self._read_stmt, (task_id, )) + if not res: + return {'status': states.PENDING, 'result': None} - return self.meta_from_decoded({ - 'task_id': task_id, - 'status': status, - 'result': self.decode(result), - 'date_done': date_done, - 'traceback': self.decode(traceback), - 'children': self.decode(children), - }) + status, result, date_done, traceback, children = res[0] - return self._retry_on_error(_do_get) + return self.meta_from_decoded({ + 'task_id': task_id, + 'status': str(status), + 'result': self.decode(str(result)), + 'date_done': date_done, + 'traceback': self.decode(str(traceback)), + 'children': self.decode(str(children)), + }) def __reduce__(self, args=(), kwargs={}): kwargs.update( dict(servers=self.servers, keyspace=self.keyspace, - column_family=self.column_family, - cassandra_options=self.cassandra_options)) + table=self.table)) return super(NewCassandraBackend, self).__reduce__(args, kwargs) diff --git a/docs/configuration.rst b/docs/configuration.rst index 21f6e99ff..735234148 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -214,8 +214,8 @@ Can be one of the following: See :ref:`conf-cassandra-result-backend`. * new_cassandra - Use `Cassandra`_ to store the results, using other backend than _cassandra_. - See :ref:`conf-cassandra-result-backend`. + Use `new_cassandra`_ to store the results, using newer database driver than _cassandra_. + See :ref:`conf-new_cassandra-result-backend`. * ironcache Use `IronCache`_ to store the results. @@ -532,30 +532,110 @@ Example configuration 'taskmeta_collection': 'my_taskmeta_collection', } -.. _conf-cassandra-result-backend: +.. _conf-new_cassandra-result-backend: -Cassandra backend settings + +new_cassandra backend settings -------------------------- .. note:: - The Cassandra backend requires the :mod:`pycassa` library: - http://pypi.python.org/pypi/pycassa/ + This Cassandra backend driver requires :mod:`cassandra-driver`. + https://pypi.python.org/pypi/cassandra-driver - To install the pycassa package use `pip` or `easy_install`: + To install, use `pip` or `easy_install`: .. code-block:: bash - $ pip install pycassa + $ pip install cassandra-driver - If you are using new_cassandra, :mod:`cassandra-driver` is required instead: - https://pypi.python.org/pypi/cassandra-driver +This backend requires the following configuration directives to be set. + +.. setting:: CASSANDRA_SERVERS + +CASSANDRA_SERVERS +~~~~~~~~~~~~~~~~~ + +List of ``host`` Cassandra servers. e.g.:: + + CASSANDRA_SERVERS = ['localhost'] - To install, use `pip` or `easy_install`: + +.. setting:: CASSANDRA_PORT + +CASSANDRA_PORT +~~~~~~~~~~~~~~ + +Port to contact the Cassandra servers on. Default is 9042. + +.. setting:: CASSANDRA_KEYSPACE + +CASSANDRA_KEYSPACE +~~~~~~~~~~~~~~~~~~ + +The keyspace in which to store the results. e.g.:: + + CASSANDRA_KEYSPACE = 'tasks_keyspace' + +.. setting:: CASSANDRA_COLUMN_FAMILY + +CASSANDRA_TABLE +~~~~~~~~~~~~~~~~~~~~~~~ + +The table (column family) in which to store the results. e.g.:: + + CASSANDRA_TABLE = 'tasks' + +.. setting:: CASSANDRA_READ_CONSISTENCY + +CASSANDRA_READ_CONSISTENCY +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The read consistency used. Values can be ``ONE``, ``TWO``, ``THREE``, ``QUORUM``, ``ALL``, +``LOCAL_QUORUM``, ``EACH_QUORUM``, ``LOCAL_ONE``. + +.. setting:: CASSANDRA_WRITE_CONSISTENCY + +CASSANDRA_WRITE_CONSISTENCY +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The write consistency used. Values can be ``ONE``, ``TWO``, ``THREE``, ``QUORUM``, ``ALL``, +``LOCAL_QUORUM``, ``EACH_QUORUM``, ``LOCAL_ONE``. + +.. setting:: CASSANDRA_ENTRY_TTL + +CASSANDRA_ENTRY_TTL +~~~~~~~~~~~~~~~~~~~ + +Time-to-live for status entries. They will expire and be removed after that many seconds +after adding. Default (None) means they will never expire. + +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + CASSANDRA_SERVERS = ['localhost'] + CASSANDRA_KEYSPACE = 'celery' + CASSANDRA_COLUMN_FAMILY = 'task_results' + CASSANDRA_READ_CONSISTENCY = 'ONE' + CASSANDRA_WRITE_CONSISTENCY = 'ONE' + CASSANDRA_ENTRY_TTL = 86400 + + +Cassandra backend settings +-------------------------- + +.. note:: + + The Cassandra backend requires the :mod:`pycassa` library: + http://pypi.python.org/pypi/pycassa/ + + To install the pycassa package use `pip` or `easy_install`: .. code-block:: bash - $ pip install cassandra-driver + $ pip install pycassa This backend requires the following configuration directives to be set. @@ -628,8 +708,6 @@ Options to be passed to the `pycassa connection pool`_ (optional). .. _`pycassa connection pool`: http://pycassa.github.com/pycassa/api/pycassa/pool.html -Not used in new_cassandra - Example configuration ~~~~~~~~~~~~~~~~~~~~~ From dfcfa1f256eef5349425a60ea55cf678e9e02d59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 15:44:09 +0200 Subject: [PATCH 0586/1103] better no tests than fake tests --- celery/tests/backends/test_new_cassandra.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 celery/tests/backends/test_new_cassandra.py diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py deleted file mode 100644 index e69de29bb..000000000 From 669e42e6817a26aee1cd269af869d727d73d5351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 18:04:22 +0200 Subject: [PATCH 0587/1103] cassandra deprecated --- celery/backends/cassandra.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index aa8e688cc..a427688f9 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -17,6 +17,7 @@ import socket import time +import warnings from celery import states from celery.exceptions import ImproperlyConfigured @@ -98,6 +99,9 @@ def __init__(self, servers=None, keyspace=None, column_family=None, self._column_family = None + warnings.warn("cassandra backend is deprecated. Use new_cassandra instead.", + DeprecationWarning) + def _retry_on_error(self, fun, *args, **kwargs): ts = monotonic() + self._retry_timeout while 1: From 7023694a6df89d5d5c7a60a45fc79cde9562b926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 18:38:29 +0200 Subject: [PATCH 0588/1103] tests added --- celery/backends/new_cassandra.py | 26 +++-- celery/tests/backends/test_new_cassandra.py | 102 ++++++++++++++++++++ 2 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 celery/tests/backends/test_new_cassandra.py diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index fe764c9d2..e83031ec5 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -25,7 +25,7 @@ class NewCassandraBackend(BaseBackend): - """New Cassandra backend utilizing DataStax's driver + """New Cassandra backend utilizing DataStax driver .. attribute:: servers @@ -38,7 +38,7 @@ class NewCassandraBackend(BaseBackend): servers = [] keyspace = None table = None - supports_autoexpire = True + supports_autoexpire = True # autoexpire supported via entry_ttl def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, port=9042, **kwargs): @@ -103,8 +103,11 @@ def process_cleanup(self): self._session = None def _get_connection(self, write=False): - # only writers can create the table to get rid of two processes - # creating table at same time and Cassandra choking on that + """ + Prepare the connection for action + + :param write: bool - are we a writer? + """ if self._connection is None: self._connection = cassandra.cluster.Cluster(self.servers, port=self.port) @@ -123,6 +126,14 @@ def _get_connection(self, write=False): self._read_stmt.consistency_level = self.read_consistency if write: + # Only possible writers "workers" are allowed to issue + # CREATE TABLE. This is to prevent conflicting situations + # where both task-creator and task-executor would issue it + # at the same time. + + # Anyway, if you are doing anything critical, you should + # have probably created this table in advance, in which case + # this query will be a no-op (instant fail with AlreadyExists) self._make_stmt = cassandra.query.SimpleStatement( '''CREATE TABLE '''+self.table+''' ( task_id text, @@ -145,9 +156,12 @@ def _store_result(self, task_id, result, status, self._get_connection(write=True) self._session.execute(self._write_stmt, ( - task_id, status, buffer(self.encode(result)), + task_id, + status, + buffer(self.encode(result)), self.app.now().strftime('%Y-%m-%dT%H:%M:%SZ'), - buffer(self.encode(traceback)), buffer(self.encode(self.current_task_children(request))) + buffer(self.encode(traceback)), + buffer(self.encode(self.current_task_children(request))) )) def _get_task_meta_for(self, task_id): diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py new file mode 100644 index 000000000..01ecea0e9 --- /dev/null +++ b/celery/tests/backends/test_new_cassandra.py @@ -0,0 +1,102 @@ +from __future__ import absolute_import + +from pickle import loads, dumps + +from celery import states +from celery.exceptions import ImproperlyConfigured +from celery.tests.case import ( + AppCase, Mock, mock_module, depends_on_current_app, MagicMock +) + +class Object(object): + pass + +class test_NewCassandraBackend(AppCase): + + def setup(self): + self.app.conf.update( + CASSANDRA_SERVERS=['example.com'], + CASSANDRA_KEYSPACE='celery', + CASSANDRA_TABLE='task_results', + ) + + def test_init_no_cassandra(self): + """ + Tests behaviour when no python-driver is installed. + new_cassandra should raise ImproperlyConfigured + """ + with mock_module('cassandra'): + from celery.backends import new_cassandra as mod + prev, mod.cassandra = mod.cassandra, None + try: + with self.assertRaises(ImproperlyConfigured): + mod.NewCassandraBackend(app=self.app) + finally: + mod.cassandra = prev + + def test_init_with_and_without_LOCAL_QUROM(self): + with mock_module('cassandra'): + from celery.backends import new_cassandra as mod + mod.cassandra = Mock() + cons = mod.cassandra.ConsistencyLevel = Object() + cons.LOCAL_QUORUM = 'foo' + + self.app.conf.CASSANDRA_READ_CONSISTENCY = 'LOCAL_FOO' + self.app.conf.CASSANDRA_WRITE_CONSISTENCY = 'LOCAL_FOO' + + mod.NewCassandraBackend(app=self.app) + cons.LOCAL_FOO = 'bar' + mod.NewCassandraBackend(app=self.app) + + # no servers raises ImproperlyConfigured + with self.assertRaises(ImproperlyConfigured): + self.app.conf.CASSANDRA_SERVERS = None + mod.NewCassandraBackend( + app=self.app, keyspace='b', column_family='c', + ) + + @depends_on_current_app + def test_reduce(self): + with mock_module('cassandra'): + from celery.backends.new_cassandra import NewCassandraBackend + self.assertTrue(loads(dumps(NewCassandraBackend(app=self.app)))) + + def test_get_task_meta_for(self): + with mock_module('cassandra'): + from celery.backends import new_cassandra as mod + mod.cassandra = Mock() + x = mod.NewCassandraBackend(app=self.app) + x._connection = True + session = x._session = Mock() + execute = session.execute = Mock() + execute.return_value = [ + [states.SUCCESS, '1', 'date', '', None] + ] + x.decode = Mock() + meta = x._get_task_meta_for('task_id') + self.assertEqual(meta['status'], states.SUCCESS) + + x._session.execute.return_value = [] + meta = x._get_task_meta_for('task_id') + self.assertEqual(meta['status'], states.PENDING) + + + def test_store_result(self): + with mock_module('cassandra'): + from celery.backends import new_cassandra as mod + mod.cassandra = Mock() + + x = mod.NewCassandraBackend(app=self.app) + x._connection = True + session = x._session = Mock() + execute = session.execute = Mock() + x._store_result('task_id', 'result', states.SUCCESS) + + def test_process_cleanup(self): + with mock_module('cassandra'): + from celery.backends import new_cassandra as mod + x = mod.NewCassandraBackend(app=self.app) + x.process_cleanup() + + self.assertIsNone(x._connection) + self.assertIsNone(x._session) From 22804bb22237b7cc9913c923c6e2d43c53077723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 18:43:52 +0200 Subject: [PATCH 0589/1103] PEP8 --- celery/backends/new_cassandra.py | 12 ++++++------ celery/tests/backends/test_new_cassandra.py | 7 ++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index e83031ec5..974b9e95e 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -175,12 +175,12 @@ def _get_task_meta_for(self, task_id): status, result, date_done, traceback, children = res[0] return self.meta_from_decoded({ - 'task_id': task_id, - 'status': str(status), - 'result': self.decode(str(result)), - 'date_done': date_done, - 'traceback': self.decode(str(traceback)), - 'children': self.decode(str(children)), + 'task_id': task_id, + 'status': str(status), + 'result': self.decode(str(result)), + 'date_done': date_done, + 'traceback': self.decode(str(traceback)), + 'children': self.decode(str(children)), }) def __reduce__(self, args=(), kwargs={}): diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 01ecea0e9..1fbc18909 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -5,12 +5,14 @@ from celery import states from celery.exceptions import ImproperlyConfigured from celery.tests.case import ( - AppCase, Mock, mock_module, depends_on_current_app, MagicMock + AppCase, Mock, mock_module, depends_on_current_app ) + class Object(object): pass + class test_NewCassandraBackend(AppCase): def setup(self): @@ -80,7 +82,6 @@ def test_get_task_meta_for(self): meta = x._get_task_meta_for('task_id') self.assertEqual(meta['status'], states.PENDING) - def test_store_result(self): with mock_module('cassandra'): from celery.backends import new_cassandra as mod @@ -89,7 +90,7 @@ def test_store_result(self): x = mod.NewCassandraBackend(app=self.app) x._connection = True session = x._session = Mock() - execute = session.execute = Mock() + session.execute = Mock() x._store_result('task_id', 'result', states.SUCCESS) def test_process_cleanup(self): From be76bea992cc10707539d808eaa17b26ba001578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 18:58:39 +0200 Subject: [PATCH 0590/1103] "detailed mode" is the default and only --- celery/backends/new_cassandra.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index 974b9e95e..3415feb34 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -122,7 +122,7 @@ def _get_connection(self, write=False): self._read_stmt = cassandra.query.SimpleStatement( '''SELECT status, result, date_done, traceback, children FROM '''+self.table+''' - WHERE task_id=%s''') + WHERE task_id=%s LIMIT 1''') self._read_stmt.consistency_level = self.read_consistency if write: @@ -139,11 +139,12 @@ def _get_connection(self, write=False): task_id text, status text, result blob, - date_done text, + date_done timestamp, traceback blob, children blob, - PRIMARY KEY (task_id) - );''') + PRIMARY KEY ((task_id), date_done) + ) + WITH CLUSTERING ORDER BY (date_done DESC);''') self._make_stmt.consistency_level = self.write_consistency try: self._session.execute(self._make_stmt) @@ -159,7 +160,7 @@ def _store_result(self, task_id, result, status, task_id, status, buffer(self.encode(result)), - self.app.now().strftime('%Y-%m-%dT%H:%M:%SZ'), + self.app.now(), buffer(self.encode(traceback)), buffer(self.encode(self.current_task_children(request))) )) @@ -178,7 +179,7 @@ def _get_task_meta_for(self, task_id): 'task_id': task_id, 'status': str(status), 'result': self.decode(str(result)), - 'date_done': date_done, + 'date_done': date_done.strftime('%Y-%m-%dT%H:%M:%SZ'), 'traceback': self.decode(str(traceback)), 'children': self.decode(str(children)), }) From c5f883d6446b8206062011584192b2d41f95dda5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 19:11:05 +0200 Subject: [PATCH 0591/1103] fix for unit test --- celery/tests/backends/test_new_cassandra.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 1fbc18909..096718bf4 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -1,7 +1,7 @@ from __future__ import absolute_import from pickle import loads, dumps - +from datetime import datetime from celery import states from celery.exceptions import ImproperlyConfigured from celery.tests.case import ( @@ -72,7 +72,7 @@ def test_get_task_meta_for(self): session = x._session = Mock() execute = session.execute = Mock() execute.return_value = [ - [states.SUCCESS, '1', 'date', '', None] + [states.SUCCESS, '1', datetime.now(), '', None] ] x.decode = Mock() meta = x._get_task_meta_for('task_id') From be992be02b2c019bc2cd8e0f78cf190d7ad012e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 19:20:29 +0200 Subject: [PATCH 0592/1103] py3k (buffer, memoryview) --- celery/backends/new_cassandra.py | 6 +++++- celery/tests/backends/test_new_cassandra.py | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index 3415feb34..e6068ceee 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -8,6 +8,7 @@ """ from __future__ import absolute_import +import sys try: # pragma: no cover import cassandra except ImportError: # pragma: no cover @@ -16,7 +17,6 @@ from celery import states from celery.exceptions import ImproperlyConfigured from celery.utils.log import get_logger - from .base import BaseBackend __all__ = ['NewCassandraBackend'] @@ -156,6 +156,10 @@ def _store_result(self, task_id, result, status, """Store return value and status of an executed task.""" self._get_connection(write=True) + import sys + if sys.version_info > (3,): + buffer = memoryview + self._session.execute(self._write_stmt, ( task_id, status, diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 096718bf4..94cc0b3a7 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -8,7 +8,6 @@ AppCase, Mock, mock_module, depends_on_current_app ) - class Object(object): pass From d2e4c5fa0e9215d7e0f32529cca8ad27726db552 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 19:33:11 +0200 Subject: [PATCH 0593/1103] more confusion with binary types in Py3K --- celery/backends/new_cassandra.py | 21 ++++++++++++--------- celery/tests/backends/test_new_cassandra.py | 6 ++++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index e6068ceee..80cc20a80 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -9,6 +9,7 @@ from __future__ import absolute_import import sys +import six try: # pragma: no cover import cassandra except ImportError: # pragma: no cover @@ -157,16 +158,18 @@ def _store_result(self, task_id, result, status, self._get_connection(write=True) import sys - if sys.version_info > (3,): - buffer = memoryview + if six.PY3: + buf = lambda x: bytes(x, 'utf8') + else: + buf = buffer self._session.execute(self._write_stmt, ( task_id, status, - buffer(self.encode(result)), + buf(self.encode(result)), self.app.now(), - buffer(self.encode(traceback)), - buffer(self.encode(self.current_task_children(request))) + buf(self.encode(traceback)), + buf(self.encode(self.current_task_children(request))) )) def _get_task_meta_for(self, task_id): @@ -181,11 +184,11 @@ def _get_task_meta_for(self, task_id): return self.meta_from_decoded({ 'task_id': task_id, - 'status': str(status), - 'result': self.decode(str(result)), + 'status': status, + 'result': self.decode(result), 'date_done': date_done.strftime('%Y-%m-%dT%H:%M:%SZ'), - 'traceback': self.decode(str(traceback)), - 'children': self.decode(str(children)), + 'traceback': self.decode(traceback), + 'children': self.decode(children), }) def __reduce__(self, args=(), kwargs={}): diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 94cc0b3a7..6e8f58463 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -1,7 +1,8 @@ from __future__ import absolute_import - from pickle import loads, dumps from datetime import datetime + +import six from celery import states from celery.exceptions import ImproperlyConfigured from celery.tests.case import ( @@ -71,7 +72,8 @@ def test_get_task_meta_for(self): session = x._session = Mock() execute = session.execute = Mock() execute.return_value = [ - [states.SUCCESS, '1', datetime.now(), '', None] + [states.SUCCESS, '1', datetime.now(), six.binary_type(''), + six.binary_type('')] ] x.decode = Mock() meta = x._get_task_meta_for('task_id') From 1b478a05e47724d5451e06d7e209f9eb6598d1eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 19:36:01 +0200 Subject: [PATCH 0594/1103] ditto --- celery/tests/backends/test_new_cassandra.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 6e8f58463..ede4fb944 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -72,8 +72,7 @@ def test_get_task_meta_for(self): session = x._session = Mock() execute = session.execute = Mock() execute.return_value = [ - [states.SUCCESS, '1', datetime.now(), six.binary_type(''), - six.binary_type('')] + [states.SUCCESS, '1', datetime.now(), b'', b''] ] x.decode = Mock() meta = x._get_task_meta_for('task_id') From eb2e128f542d75de260781a63d3a2be878f9ee62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 20:04:40 +0200 Subject: [PATCH 0595/1103] mutable class level objects evicted --- celery/backends/new_cassandra.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index 80cc20a80..80f308c4c 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -36,9 +36,6 @@ class NewCassandraBackend(BaseBackend): module :mod:`cassandra` is not available. """ - servers = [] - keyspace = None - table = None supports_autoexpire = True # autoexpire supported via entry_ttl def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, @@ -58,16 +55,13 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, conf = self.app.conf self.servers = (servers or - conf.get('CASSANDRA_SERVERS') or - self.servers) + conf.get('CASSANDRA_SERVERS')) self.port = (port or conf.get('CASSANDRA_PORT')) self.keyspace = (keyspace or - conf.get('CASSANDRA_KEYSPACE') or - self.keyspace) + conf.get('CASSANDRA_KEYSPACE')) self.table = (table or - conf.get('CASSANDRA_TABLE') or - self.table) + conf.get('CASSANDRA_TABLE')) expires = (entry_ttl or conf.get('CASSANDRA_ENTRY_TTL', None)) if expires is not None: From 1ec4a5cfd8c04d560b73580760683e8187274943 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 20:19:47 +0200 Subject: [PATCH 0596/1103] code review fixes %s are here to stay - I'll need them later for Cassandra queries. I have no idea how to use celery.five to detect Python version. --- celery/backends/new_cassandra.py | 37 +++++++++------------ celery/tests/backends/test_new_cassandra.py | 1 - 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index 80f308c4c..ce7775461 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -9,7 +9,6 @@ from __future__ import absolute_import import sys -import six try: # pragma: no cover import cassandra except ImportError: # pragma: no cover @@ -55,13 +54,17 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, conf = self.app.conf self.servers = (servers or - conf.get('CASSANDRA_SERVERS')) + conf.get('CASSANDRA_SERVERS', None)) self.port = (port or - conf.get('CASSANDRA_PORT')) + conf.get('CASSANDRA_PORT', None)) self.keyspace = (keyspace or - conf.get('CASSANDRA_KEYSPACE')) + conf.get('CASSANDRA_KEYSPACE', None)) self.table = (table or - conf.get('CASSANDRA_TABLE')) + conf.get('CASSANDRA_TABLE', None)) + + if not self.servers or not self.keyspace or not self.table: + raise ImproperlyConfigured('Cassandra backend not configured.') + expires = (entry_ttl or conf.get('CASSANDRA_ENTRY_TTL', None)) if expires is not None: @@ -71,20 +74,11 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, read_cons = conf.get('CASSANDRA_READ_CONSISTENCY') or 'LOCAL_QUORUM' write_cons = conf.get('CASSANDRA_WRITE_CONSISTENCY') or 'LOCAL_QUORUM' - try: - self.read_consistency = getattr(cassandra.ConsistencyLevel, - read_cons) - except AttributeError: - self.read_consistency = cassandra.ConsistencyLevel.LOCAL_QUORUM - try: - self.write_consistency = getattr(cassandra.ConsistencyLevel, - write_cons) - except AttributeError: - self.write_consistency = cassandra.ConsistencyLevel.LOCAL_QUORUM - if not self.servers or not self.keyspace or not self.table: - raise ImproperlyConfigured( - 'Cassandra backend not configured.') + self.read_consistency = getattr(cassandra.ConsistencyLevel, + read_cons, cassandra.ConsistencyLevel.LOCAL_QUORUM) + self.write_consistency = getattr(cassandra.ConsistencyLevel, + write_cons, cassandra.ConsistencyLevel.LOCAL_QUORUM) self._connection = None self._session = None @@ -108,8 +102,10 @@ def _get_connection(self, write=False): port=self.port) self._session = self._connection.connect(self.keyspace) + # We are forced to do concatenation below, as formatting would + # blow up on superficial %s that will be processed by Cassandra self._write_stmt = cassandra.query.SimpleStatement( - 'INSERT INTO '+self.table+' (task_id, status, result,''' + 'INSERT INTO %s (task_id, status, result,''' ''' date_done, traceback, children) VALUES''' ' (%s, %s, %s, %s, %s, %s) '+self.cqlexpires+';') self._write_stmt.consistency_level = self.write_consistency @@ -151,8 +147,7 @@ def _store_result(self, task_id, result, status, """Store return value and status of an executed task.""" self._get_connection(write=True) - import sys - if six.PY3: + if sys.version_info >= (3,): buf = lambda x: bytes(x, 'utf8') else: buf = buffer diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index ede4fb944..17c0ace85 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -2,7 +2,6 @@ from pickle import loads, dumps from datetime import datetime -import six from celery import states from celery.exceptions import ImproperlyConfigured from celery.tests.case import ( From 2170b15ec87000ea63c176c5978ff8048f203059 Mon Sep 17 00:00:00 2001 From: Juan Rossi Date: Mon, 24 Aug 2015 19:24:58 -0300 Subject: [PATCH 0597/1103] Added headers arg to apply_async docs to fix #2750 --- CONTRIBUTORS.txt | 1 + celery/app/task.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 977cd22d5..ac5541ef4 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -189,3 +189,4 @@ James Pulec, 2015/04/19 Alexander Lebedev, 2015/04/25 Frantisek Holop, 2015/05/21 Feanil Patel, 2015/05/21 +Juan Rossi, 2015/08/10 diff --git a/celery/app/task.py b/celery/app/task.py index 920232529..f2fe11fae 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -437,13 +437,18 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, if an error occurs while executing the task. :keyword producer: :class:`kombu.Producer` instance to use. + :keyword add_to_parent: If set to True (default) and the task is applied while executing another task, then the result will be appended to the parent tasks ``request.children`` attribute. Trailing can also be disabled by default using the :attr:`trail` attribute + :keyword publisher: Deprecated alias to ``producer``. + :keyword headers: Message headers to be sent in the + task (a :class:`dict`) + :rtype :class:`celery.result.AsyncResult`: if :setting:`CELERY_ALWAYS_EAGER` is not set, otherwise :class:`celery.result.EagerResult`: From 6d4bc35d0003fa41d282bb6a7eb023300df22e1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Tue, 25 Aug 2015 08:51:14 +0200 Subject: [PATCH 0598/1103] overzealous code fix removed --- celery/backends/new_cassandra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index ce7775461..3c530f022 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -105,7 +105,7 @@ def _get_connection(self, write=False): # We are forced to do concatenation below, as formatting would # blow up on superficial %s that will be processed by Cassandra self._write_stmt = cassandra.query.SimpleStatement( - 'INSERT INTO %s (task_id, status, result,''' + 'INSERT INTO '+self.table+''' (task_id, status, result,''' ''' date_done, traceback, children) VALUES''' ' (%s, %s, %s, %s, %s, %s) '+self.cqlexpires+';') self._write_stmt.consistency_level = self.write_consistency From d4a48a480b036e0ddb9816336f10baf3e472f318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Tue, 25 Aug 2015 08:57:59 +0200 Subject: [PATCH 0599/1103] lol, CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 977cd22d5..f3a5fb9c8 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -189,3 +189,4 @@ James Pulec, 2015/04/19 Alexander Lebedev, 2015/04/25 Frantisek Holop, 2015/05/21 Feanil Patel, 2015/05/21 +Piotr Maślanka, 2015/08/24 From 89d01692c2f2749a5806b87d684f895649babda7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Tue, 25 Aug 2015 09:02:04 +0200 Subject: [PATCH 0600/1103] doc coherence --- docs/configuration.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 735234148..b6dd3bd4b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -637,7 +637,6 @@ Cassandra backend settings $ pip install pycassa - This backend requires the following configuration directives to be set. .. setting:: CASSANDRA_SERVERS @@ -649,10 +648,6 @@ List of ``host:port`` Cassandra servers. e.g.:: CASSANDRA_SERVERS = ['localhost:9160'] -Omit the ``port`` part when using new_cassandra. e.g.:: - - CASSANDRA_SERVERS = ['localhost'] - .. setting:: CASSANDRA_KEYSPACE CASSANDRA_KEYSPACE @@ -699,8 +694,6 @@ use the ``TimeUUID`` type as a comparator:: create column family task_results with comparator = TimeUUIDType; -new_cassandra uses detailed mode by default, and that cannot be disabled. - CASSANDRA_OPTIONS ~~~~~~~~~~~~~~~~~~~~~~~~~~~ From ec353d2a7cc5d2d8d43e88488140eaf2693e5c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 10:44:15 +0200 Subject: [PATCH 0601/1103] files in place for DataStax cassandra driver --- celery/backends/new_cassandra.py | 173 ++++++++++++++++++ celery/tests/backends/test_new_cassandra.py | 0 docs/includes/installation.txt | 5 +- .../celery.backends.new_cassandra.rst | 11 ++ requirements/extras/new_cassandra.txt | 1 + setup.py | 1 + 6 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 celery/backends/new_cassandra.py create mode 100644 celery/tests/backends/test_new_cassandra.py create mode 100644 docs/internals/reference/celery.backends.new_cassandra.rst create mode 100644 requirements/extras/new_cassandra.txt diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py new file mode 100644 index 000000000..67c388902 --- /dev/null +++ b/celery/backends/new_cassandra.py @@ -0,0 +1,173 @@ +# -* coding: utf-8 -*- +""" + celery.backends.cassandra + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Apache Cassandra result store backend. + +""" +from __future__ import absolute_import + +try: # pragma: no cover + import cassandra +except ImportError: # pragma: no cover + cassandra = None # noqa + +import time + +from celery import states +from celery.exceptions import ImproperlyConfigured +from celery.five import monotonic +from celery.utils.log import get_logger + +from .base import BaseBackend + +__all__ = ['NewCassandraBackend'] + +logger = get_logger(__name__) + + +class NewCassandraBackend(BaseBackend): + """New Cassandra backend utilizing DataStax's driver + + .. attribute:: servers + + List of Cassandra servers with format: ``hostname:port`` or ``hostname`` + + :raises celery.exceptions.ImproperlyConfigured: if + module :mod:`pycassa` is not available. + + """ + servers = [] + keyspace = None + column_family = None + detailed_mode = False + _retry_timeout = 300 + _retry_wait = 3 + supports_autoexpire = True + + def __init__(self, servers=None, keyspace=None, column_family=None, + cassandra_options=None, detailed_mode=False, port=9042, **kwargs): + """Initialize Cassandra backend. + + Raises :class:`celery.exceptions.ImproperlyConfigured` if + the :setting:`CASSANDRA_SERVERS` setting is not set. + + """ + super(NewCassandraBackend, self).__init__(**kwargs) + + if not cassandra: + raise ImproperlyConfigured( + 'You need to install the cassandra library to use the ' + 'Cassandra backend. See https://github.com/datastax/python-driver') + + conf = self.app.conf + self.servers = (servers or + conf.get('CASSANDRA_SERVERS') or + self.servers) + self.port = (port or + conf.get('CASSANDRA_PORT')) + self.keyspace = (keyspace or + conf.get('CASSANDRA_KEYSPACE') or + self.keyspace) + self.column_family = (column_family or + conf.get('CASSANDRA_COLUMN_FAMILY') or + self.column_family) + self.cassandra_options = dict(conf.get('CASSANDRA_OPTIONS') or {}, + **cassandra_options or {}) + self.detailed_mode = (detailed_mode or + conf.get('CASSANDRA_DETAILED_MODE') or + self.detailed_mode) + read_cons = conf.get('CASSANDRA_READ_CONSISTENCY') or 'LOCAL_QUORUM' + write_cons = conf.get('CASSANDRA_WRITE_CONSISTENCY') or 'LOCAL_QUORUM' + try: + self.read_consistency = getattr(cassandra.ConsistencyLevel, + read_cons) + except AttributeError: + self.read_consistency = cassandra.ConsistencyLevel.LOCAL_QUORUM + try: + self.write_consistency = getattr(cassandra.ConsistencyLevel, + write_cons) + except AttributeError: + self.write_consistency = cassandra.ConsistencyLevel.LOCAL_QUORUM + + if not self.servers or not self.keyspace or not self.column_family: + raise ImproperlyConfigured( + 'Cassandra backend not configured.') + + self._connection = None + + def _get_connection(self): + if self._connection is None: + self._connection = cassandra.Cluster(self.servers, port=self.port) + + def _retry_on_error(self, fun, *args, **kwargs): + ts = monotonic() + self._retry_timeout + while 1: + try: + return fun(*args, **kwargs) + except (cassandra.Unavailable, + cassandra.Timeout, + cassandra.InvalidRequest) as exc: + if monotonic() > ts: + raise + logger.warning('Cassandra error: %r. Retrying...', exc) + time.sleep(self._retry_wait) + + def _store_result(self, task_id, result, status, + traceback=None, request=None, **kwargs): + """Store return value and status of an executed task.""" + + def _do_store(): + self._get_connection() + date_done = self.app.now() + + + + meta = {'status': status, + 'date_done': date_done.strftime('%Y-%m-%dT%H:%M:%SZ'), + 'traceback': self.encode(traceback), + 'result': self.encode(result), + 'children': self.encode( + self.current_task_children(request), + )} + if self.detailed_mode: + cf.insert( + task_id, {date_done: self.encode(meta)}, ttl=self.expires, + ) + else: + cf.insert(task_id, meta, ttl=self.expires) + + return self._retry_on_error(_do_store) + + def _get_task_meta_for(self, task_id): + """Get task metadata for a task by id.""" + + def _do_get(): + cf = self._get_column_family() + try: + if self.detailed_mode: + row = cf.get(task_id, column_reversed=True, column_count=1) + return self.decode(list(row.values())[0]) + else: + obj = cf.get(task_id) + return self.meta_from_decoded({ + 'task_id': task_id, + 'status': obj['status'], + 'result': self.decode(obj['result']), + 'date_done': obj['date_done'], + 'traceback': self.decode(obj['traceback']), + 'children': self.decode(obj['children']), + }) + except (KeyError, pycassa.NotFoundException): + return {'status': states.PENDING, 'result': None} + + return self._retry_on_error(_do_get) + + def __reduce__(self, args=(), kwargs={}): + kwargs.update( + dict(servers=self.servers, + keyspace=self.keyspace, + column_family=self.column_family, + cassandra_options=self.cassandra_options)) + return super(NewCassandraBackend, self).__reduce__(args, kwargs) diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py new file mode 100644 index 000000000..e69de29bb diff --git a/docs/includes/installation.txt b/docs/includes/installation.txt index 2ab46ab35..18c2ab9b6 100644 --- a/docs/includes/installation.txt +++ b/docs/includes/installation.txt @@ -78,7 +78,10 @@ Transports and Backends for using memcached as a result backend. :celery[cassandra]: - for using Apache Cassandra as a result backend. + for using Apache Cassandra as a result backend with pycassa driver. + +:celery[new_cassandra]: + for using Apache Cassandra as a result backend with DataStax driver. :celery[couchdb]: for using CouchDB as a message transport (*experimental*). diff --git a/docs/internals/reference/celery.backends.new_cassandra.rst b/docs/internals/reference/celery.backends.new_cassandra.rst new file mode 100644 index 000000000..e7696fa62 --- /dev/null +++ b/docs/internals/reference/celery.backends.new_cassandra.rst @@ -0,0 +1,11 @@ +================================================ + celery.backends.new_cassandra +================================================ + +.. contents:: + :local: +.. currentmodule:: celery.backends.new_cassandra + +.. automodule:: celery.backends.new_cassandra + :members: + :undoc-members: diff --git a/requirements/extras/new_cassandra.txt b/requirements/extras/new_cassandra.txt new file mode 100644 index 000000000..a94062dad --- /dev/null +++ b/requirements/extras/new_cassandra.txt @@ -0,0 +1 @@ +cassandra-driver \ No newline at end of file diff --git a/setup.py b/setup.py index 136318076..01cc1c427 100644 --- a/setup.py +++ b/setup.py @@ -160,6 +160,7 @@ def reqs(*f): 'eventlet', 'gevent', 'msgpack', 'yaml', 'redis', 'mongodb', 'sqs', 'couchdb', 'riak', 'beanstalk', 'zookeeper', 'zeromq', 'sqlalchemy', 'librabbitmq', 'pyro', 'slmq', + 'new_cassandra', } extras_require = {x: extras(x + '.txt') for x in features} extra['extras_require'] = extras_require From 0da49c0c416290ec0a97b9b230953813c736f73d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 11:23:14 +0200 Subject: [PATCH 0602/1103] needs preliminary tests --- celery/backends/new_cassandra.py | 82 +++++++++++++++++++------------- docs/configuration.rst | 22 +++++++++ 2 files changed, 72 insertions(+), 32 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index 67c388902..ec272fa55 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -96,10 +96,41 @@ def __init__(self, servers=None, keyspace=None, column_family=None, 'Cassandra backend not configured.') self._connection = None + self._session = None def _get_connection(self): if self._connection is None: self._connection = cassandra.Cluster(self.servers, port=self.port) + self._session = self._connection.connect(self.keyspace) + + self._write_stmt = self._session.prepare('''INSERT INTO '''+ + self.column_family+''' (task_id,status, result,date_done,''' + '''traceback, children) VALUES (?, ?, ?, ?, ?, ?) ''' + '''USING TTL '''+str(self.expires), + consistency_level=self.write_consistency) + + self._make_stmt = self._session.prepare( + '''CREATE TABLE '''+self.column_family+''' ( + task_id text, + status text, + result text, + date_done timestamp, + traceback text, + children text, + PRIMARY KEY ((task_id), date_done) + ) WITH CLUSTERING ORDER BY (date_done DESC) + WITH default_time_to_live = '''+str(self.expires)+';') + + self._read_stmt = self._session.prepare( + '''SELECT task_id, status, result, date_done, traceback, children + FROM '''+self.column_family+''' + WHERE task_id=? LIMIT 1''', + consistency_level=self.read_consistency) + + try: + self._session.execute(self._make_stmt) + except cassandra.AlreadyExists: + pass def _retry_on_error(self, fun, *args, **kwargs): ts = monotonic() + self._retry_timeout @@ -122,46 +153,33 @@ def _do_store(): self._get_connection() date_done = self.app.now() - - - meta = {'status': status, - 'date_done': date_done.strftime('%Y-%m-%dT%H:%M:%SZ'), - 'traceback': self.encode(traceback), - 'result': self.encode(result), - 'children': self.encode( - self.current_task_children(request), - )} - if self.detailed_mode: - cf.insert( - task_id, {date_done: self.encode(meta)}, ttl=self.expires, - ) - else: - cf.insert(task_id, meta, ttl=self.expires) - + self._session.execute(self._write_stmt, ( + task_id, status, result, + self.app.now().strftime('%Y-%m-%dT%H:%M:%SZ'), + traceback, self.encode(self.current_task_children(request)) + )) return self._retry_on_error(_do_store) def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" def _do_get(): - cf = self._get_column_family() - try: - if self.detailed_mode: - row = cf.get(task_id, column_reversed=True, column_count=1) - return self.decode(list(row.values())[0]) - else: - obj = cf.get(task_id) - return self.meta_from_decoded({ - 'task_id': task_id, - 'status': obj['status'], - 'result': self.decode(obj['result']), - 'date_done': obj['date_done'], - 'traceback': self.decode(obj['traceback']), - 'children': self.decode(obj['children']), - }) - except (KeyError, pycassa.NotFoundException): + + res = self._session.execute(self._read_stmt, (task_id, )) + if not res: return {'status': states.PENDING, 'result': None} + task_id, status, result, date_done, traceback, children = res[0] + + return self.meta_from_decoded({ + 'task_id': task_id, + 'status': status, + 'result': self.decode(result), + 'date_done': date_done, + 'traceback': self.decode(traceback), + 'children': self.decode(children), + }) + return self._retry_on_error(_do_get) def __reduce__(self, args=(), kwargs={}): diff --git a/docs/configuration.rst b/docs/configuration.rst index 49ca75877..21f6e99ff 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -213,6 +213,10 @@ Can be one of the following: Use `Cassandra`_ to store the results. See :ref:`conf-cassandra-result-backend`. +* new_cassandra + Use `Cassandra`_ to store the results, using other backend than _cassandra_. + See :ref:`conf-cassandra-result-backend`. + * ironcache Use `IronCache`_ to store the results. See :ref:`conf-ironcache-result-backend`. @@ -544,6 +548,16 @@ Cassandra backend settings $ pip install pycassa + If you are using new_cassandra, :mod:`cassandra-driver` is required instead: + https://pypi.python.org/pypi/cassandra-driver + + To install, use `pip` or `easy_install`: + + .. code-block:: bash + + $ pip install cassandra-driver + + This backend requires the following configuration directives to be set. .. setting:: CASSANDRA_SERVERS @@ -555,6 +569,10 @@ List of ``host:port`` Cassandra servers. e.g.:: CASSANDRA_SERVERS = ['localhost:9160'] +Omit the ``port`` part when using new_cassandra. e.g.:: + + CASSANDRA_SERVERS = ['localhost'] + .. setting:: CASSANDRA_KEYSPACE CASSANDRA_KEYSPACE @@ -601,6 +619,8 @@ use the ``TimeUUID`` type as a comparator:: create column family task_results with comparator = TimeUUIDType; +new_cassandra uses detailed mode by default, and that cannot be disabled. + CASSANDRA_OPTIONS ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -608,6 +628,8 @@ Options to be passed to the `pycassa connection pool`_ (optional). .. _`pycassa connection pool`: http://pycassa.github.com/pycassa/api/pycassa/pool.html +Not used in new_cassandra + Example configuration ~~~~~~~~~~~~~~~~~~~~~ From 0b294d5d846901fbe408fb4bff859ea999dd70c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 15:41:18 +0200 Subject: [PATCH 0603/1103] works for me --- celery/backends/__init__.py | 1 + celery/backends/new_cassandra.py | 174 ++++++++++++++----------------- docs/configuration.rst | 106 ++++++++++++++++--- 3 files changed, 173 insertions(+), 108 deletions(-) diff --git a/celery/backends/__init__.py b/celery/backends/__init__.py index eec585227..afff815c2 100644 --- a/celery/backends/__init__.py +++ b/celery/backends/__init__.py @@ -30,6 +30,7 @@ 'db': 'celery.backends.database:DatabaseBackend', 'database': 'celery.backends.database:DatabaseBackend', 'cassandra': 'celery.backends.cassandra:CassandraBackend', + 'new_cassandra': 'celery.backends.new_cassandra:NewCassandraBackend', 'couchbase': 'celery.backends.couchbase:CouchBaseBackend', 'couchdb': 'celery.backends.couchdb:CouchDBBackend', 'riak': 'celery.backends.riak:RiakBackend', diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index ec272fa55..fe764c9d2 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -1,9 +1,9 @@ # -* coding: utf-8 -*- """ - celery.backends.cassandra + celery.backends.new_cassandra ~~~~~~~~~~~~~~~~~~~~~~~~~ - Apache Cassandra result store backend. + Apache Cassandra result store backend using DataStax driver """ from __future__ import absolute_import @@ -13,11 +13,8 @@ except ImportError: # pragma: no cover cassandra = None # noqa -import time - from celery import states from celery.exceptions import ImproperlyConfigured -from celery.five import monotonic from celery.utils.log import get_logger from .base import BaseBackend @@ -32,22 +29,19 @@ class NewCassandraBackend(BaseBackend): .. attribute:: servers - List of Cassandra servers with format: ``hostname:port`` or ``hostname`` + List of Cassandra servers with format: ``hostname`` :raises celery.exceptions.ImproperlyConfigured: if - module :mod:`pycassa` is not available. + module :mod:`cassandra` is not available. """ servers = [] keyspace = None - column_family = None - detailed_mode = False - _retry_timeout = 300 - _retry_wait = 3 + table = None supports_autoexpire = True - def __init__(self, servers=None, keyspace=None, column_family=None, - cassandra_options=None, detailed_mode=False, port=9042, **kwargs): + def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, + port=9042, **kwargs): """Initialize Cassandra backend. Raises :class:`celery.exceptions.ImproperlyConfigured` if @@ -70,14 +64,16 @@ def __init__(self, servers=None, keyspace=None, column_family=None, self.keyspace = (keyspace or conf.get('CASSANDRA_KEYSPACE') or self.keyspace) - self.column_family = (column_family or - conf.get('CASSANDRA_COLUMN_FAMILY') or - self.column_family) - self.cassandra_options = dict(conf.get('CASSANDRA_OPTIONS') or {}, - **cassandra_options or {}) - self.detailed_mode = (detailed_mode or - conf.get('CASSANDRA_DETAILED_MODE') or - self.detailed_mode) + self.table = (table or + conf.get('CASSANDRA_TABLE') or + self.table) + expires = (entry_ttl or conf.get('CASSANDRA_ENTRY_TTL', None)) + + if expires is not None: + self.cqlexpires = ' USING TTL %s' % (expires, ) + else: + self.cqlexpires = '' + read_cons = conf.get('CASSANDRA_READ_CONSISTENCY') or 'LOCAL_QUORUM' write_cons = conf.get('CASSANDRA_WRITE_CONSISTENCY') or 'LOCAL_QUORUM' try: @@ -91,101 +87,91 @@ def __init__(self, servers=None, keyspace=None, column_family=None, except AttributeError: self.write_consistency = cassandra.ConsistencyLevel.LOCAL_QUORUM - if not self.servers or not self.keyspace or not self.column_family: + if not self.servers or not self.keyspace or not self.table: raise ImproperlyConfigured( 'Cassandra backend not configured.') self._connection = None self._session = None - - def _get_connection(self): + self._write_stmt = None + self._read_stmt = None + + def process_cleanup(self): + if self._connection is not None: + self._session.shutdown() + self._connection = None + self._session = None + + def _get_connection(self, write=False): + # only writers can create the table to get rid of two processes + # creating table at same time and Cassandra choking on that if self._connection is None: - self._connection = cassandra.Cluster(self.servers, port=self.port) + self._connection = cassandra.cluster.Cluster(self.servers, + port=self.port) self._session = self._connection.connect(self.keyspace) - self._write_stmt = self._session.prepare('''INSERT INTO '''+ - self.column_family+''' (task_id,status, result,date_done,''' - '''traceback, children) VALUES (?, ?, ?, ?, ?, ?) ''' - '''USING TTL '''+str(self.expires), - consistency_level=self.write_consistency) - - self._make_stmt = self._session.prepare( - '''CREATE TABLE '''+self.column_family+''' ( - task_id text, - status text, - result text, - date_done timestamp, - traceback text, - children text, - PRIMARY KEY ((task_id), date_done) - ) WITH CLUSTERING ORDER BY (date_done DESC) - WITH default_time_to_live = '''+str(self.expires)+';') - - self._read_stmt = self._session.prepare( - '''SELECT task_id, status, result, date_done, traceback, children - FROM '''+self.column_family+''' - WHERE task_id=? LIMIT 1''', - consistency_level=self.read_consistency) - - try: - self._session.execute(self._make_stmt) - except cassandra.AlreadyExists: - pass - - def _retry_on_error(self, fun, *args, **kwargs): - ts = monotonic() + self._retry_timeout - while 1: - try: - return fun(*args, **kwargs) - except (cassandra.Unavailable, - cassandra.Timeout, - cassandra.InvalidRequest) as exc: - if monotonic() > ts: - raise - logger.warning('Cassandra error: %r. Retrying...', exc) - time.sleep(self._retry_wait) + self._write_stmt = cassandra.query.SimpleStatement( + 'INSERT INTO '+self.table+' (task_id, status, result,''' + ''' date_done, traceback, children) VALUES''' + ' (%s, %s, %s, %s, %s, %s) '+self.cqlexpires+';') + self._write_stmt.consistency_level = self.write_consistency + + self._read_stmt = cassandra.query.SimpleStatement( + '''SELECT status, result, date_done, traceback, children + FROM '''+self.table+''' + WHERE task_id=%s''') + self._read_stmt.consistency_level = self.read_consistency + + if write: + self._make_stmt = cassandra.query.SimpleStatement( + '''CREATE TABLE '''+self.table+''' ( + task_id text, + status text, + result blob, + date_done text, + traceback blob, + children blob, + PRIMARY KEY (task_id) + );''') + self._make_stmt.consistency_level = self.write_consistency + try: + self._session.execute(self._make_stmt) + except cassandra.AlreadyExists: + pass def _store_result(self, task_id, result, status, traceback=None, request=None, **kwargs): """Store return value and status of an executed task.""" + self._get_connection(write=True) - def _do_store(): - self._get_connection() - date_done = self.app.now() - - self._session.execute(self._write_stmt, ( - task_id, status, result, - self.app.now().strftime('%Y-%m-%dT%H:%M:%SZ'), - traceback, self.encode(self.current_task_children(request)) - )) - return self._retry_on_error(_do_store) + self._session.execute(self._write_stmt, ( + task_id, status, buffer(self.encode(result)), + self.app.now().strftime('%Y-%m-%dT%H:%M:%SZ'), + buffer(self.encode(traceback)), buffer(self.encode(self.current_task_children(request))) + )) def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" + self._get_connection() - def _do_get(): - - res = self._session.execute(self._read_stmt, (task_id, )) - if not res: - return {'status': states.PENDING, 'result': None} - - task_id, status, result, date_done, traceback, children = res[0] + res = self._session.execute(self._read_stmt, (task_id, )) + if not res: + return {'status': states.PENDING, 'result': None} - return self.meta_from_decoded({ - 'task_id': task_id, - 'status': status, - 'result': self.decode(result), - 'date_done': date_done, - 'traceback': self.decode(traceback), - 'children': self.decode(children), - }) + status, result, date_done, traceback, children = res[0] - return self._retry_on_error(_do_get) + return self.meta_from_decoded({ + 'task_id': task_id, + 'status': str(status), + 'result': self.decode(str(result)), + 'date_done': date_done, + 'traceback': self.decode(str(traceback)), + 'children': self.decode(str(children)), + }) def __reduce__(self, args=(), kwargs={}): kwargs.update( dict(servers=self.servers, keyspace=self.keyspace, - column_family=self.column_family, - cassandra_options=self.cassandra_options)) + table=self.table)) return super(NewCassandraBackend, self).__reduce__(args, kwargs) diff --git a/docs/configuration.rst b/docs/configuration.rst index 21f6e99ff..735234148 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -214,8 +214,8 @@ Can be one of the following: See :ref:`conf-cassandra-result-backend`. * new_cassandra - Use `Cassandra`_ to store the results, using other backend than _cassandra_. - See :ref:`conf-cassandra-result-backend`. + Use `new_cassandra`_ to store the results, using newer database driver than _cassandra_. + See :ref:`conf-new_cassandra-result-backend`. * ironcache Use `IronCache`_ to store the results. @@ -532,30 +532,110 @@ Example configuration 'taskmeta_collection': 'my_taskmeta_collection', } -.. _conf-cassandra-result-backend: +.. _conf-new_cassandra-result-backend: -Cassandra backend settings + +new_cassandra backend settings -------------------------- .. note:: - The Cassandra backend requires the :mod:`pycassa` library: - http://pypi.python.org/pypi/pycassa/ + This Cassandra backend driver requires :mod:`cassandra-driver`. + https://pypi.python.org/pypi/cassandra-driver - To install the pycassa package use `pip` or `easy_install`: + To install, use `pip` or `easy_install`: .. code-block:: bash - $ pip install pycassa + $ pip install cassandra-driver - If you are using new_cassandra, :mod:`cassandra-driver` is required instead: - https://pypi.python.org/pypi/cassandra-driver +This backend requires the following configuration directives to be set. + +.. setting:: CASSANDRA_SERVERS + +CASSANDRA_SERVERS +~~~~~~~~~~~~~~~~~ + +List of ``host`` Cassandra servers. e.g.:: + + CASSANDRA_SERVERS = ['localhost'] - To install, use `pip` or `easy_install`: + +.. setting:: CASSANDRA_PORT + +CASSANDRA_PORT +~~~~~~~~~~~~~~ + +Port to contact the Cassandra servers on. Default is 9042. + +.. setting:: CASSANDRA_KEYSPACE + +CASSANDRA_KEYSPACE +~~~~~~~~~~~~~~~~~~ + +The keyspace in which to store the results. e.g.:: + + CASSANDRA_KEYSPACE = 'tasks_keyspace' + +.. setting:: CASSANDRA_COLUMN_FAMILY + +CASSANDRA_TABLE +~~~~~~~~~~~~~~~~~~~~~~~ + +The table (column family) in which to store the results. e.g.:: + + CASSANDRA_TABLE = 'tasks' + +.. setting:: CASSANDRA_READ_CONSISTENCY + +CASSANDRA_READ_CONSISTENCY +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The read consistency used. Values can be ``ONE``, ``TWO``, ``THREE``, ``QUORUM``, ``ALL``, +``LOCAL_QUORUM``, ``EACH_QUORUM``, ``LOCAL_ONE``. + +.. setting:: CASSANDRA_WRITE_CONSISTENCY + +CASSANDRA_WRITE_CONSISTENCY +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The write consistency used. Values can be ``ONE``, ``TWO``, ``THREE``, ``QUORUM``, ``ALL``, +``LOCAL_QUORUM``, ``EACH_QUORUM``, ``LOCAL_ONE``. + +.. setting:: CASSANDRA_ENTRY_TTL + +CASSANDRA_ENTRY_TTL +~~~~~~~~~~~~~~~~~~~ + +Time-to-live for status entries. They will expire and be removed after that many seconds +after adding. Default (None) means they will never expire. + +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + CASSANDRA_SERVERS = ['localhost'] + CASSANDRA_KEYSPACE = 'celery' + CASSANDRA_COLUMN_FAMILY = 'task_results' + CASSANDRA_READ_CONSISTENCY = 'ONE' + CASSANDRA_WRITE_CONSISTENCY = 'ONE' + CASSANDRA_ENTRY_TTL = 86400 + + +Cassandra backend settings +-------------------------- + +.. note:: + + The Cassandra backend requires the :mod:`pycassa` library: + http://pypi.python.org/pypi/pycassa/ + + To install the pycassa package use `pip` or `easy_install`: .. code-block:: bash - $ pip install cassandra-driver + $ pip install pycassa This backend requires the following configuration directives to be set. @@ -628,8 +708,6 @@ Options to be passed to the `pycassa connection pool`_ (optional). .. _`pycassa connection pool`: http://pycassa.github.com/pycassa/api/pycassa/pool.html -Not used in new_cassandra - Example configuration ~~~~~~~~~~~~~~~~~~~~~ From 8221c1fa52ef47288995fcdf3ce658e33484cffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 15:44:09 +0200 Subject: [PATCH 0604/1103] better no tests than fake tests --- celery/tests/backends/test_new_cassandra.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 celery/tests/backends/test_new_cassandra.py diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py deleted file mode 100644 index e69de29bb..000000000 From 3890fe43ba971f9c792fde9d379d467164dc6789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 18:04:22 +0200 Subject: [PATCH 0605/1103] cassandra deprecated --- celery/backends/cassandra.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index aa8e688cc..a427688f9 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -17,6 +17,7 @@ import socket import time +import warnings from celery import states from celery.exceptions import ImproperlyConfigured @@ -98,6 +99,9 @@ def __init__(self, servers=None, keyspace=None, column_family=None, self._column_family = None + warnings.warn("cassandra backend is deprecated. Use new_cassandra instead.", + DeprecationWarning) + def _retry_on_error(self, fun, *args, **kwargs): ts = monotonic() + self._retry_timeout while 1: From db0e0314413e4ba58df58acb82c1af9414679ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 18:38:29 +0200 Subject: [PATCH 0606/1103] tests added --- celery/backends/new_cassandra.py | 26 +++-- celery/tests/backends/test_new_cassandra.py | 102 ++++++++++++++++++++ 2 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 celery/tests/backends/test_new_cassandra.py diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index fe764c9d2..e83031ec5 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -25,7 +25,7 @@ class NewCassandraBackend(BaseBackend): - """New Cassandra backend utilizing DataStax's driver + """New Cassandra backend utilizing DataStax driver .. attribute:: servers @@ -38,7 +38,7 @@ class NewCassandraBackend(BaseBackend): servers = [] keyspace = None table = None - supports_autoexpire = True + supports_autoexpire = True # autoexpire supported via entry_ttl def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, port=9042, **kwargs): @@ -103,8 +103,11 @@ def process_cleanup(self): self._session = None def _get_connection(self, write=False): - # only writers can create the table to get rid of two processes - # creating table at same time and Cassandra choking on that + """ + Prepare the connection for action + + :param write: bool - are we a writer? + """ if self._connection is None: self._connection = cassandra.cluster.Cluster(self.servers, port=self.port) @@ -123,6 +126,14 @@ def _get_connection(self, write=False): self._read_stmt.consistency_level = self.read_consistency if write: + # Only possible writers "workers" are allowed to issue + # CREATE TABLE. This is to prevent conflicting situations + # where both task-creator and task-executor would issue it + # at the same time. + + # Anyway, if you are doing anything critical, you should + # have probably created this table in advance, in which case + # this query will be a no-op (instant fail with AlreadyExists) self._make_stmt = cassandra.query.SimpleStatement( '''CREATE TABLE '''+self.table+''' ( task_id text, @@ -145,9 +156,12 @@ def _store_result(self, task_id, result, status, self._get_connection(write=True) self._session.execute(self._write_stmt, ( - task_id, status, buffer(self.encode(result)), + task_id, + status, + buffer(self.encode(result)), self.app.now().strftime('%Y-%m-%dT%H:%M:%SZ'), - buffer(self.encode(traceback)), buffer(self.encode(self.current_task_children(request))) + buffer(self.encode(traceback)), + buffer(self.encode(self.current_task_children(request))) )) def _get_task_meta_for(self, task_id): diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py new file mode 100644 index 000000000..01ecea0e9 --- /dev/null +++ b/celery/tests/backends/test_new_cassandra.py @@ -0,0 +1,102 @@ +from __future__ import absolute_import + +from pickle import loads, dumps + +from celery import states +from celery.exceptions import ImproperlyConfigured +from celery.tests.case import ( + AppCase, Mock, mock_module, depends_on_current_app, MagicMock +) + +class Object(object): + pass + +class test_NewCassandraBackend(AppCase): + + def setup(self): + self.app.conf.update( + CASSANDRA_SERVERS=['example.com'], + CASSANDRA_KEYSPACE='celery', + CASSANDRA_TABLE='task_results', + ) + + def test_init_no_cassandra(self): + """ + Tests behaviour when no python-driver is installed. + new_cassandra should raise ImproperlyConfigured + """ + with mock_module('cassandra'): + from celery.backends import new_cassandra as mod + prev, mod.cassandra = mod.cassandra, None + try: + with self.assertRaises(ImproperlyConfigured): + mod.NewCassandraBackend(app=self.app) + finally: + mod.cassandra = prev + + def test_init_with_and_without_LOCAL_QUROM(self): + with mock_module('cassandra'): + from celery.backends import new_cassandra as mod + mod.cassandra = Mock() + cons = mod.cassandra.ConsistencyLevel = Object() + cons.LOCAL_QUORUM = 'foo' + + self.app.conf.CASSANDRA_READ_CONSISTENCY = 'LOCAL_FOO' + self.app.conf.CASSANDRA_WRITE_CONSISTENCY = 'LOCAL_FOO' + + mod.NewCassandraBackend(app=self.app) + cons.LOCAL_FOO = 'bar' + mod.NewCassandraBackend(app=self.app) + + # no servers raises ImproperlyConfigured + with self.assertRaises(ImproperlyConfigured): + self.app.conf.CASSANDRA_SERVERS = None + mod.NewCassandraBackend( + app=self.app, keyspace='b', column_family='c', + ) + + @depends_on_current_app + def test_reduce(self): + with mock_module('cassandra'): + from celery.backends.new_cassandra import NewCassandraBackend + self.assertTrue(loads(dumps(NewCassandraBackend(app=self.app)))) + + def test_get_task_meta_for(self): + with mock_module('cassandra'): + from celery.backends import new_cassandra as mod + mod.cassandra = Mock() + x = mod.NewCassandraBackend(app=self.app) + x._connection = True + session = x._session = Mock() + execute = session.execute = Mock() + execute.return_value = [ + [states.SUCCESS, '1', 'date', '', None] + ] + x.decode = Mock() + meta = x._get_task_meta_for('task_id') + self.assertEqual(meta['status'], states.SUCCESS) + + x._session.execute.return_value = [] + meta = x._get_task_meta_for('task_id') + self.assertEqual(meta['status'], states.PENDING) + + + def test_store_result(self): + with mock_module('cassandra'): + from celery.backends import new_cassandra as mod + mod.cassandra = Mock() + + x = mod.NewCassandraBackend(app=self.app) + x._connection = True + session = x._session = Mock() + execute = session.execute = Mock() + x._store_result('task_id', 'result', states.SUCCESS) + + def test_process_cleanup(self): + with mock_module('cassandra'): + from celery.backends import new_cassandra as mod + x = mod.NewCassandraBackend(app=self.app) + x.process_cleanup() + + self.assertIsNone(x._connection) + self.assertIsNone(x._session) From 3c9cab98e485869518c3f4001925e430d2c66f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 18:43:52 +0200 Subject: [PATCH 0607/1103] PEP8 --- celery/backends/new_cassandra.py | 12 ++++++------ celery/tests/backends/test_new_cassandra.py | 7 ++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index e83031ec5..974b9e95e 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -175,12 +175,12 @@ def _get_task_meta_for(self, task_id): status, result, date_done, traceback, children = res[0] return self.meta_from_decoded({ - 'task_id': task_id, - 'status': str(status), - 'result': self.decode(str(result)), - 'date_done': date_done, - 'traceback': self.decode(str(traceback)), - 'children': self.decode(str(children)), + 'task_id': task_id, + 'status': str(status), + 'result': self.decode(str(result)), + 'date_done': date_done, + 'traceback': self.decode(str(traceback)), + 'children': self.decode(str(children)), }) def __reduce__(self, args=(), kwargs={}): diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 01ecea0e9..1fbc18909 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -5,12 +5,14 @@ from celery import states from celery.exceptions import ImproperlyConfigured from celery.tests.case import ( - AppCase, Mock, mock_module, depends_on_current_app, MagicMock + AppCase, Mock, mock_module, depends_on_current_app ) + class Object(object): pass + class test_NewCassandraBackend(AppCase): def setup(self): @@ -80,7 +82,6 @@ def test_get_task_meta_for(self): meta = x._get_task_meta_for('task_id') self.assertEqual(meta['status'], states.PENDING) - def test_store_result(self): with mock_module('cassandra'): from celery.backends import new_cassandra as mod @@ -89,7 +90,7 @@ def test_store_result(self): x = mod.NewCassandraBackend(app=self.app) x._connection = True session = x._session = Mock() - execute = session.execute = Mock() + session.execute = Mock() x._store_result('task_id', 'result', states.SUCCESS) def test_process_cleanup(self): From 51e4a585daaf7574e9e2f8d5f1b09e9bc8989f29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 18:58:39 +0200 Subject: [PATCH 0608/1103] "detailed mode" is the default and only --- celery/backends/new_cassandra.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index 974b9e95e..3415feb34 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -122,7 +122,7 @@ def _get_connection(self, write=False): self._read_stmt = cassandra.query.SimpleStatement( '''SELECT status, result, date_done, traceback, children FROM '''+self.table+''' - WHERE task_id=%s''') + WHERE task_id=%s LIMIT 1''') self._read_stmt.consistency_level = self.read_consistency if write: @@ -139,11 +139,12 @@ def _get_connection(self, write=False): task_id text, status text, result blob, - date_done text, + date_done timestamp, traceback blob, children blob, - PRIMARY KEY (task_id) - );''') + PRIMARY KEY ((task_id), date_done) + ) + WITH CLUSTERING ORDER BY (date_done DESC);''') self._make_stmt.consistency_level = self.write_consistency try: self._session.execute(self._make_stmt) @@ -159,7 +160,7 @@ def _store_result(self, task_id, result, status, task_id, status, buffer(self.encode(result)), - self.app.now().strftime('%Y-%m-%dT%H:%M:%SZ'), + self.app.now(), buffer(self.encode(traceback)), buffer(self.encode(self.current_task_children(request))) )) @@ -178,7 +179,7 @@ def _get_task_meta_for(self, task_id): 'task_id': task_id, 'status': str(status), 'result': self.decode(str(result)), - 'date_done': date_done, + 'date_done': date_done.strftime('%Y-%m-%dT%H:%M:%SZ'), 'traceback': self.decode(str(traceback)), 'children': self.decode(str(children)), }) From 29708476069e1a15d86c5eb9b390609712fcc43b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 19:11:05 +0200 Subject: [PATCH 0609/1103] fix for unit test --- celery/tests/backends/test_new_cassandra.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 1fbc18909..096718bf4 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -1,7 +1,7 @@ from __future__ import absolute_import from pickle import loads, dumps - +from datetime import datetime from celery import states from celery.exceptions import ImproperlyConfigured from celery.tests.case import ( @@ -72,7 +72,7 @@ def test_get_task_meta_for(self): session = x._session = Mock() execute = session.execute = Mock() execute.return_value = [ - [states.SUCCESS, '1', 'date', '', None] + [states.SUCCESS, '1', datetime.now(), '', None] ] x.decode = Mock() meta = x._get_task_meta_for('task_id') From 4a02f14c19fb090a34691be007dbed2b26ccf37e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 19:20:29 +0200 Subject: [PATCH 0610/1103] py3k (buffer, memoryview) --- celery/backends/new_cassandra.py | 6 +++++- celery/tests/backends/test_new_cassandra.py | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index 3415feb34..e6068ceee 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -8,6 +8,7 @@ """ from __future__ import absolute_import +import sys try: # pragma: no cover import cassandra except ImportError: # pragma: no cover @@ -16,7 +17,6 @@ from celery import states from celery.exceptions import ImproperlyConfigured from celery.utils.log import get_logger - from .base import BaseBackend __all__ = ['NewCassandraBackend'] @@ -156,6 +156,10 @@ def _store_result(self, task_id, result, status, """Store return value and status of an executed task.""" self._get_connection(write=True) + import sys + if sys.version_info > (3,): + buffer = memoryview + self._session.execute(self._write_stmt, ( task_id, status, diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 096718bf4..94cc0b3a7 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -8,7 +8,6 @@ AppCase, Mock, mock_module, depends_on_current_app ) - class Object(object): pass From 3a144467749bf5ba389a74fcb64f8bb4b0b00526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 19:33:11 +0200 Subject: [PATCH 0611/1103] more confusion with binary types in Py3K --- celery/backends/new_cassandra.py | 21 ++++++++++++--------- celery/tests/backends/test_new_cassandra.py | 6 ++++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index e6068ceee..80cc20a80 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -9,6 +9,7 @@ from __future__ import absolute_import import sys +import six try: # pragma: no cover import cassandra except ImportError: # pragma: no cover @@ -157,16 +158,18 @@ def _store_result(self, task_id, result, status, self._get_connection(write=True) import sys - if sys.version_info > (3,): - buffer = memoryview + if six.PY3: + buf = lambda x: bytes(x, 'utf8') + else: + buf = buffer self._session.execute(self._write_stmt, ( task_id, status, - buffer(self.encode(result)), + buf(self.encode(result)), self.app.now(), - buffer(self.encode(traceback)), - buffer(self.encode(self.current_task_children(request))) + buf(self.encode(traceback)), + buf(self.encode(self.current_task_children(request))) )) def _get_task_meta_for(self, task_id): @@ -181,11 +184,11 @@ def _get_task_meta_for(self, task_id): return self.meta_from_decoded({ 'task_id': task_id, - 'status': str(status), - 'result': self.decode(str(result)), + 'status': status, + 'result': self.decode(result), 'date_done': date_done.strftime('%Y-%m-%dT%H:%M:%SZ'), - 'traceback': self.decode(str(traceback)), - 'children': self.decode(str(children)), + 'traceback': self.decode(traceback), + 'children': self.decode(children), }) def __reduce__(self, args=(), kwargs={}): diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 94cc0b3a7..6e8f58463 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -1,7 +1,8 @@ from __future__ import absolute_import - from pickle import loads, dumps from datetime import datetime + +import six from celery import states from celery.exceptions import ImproperlyConfigured from celery.tests.case import ( @@ -71,7 +72,8 @@ def test_get_task_meta_for(self): session = x._session = Mock() execute = session.execute = Mock() execute.return_value = [ - [states.SUCCESS, '1', datetime.now(), '', None] + [states.SUCCESS, '1', datetime.now(), six.binary_type(''), + six.binary_type('')] ] x.decode = Mock() meta = x._get_task_meta_for('task_id') From de6288b47e4db605a547b063ef5ec147f1b091fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 19:36:01 +0200 Subject: [PATCH 0612/1103] ditto --- celery/tests/backends/test_new_cassandra.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 6e8f58463..ede4fb944 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -72,8 +72,7 @@ def test_get_task_meta_for(self): session = x._session = Mock() execute = session.execute = Mock() execute.return_value = [ - [states.SUCCESS, '1', datetime.now(), six.binary_type(''), - six.binary_type('')] + [states.SUCCESS, '1', datetime.now(), b'', b''] ] x.decode = Mock() meta = x._get_task_meta_for('task_id') From 88cf262f16907e531d045056f8013570f873ca8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 20:04:40 +0200 Subject: [PATCH 0613/1103] mutable class level objects evicted --- celery/backends/new_cassandra.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index 80cc20a80..80f308c4c 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -36,9 +36,6 @@ class NewCassandraBackend(BaseBackend): module :mod:`cassandra` is not available. """ - servers = [] - keyspace = None - table = None supports_autoexpire = True # autoexpire supported via entry_ttl def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, @@ -58,16 +55,13 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, conf = self.app.conf self.servers = (servers or - conf.get('CASSANDRA_SERVERS') or - self.servers) + conf.get('CASSANDRA_SERVERS')) self.port = (port or conf.get('CASSANDRA_PORT')) self.keyspace = (keyspace or - conf.get('CASSANDRA_KEYSPACE') or - self.keyspace) + conf.get('CASSANDRA_KEYSPACE')) self.table = (table or - conf.get('CASSANDRA_TABLE') or - self.table) + conf.get('CASSANDRA_TABLE')) expires = (entry_ttl or conf.get('CASSANDRA_ENTRY_TTL', None)) if expires is not None: From 4a22220269732eb699963c1391c71ac966cbfbe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Mon, 24 Aug 2015 20:19:47 +0200 Subject: [PATCH 0614/1103] code review fixes %s are here to stay - I'll need them later for Cassandra queries. I have no idea how to use celery.five to detect Python version. --- celery/backends/new_cassandra.py | 37 +++++++++------------ celery/tests/backends/test_new_cassandra.py | 1 - 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index 80f308c4c..ce7775461 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -9,7 +9,6 @@ from __future__ import absolute_import import sys -import six try: # pragma: no cover import cassandra except ImportError: # pragma: no cover @@ -55,13 +54,17 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, conf = self.app.conf self.servers = (servers or - conf.get('CASSANDRA_SERVERS')) + conf.get('CASSANDRA_SERVERS', None)) self.port = (port or - conf.get('CASSANDRA_PORT')) + conf.get('CASSANDRA_PORT', None)) self.keyspace = (keyspace or - conf.get('CASSANDRA_KEYSPACE')) + conf.get('CASSANDRA_KEYSPACE', None)) self.table = (table or - conf.get('CASSANDRA_TABLE')) + conf.get('CASSANDRA_TABLE', None)) + + if not self.servers or not self.keyspace or not self.table: + raise ImproperlyConfigured('Cassandra backend not configured.') + expires = (entry_ttl or conf.get('CASSANDRA_ENTRY_TTL', None)) if expires is not None: @@ -71,20 +74,11 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, read_cons = conf.get('CASSANDRA_READ_CONSISTENCY') or 'LOCAL_QUORUM' write_cons = conf.get('CASSANDRA_WRITE_CONSISTENCY') or 'LOCAL_QUORUM' - try: - self.read_consistency = getattr(cassandra.ConsistencyLevel, - read_cons) - except AttributeError: - self.read_consistency = cassandra.ConsistencyLevel.LOCAL_QUORUM - try: - self.write_consistency = getattr(cassandra.ConsistencyLevel, - write_cons) - except AttributeError: - self.write_consistency = cassandra.ConsistencyLevel.LOCAL_QUORUM - if not self.servers or not self.keyspace or not self.table: - raise ImproperlyConfigured( - 'Cassandra backend not configured.') + self.read_consistency = getattr(cassandra.ConsistencyLevel, + read_cons, cassandra.ConsistencyLevel.LOCAL_QUORUM) + self.write_consistency = getattr(cassandra.ConsistencyLevel, + write_cons, cassandra.ConsistencyLevel.LOCAL_QUORUM) self._connection = None self._session = None @@ -108,8 +102,10 @@ def _get_connection(self, write=False): port=self.port) self._session = self._connection.connect(self.keyspace) + # We are forced to do concatenation below, as formatting would + # blow up on superficial %s that will be processed by Cassandra self._write_stmt = cassandra.query.SimpleStatement( - 'INSERT INTO '+self.table+' (task_id, status, result,''' + 'INSERT INTO %s (task_id, status, result,''' ''' date_done, traceback, children) VALUES''' ' (%s, %s, %s, %s, %s, %s) '+self.cqlexpires+';') self._write_stmt.consistency_level = self.write_consistency @@ -151,8 +147,7 @@ def _store_result(self, task_id, result, status, """Store return value and status of an executed task.""" self._get_connection(write=True) - import sys - if six.PY3: + if sys.version_info >= (3,): buf = lambda x: bytes(x, 'utf8') else: buf = buffer diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index ede4fb944..17c0ace85 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -2,7 +2,6 @@ from pickle import loads, dumps from datetime import datetime -import six from celery import states from celery.exceptions import ImproperlyConfigured from celery.tests.case import ( From 89a121158918f4053a8d6a5f10cfbe1c4598eb9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Tue, 25 Aug 2015 08:51:14 +0200 Subject: [PATCH 0615/1103] overzealous code fix removed --- celery/backends/new_cassandra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index ce7775461..3c530f022 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -105,7 +105,7 @@ def _get_connection(self, write=False): # We are forced to do concatenation below, as formatting would # blow up on superficial %s that will be processed by Cassandra self._write_stmt = cassandra.query.SimpleStatement( - 'INSERT INTO %s (task_id, status, result,''' + 'INSERT INTO '+self.table+''' (task_id, status, result,''' ''' date_done, traceback, children) VALUES''' ' (%s, %s, %s, %s, %s, %s) '+self.cqlexpires+';') self._write_stmt.consistency_level = self.write_consistency From 70ad651d840ef18da8f9575ed4f691a2ed13f071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Tue, 25 Aug 2015 08:57:59 +0200 Subject: [PATCH 0616/1103] lol, CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index ac5541ef4..29c608eb7 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -190,3 +190,4 @@ Alexander Lebedev, 2015/04/25 Frantisek Holop, 2015/05/21 Feanil Patel, 2015/05/21 Juan Rossi, 2015/08/10 +Piotr Maślanka, 2015/08/24 \ No newline at end of file From ed054b147520394e632eff1a41252aa66d794d78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Tue, 25 Aug 2015 09:02:04 +0200 Subject: [PATCH 0617/1103] doc coherence --- docs/configuration.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 735234148..b6dd3bd4b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -637,7 +637,6 @@ Cassandra backend settings $ pip install pycassa - This backend requires the following configuration directives to be set. .. setting:: CASSANDRA_SERVERS @@ -649,10 +648,6 @@ List of ``host:port`` Cassandra servers. e.g.:: CASSANDRA_SERVERS = ['localhost:9160'] -Omit the ``port`` part when using new_cassandra. e.g.:: - - CASSANDRA_SERVERS = ['localhost'] - .. setting:: CASSANDRA_KEYSPACE CASSANDRA_KEYSPACE @@ -699,8 +694,6 @@ use the ``TimeUUID`` type as a comparator:: create column family task_results with comparator = TimeUUIDType; -new_cassandra uses detailed mode by default, and that cannot be disabled. - CASSANDRA_OPTIONS ~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 72253cfefbef08cc67e25cb2d6ab8666db9f80c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Tue, 25 Aug 2015 09:48:19 +0200 Subject: [PATCH 0618/1103] contrib --- CONTRIBUTORS.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index bb35be00a..29c608eb7 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -189,9 +189,5 @@ James Pulec, 2015/04/19 Alexander Lebedev, 2015/04/25 Frantisek Holop, 2015/05/21 Feanil Patel, 2015/05/21 -<<<<<<< HEAD Juan Rossi, 2015/08/10 -Piotr Maślanka, 2015/08/24 -======= -Piotr Maślanka, 2015/08/24 ->>>>>>> origin/new-cassandra-backend +Piotr Maślanka, 2015/08/24 \ No newline at end of file From 5d98164335838b4327387b5b6dcc89c32018ef8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Tue, 25 Aug 2015 10:31:44 +0200 Subject: [PATCH 0619/1103] whats-new updated --- docs/whatsnew-3.2.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/whatsnew-3.2.rst b/docs/whatsnew-3.2.rst index c7effaef1..df39c186f 100644 --- a/docs/whatsnew-3.2.rst +++ b/docs/whatsnew-3.2.rst @@ -99,6 +99,11 @@ Bla bla - blah blah +New Cassandra Backend +===================== +New Cassandra backend will be called new_cassandra and utilize python-driver. +Old backend is now deprecated. + Event Batching ============== From b4daa0fd5c301fcf8d25bdb7469dd6ee7c4a59f6 Mon Sep 17 00:00:00 2001 From: jerry Date: Thu, 27 Aug 2015 10:52:43 -0400 Subject: [PATCH 0620/1103] Adding documentation for multiple broker URLs #2749 --- docs/configuration.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/configuration.rst b/docs/configuration.rst index 49ca75877..25c8aa2ad 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1165,6 +1165,20 @@ default is ``amqp``, which uses ``librabbitmq`` by default or falls back to ``couchdb``. It can also be a fully qualified path to your own transport implementation. +More than broker URL, of the same transport, can also be specified. +The broker URLs can be passed in as a single string that is semicolon delimited:: + + BROKER_URL = 'transport://userid:password@hostname:port//;transport://userid:password@hostname:port//' + +Or as a list:: + + BROKER_URL = [ + 'transport://userid:password@localhost:port//', + 'transport://userid:password@hostname:port//' + ] + +The brokers will then be used in the :setting:`BROKER_FAILOVER_STRATEGY`. + See :ref:`kombu:connection-urls` in the Kombu documentation for more information. From 27866e7bd0a3d86cb55ae9c7cad2137233442353 Mon Sep 17 00:00:00 2001 From: Josh Kupershmidt Date: Thu, 27 Aug 2015 21:11:23 -0400 Subject: [PATCH 0621/1103] Fix for example code demonstrating celery events cam. --- docs/userguide/monitoring.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index 2618ab897..3ba1fe5ea 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -511,7 +511,7 @@ Here is an example camera, dumping the snapshot to screen: return print('Workers: {0}'.format(pformat(state.workers, indent=4))) print('Tasks: {0}'.format(pformat(state.tasks, indent=4))) - print('Total: {0.event_count} events, %s {0.task_count}'.format( + print('Total: {0.event_count} events, {0.task_count} tasks'.format( state)) See the API reference for :mod:`celery.events.state` to read more From 10f4302cbe4d20b5fe68b1aa0064ab05188f7ba2 Mon Sep 17 00:00:00 2001 From: JocelynDelalande Date: Wed, 3 Jun 2015 18:31:47 +0200 Subject: [PATCH 0622/1103] Missing exceptions in celery.exceptions.__all__ I see no reason why those two are not present. --- celery/exceptions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/exceptions.py b/celery/exceptions.py index 96f1bda13..39e764918 100644 --- a/celery/exceptions.py +++ b/celery/exceptions.py @@ -21,12 +21,12 @@ 'SecurityError', 'Ignore', 'QueueNotFound', 'WorkerShutdown', 'WorkerTerminate', 'ImproperlyConfigured', 'NotRegistered', 'AlreadyRegistered', - 'TimeoutError', 'MaxRetriesExceededError', 'Retry', + 'TimeoutError', 'MaxRetriesExceededError', 'Retry', 'Reject', 'TaskRevokedError', 'NotConfigured', 'AlwaysEagerIgnored', 'InvalidTaskError', 'ChordError', 'CPendingDeprecationWarning', 'CDeprecationWarning', 'FixupWarning', 'DuplicateNodenameWarning', 'SoftTimeLimitExceeded', 'TimeLimitExceeded', 'WorkerLostError', - 'Terminated', + 'Terminated', 'IncompleteStream' ] UNREGISTERED_FMT = """\ From e436454d02dcbba4f4410868ad109c54047c2c15 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Sep 2015 12:01:21 -0700 Subject: [PATCH 0623/1103] Django autodiscovery no longer requires an argument to work. If `app.autodiscover_tasks()` is called without a packages argument, the Django fixup will now take the list of modules from the app config registry. Closes #2596 --- celery/app/base.py | 24 +++++++++++++++++++----- celery/fixups/django.py | 9 +++++++++ docs/django/first-steps-with-django.rst | 7 ++++--- examples/django/proj/celery.py | 6 ++++-- 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 21717a975..8912685fb 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -348,17 +348,31 @@ def setup_security(self, allowed_serializers=None, key=None, cert=None, return setup_security(allowed_serializers, key, cert, store, digest, serializer, app=self) - def autodiscover_tasks(self, packages, related_name='tasks', force=False): + def autodiscover_tasks(self, packages=None, + related_name='tasks', force=False): if force: return self._autodiscover_tasks(packages, related_name) signals.import_modules.connect(promise( self._autodiscover_tasks, (packages, related_name), ), weak=False, sender=self) - def _autodiscover_tasks(self, packages, related_name='tasks', **kwargs): - # argument may be lazy - packages = packages() if callable(packages) else packages - self.loader.autodiscover_tasks(packages, related_name) + def _autodiscover_tasks(self, packages, related_name, **kwargs): + if packages: + return self._autodiscover_tasks_from_names(packages, related_name) + return self._autodiscover_tasks_from_fixups(related_name) + + def _autodiscover_tasks_from_names(self, packages, related_name): + # packages argument can be lazy + return self.loader.autodiscover_tasks( + packages() if callable(packages) else packages, related_name, + ) + + def _autodiscover_tasks_from_fixups(self, related_name): + return self._autodiscover_tasks_from_names([ + pkg for fixup in self._fixups + for pkg in fixup.autodiscover_tasks() + if hasattr(fixup, 'autodiscover_tasks') + ], related_name=related_name) def send_task(self, name, args=None, kwargs=None, countdown=None, eta=None, task_id=None, producer=None, connection=None, diff --git a/celery/fixups/django.py b/celery/fixups/django.py index 60b836290..d67eb3d55 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -57,6 +57,7 @@ def install(self): # Need to add project directory to path sys.path.append(os.getcwd()) + self._settings = symbol_by_name('django.conf:settings') self.app.loader.now = self.now self.app.loader.mail_admins = self.mail_admins @@ -83,6 +84,14 @@ def now(self, utc=False): def mail_admins(self, subject, body, fail_silently=False, **kwargs): return self._mail_admins(subject, body, fail_silently=fail_silently) + def autodiscover_tasks(self): + try: + from django.apps import apps + except ImportError: + return self._settings.INSTALLED_APPS + else: + return [config.name for config in apps.get_app_configs()] + @cached_property def _mail_admins(self): return symbol_by_name('django.core.mail:mail_admins') diff --git a/docs/django/first-steps-with-django.rst b/docs/django/first-steps-with-django.rst index ed259cd40..10879bc45 100644 --- a/docs/django/first-steps-with-django.rst +++ b/docs/django/first-steps-with-django.rst @@ -92,10 +92,10 @@ autodiscover these modules: .. code-block:: python - app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) + app.autodiscover_tasks() -With the line above Celery will automatically discover tasks in reusable -apps if you follow the ``tasks.py`` convention:: +With the line above Celery will automatically discover tasks from all +of your installed apps, following the ``tasks.py`` convention:: - app1/ - tasks.py @@ -104,6 +104,7 @@ apps if you follow the ``tasks.py`` convention:: - tasks.py - models.py + This way you do not have to manually add the individual modules to the :setting:`CELERY_IMPORTS` setting. The ``lambda`` so that the autodiscovery can happen only when needed, and so that importing your diff --git a/examples/django/proj/celery.py b/examples/django/proj/celery.py index aebb10850..a2eeb7444 100644 --- a/examples/django/proj/celery.py +++ b/examples/django/proj/celery.py @@ -4,7 +4,7 @@ from celery import Celery -from django.conf import settings +from django.apps import apps as django_apps # set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings') @@ -14,7 +14,9 @@ # Using a string here means the worker will not have to # pickle the object when using Windows. app.config_from_object('django.conf:settings') -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) + +# load task modules from all registered Django app configs. +app.autodiscover_tasks() @app.task(bind=True) From 9def9bdab1759c1bcfd800a0d5429e385a8f66c0 Mon Sep 17 00:00:00 2001 From: Zhaorong Ma Date: Tue, 8 Sep 2015 14:21:40 -0400 Subject: [PATCH 0624/1103] Fix ImportError With 'kombu.transport.django.KombuAppConfig' in INSTALLED_APPS, running any manage.py command throws: ImportError: No module named KombuAppConfig It is fixed by changing 'kombu.transport.django.KombuAppConfig' to 'kombu.transport.django' --- examples/django/proj/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/django/proj/settings.py b/examples/django/proj/settings.py index aa7fb38d3..9a6a7e8de 100644 --- a/examples/django/proj/settings.py +++ b/examples/django/proj/settings.py @@ -132,7 +132,7 @@ 'django.contrib.messages', 'django.contrib.staticfiles', 'django.contrib.admin', - 'kombu.transport.django.KombuAppConfig', + 'kombu.transport.django', 'demoapp', # Uncomment the next line to enable the admin: # 'django.contrib.admin', From 71947024c6f79ac1a3c13176fd2350eb298a0cde Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 8 Sep 2015 11:27:39 -0700 Subject: [PATCH 0625/1103] flakes --- celery/app/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 8912685fb..c1d80f189 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -370,8 +370,8 @@ def _autodiscover_tasks_from_names(self, packages, related_name): def _autodiscover_tasks_from_fixups(self, related_name): return self._autodiscover_tasks_from_names([ pkg for fixup in self._fixups - for pkg in fixup.autodiscover_tasks() - if hasattr(fixup, 'autodiscover_tasks') + for pkg in fixup.autodiscover_tasks() + if hasattr(fixup, 'autodiscover_tasks') ], related_name=related_name) def send_task(self, name, args=None, kwargs=None, countdown=None, From ea6c1925ab1be5e6127e17f1565754f20fafb2bd Mon Sep 17 00:00:00 2001 From: armo Date: Wed, 9 Sep 2015 03:12:17 +0300 Subject: [PATCH 0626/1103] Update tasks.rst It seems *url* argument missed. --- docs/userguide/tasks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index fe40668ac..c8bc7e849 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -1349,7 +1349,7 @@ Make your design asynchronous instead, for example by using *callbacks*. def update_page_info(url): # fetch_page -> parse_page -> store_page - chain = fetch_page.s() | parse_page.s() | store_page_info.s(url) + chain = fetch_page.s(url) | parse_page.s() | store_page_info.s(url) chain() @app.task() From 3e3d03f83d34310344216af7e44f74ad82e557f6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 9 Sep 2015 11:55:22 -0700 Subject: [PATCH 0627/1103] Fixes missing pre block in configuration docs --- docs/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 49ca75877..9e38f7fb8 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -269,7 +269,7 @@ prefix: CELERY_RESULT_BACKEND = 'db+scheme://user:password@host:port/dbname' -Examples: +Examples:: # sqlite (filename) CELERY_RESULT_BACKEND = 'db+sqlite:///results.sqlite' From ba9ab3410014585e6b84066e8fe07af70022cbba Mon Sep 17 00:00:00 2001 From: Sean Wang Date: Wed, 16 Sep 2015 11:07:23 -0700 Subject: [PATCH 0628/1103] Removed an extra on_retry entry. --- docs/userguide/tasks.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index c8bc7e849..278acdefc 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -1214,9 +1214,6 @@ Handlers The return value of this handler is ignored. -on_retry -~~~~~~~~ - .. _task-how-they-work: How it works From 726cf9b0a0738fbe7b07f6754ec4b78dece9d10a Mon Sep 17 00:00:00 2001 From: George Tantiras Date: Fri, 18 Sep 2015 11:27:45 +0300 Subject: [PATCH 0629/1103] Set celery priority in supervisord higher From supervisord [docs](http://supervisord.org/configuration.html): > Higher priorities indicate programs that start last and shut down first. --- extra/supervisord/celeryd.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extra/supervisord/celeryd.conf b/extra/supervisord/celeryd.conf index eaf59869d..0747ff836 100644 --- a/extra/supervisord/celeryd.conf +++ b/extra/supervisord/celeryd.conf @@ -29,6 +29,6 @@ stopwaitsecs = 600 ; taking care of its children as well. killasgroup=true -; if rabbitmq is supervised, set its priority higher -; so it starts first -priority=998 +; Set Celery priority higher than default (999) +; so, if rabbitmq is supervised, it will start first. +priority=1000 From 962e58905cea7cdfa37aee8c2c62250289e5b345 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 22 Sep 2015 12:26:48 -0700 Subject: [PATCH 0630/1103] Attempt to fix pypy tests --- requirements/test-ci.txt | 5 ----- tox.ini | 8 ++++++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/requirements/test-ci.txt b/requirements/test-ci.txt index 8385252ae..e71d640f2 100644 --- a/requirements/test-ci.txt +++ b/requirements/test-ci.txt @@ -1,7 +1,2 @@ coverage>=3.0 coveralls -redis -#riak >=2.0 -#pymongo -#SQLAlchemy -PyOpenSSL diff --git a/tox.ini b/tox.ini index c8c6851eb..f852ec136 100644 --- a/tox.ini +++ b/tox.ini @@ -15,6 +15,8 @@ basepython = python3.4 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt -r{toxinidir}/requirements/test-ci.txt + -r{toxinidir}/requirements/extras/auth.txt + -r{toxinidir}/requirements/extras/redis.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -U -r{toxinidir}/requirements/dev.txt @@ -25,6 +27,8 @@ basepython = python3.3 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt -r{toxinidir}/requirements/test-ci.txt + -r{toxinidir}/requirements/extras/auth.txt + -r{toxinidir}/requirements/extras/redis.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -U -r{toxinidir}/requirements/dev.txt @@ -35,6 +39,8 @@ basepython = python2.7 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/test-ci.txt + -r{toxinidir}/requirements/extras/auth.txt + -r{toxinidir}/requirements/extras/redis.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -U -r{toxinidir}/requirements/dev.txt @@ -46,6 +52,7 @@ deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/test-ci.txt -r{toxinidir}/requirements/dev.txt + -r{toxinidir}/requirements/extras/redis.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -q -U -r{toxinidir}/requirements/dev.txt @@ -58,6 +65,7 @@ deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test-pypy3.txt -r{toxinidir}/requirements/test-ci.txt -r{toxinidir}/requirements/dev.txt + -r{toxinidir}/requirements/extras/redis.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -q -U -r{toxinidir}/requirements/dev.txt From c1e1b586dbc9a9475627df510fc9289152851d18 Mon Sep 17 00:00:00 2001 From: Jocelyn Delalande Date: Mon, 14 Sep 2015 20:48:10 +0200 Subject: [PATCH 0631/1103] Detailed more on BROKER_USE_SSL, especially the dict-form --- CONTRIBUTORS.txt | 1 + docs/configuration.rst | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index ac5541ef4..3c15e7246 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -189,4 +189,5 @@ James Pulec, 2015/04/19 Alexander Lebedev, 2015/04/25 Frantisek Holop, 2015/05/21 Feanil Patel, 2015/05/21 +Jocelyn Delalande, 2015/06/03 Juan Rossi, 2015/08/10 diff --git a/docs/configuration.rst b/docs/configuration.rst index 9e38f7fb8..73b38a5ab 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1203,9 +1203,40 @@ will be performed every 5 seconds (twice the heartbeat sending rate). BROKER_USE_SSL ~~~~~~~~~~~~~~ +:transports supported: ``pyamqp`` + + +Toggles SSL usage on broker connection and SSL settings. + +If ``True`` the connection will use SSL with default SSL settings. +If set to a dict, will configure SSL connection according to the specified +policy. The format used is python `ssl.wrap_socket() +options `_. + +Default is ``False`` (no SSL). + +Note that SSL socket is generally served on a separate port by the broker. + +Example providing a client cert and validating the server cert against a custom +certificate authority: + +.. code-block:: python + + import ssl + + BROKER_USE_SSL = { + 'keyfile': '/var/ssl/private/worker-key.pem', + 'certfile': '/var/ssl/amqp-server-cert.pem', + 'ca_certs': '/var/ssl/myca.pem', + 'cert_reqs': ssl.CERT_REQUIRED + } + +.. warning:: -Use SSL to connect to the broker. Off by default. This may not be supported -by all transports. + Be careful using ``BROKER_USE_SSL=True``, it is possible that your default + configuration do not validate the server cert at all, please read Python + `ssl module security + considerations `_. .. setting:: BROKER_POOL_LIMIT From f51b4bda5df8ef8a1510f5997109d18601803d09 Mon Sep 17 00:00:00 2001 From: Jocelyn Delalande Date: Mon, 14 Sep 2015 20:50:21 +0200 Subject: [PATCH 0632/1103] Mentions BROKER_USE_SSL in security guide. --- docs/userguide/security.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/userguide/security.rst b/docs/userguide/security.rst index ef3cd9635..4ccdb9d8c 100644 --- a/docs/userguide/security.rst +++ b/docs/userguide/security.rst @@ -46,6 +46,9 @@ If your broker supports fine-grained access control, like RabbitMQ, this is something you should look at enabling. See for example http://www.rabbitmq.com/access-control.html. +If supported by your broker backend, you can enable end-to-end SSL encryption +and authentication using :setting:`BROKER_USE_SSL`. + Client ------ From b3e5ebe6e7d07d0a5bf21e4ae7996fbeff81e183 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 22 Sep 2015 13:16:26 -0700 Subject: [PATCH 0633/1103] Tests: Use number of threads at startup to verify that tests join threads --- celery/tests/case.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/celery/tests/case.py b/celery/tests/case.py index accc6a1f2..8789692b6 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -303,6 +303,10 @@ def __exit__(self, exc_type, exc_value, tb): raise self.failureException('%s not triggered' % exc_name) +def alive_threads(): + return [thread for thread in threading.enumerate() if thread.is_alive()] + + class Case(unittest.TestCase): def assertWarns(self, expected_warning): @@ -391,6 +395,7 @@ def __inner(self, *args, **kwargs): class AppCase(Case): contained = True + _threads_at_startup = [None] def __init__(self, *args, **kwargs): super(AppCase, self).__init__(*args, **kwargs) @@ -406,8 +411,13 @@ def __init__(self, *args, **kwargs): def Celery(self, *args, **kwargs): return UnitApp(*args, **kwargs) + def threads_at_startup(self): + if self._threads_at_startup[0] is None: + self._threads_at_startup[0] = alive_threads() + return self._threads_at_startup[0] + def setUp(self): - self._threads_at_setup = list(threading.enumerate()) + self._threads_at_setup = self.threads_at_startup() from celery import _state from celery import result result.task_join_will_block = \ @@ -463,9 +473,7 @@ def _teardown_app(self): if self.app is not self._current_app: self.app.close() self.app = None - self.assertEqual( - self._threads_at_setup, list(threading.enumerate()), - ) + self.assertEqual(self._threads_at_setup, alive_threads()) # Make sure no test left the shutdown flags enabled. from celery.worker import state as worker_state From 26541b6c5d41f28660adc309f45c41985215bca6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 22 Sep 2015 13:59:00 -0700 Subject: [PATCH 0634/1103] Bump tox version (fix always recreate bug) --- requirements/pkgutils.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/pkgutils.txt b/requirements/pkgutils.txt index 35cd96010..9156799f2 100644 --- a/requirements/pkgutils.txt +++ b/requirements/pkgutils.txt @@ -2,5 +2,5 @@ setuptools>=1.3.2 wheel flake8 flakeplus -tox +tox>=2.1.1 Sphinx-PyPI-upload From b1e628eab4bfe41bcb3109fc008bb40430e13771 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 22 Sep 2015 13:59:21 -0700 Subject: [PATCH 0635/1103] Fixes tests --- celery/tests/case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/tests/case.py b/celery/tests/case.py index 8789692b6..432d206b9 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -696,7 +696,7 @@ def replace_module_value(module, name, value=None): yield finally: if prev is not None: - setattr(sys, name, prev) + setattr(module, name, prev) if not has_prev: try: delattr(module, name) From d96abc2ae1e5c33f2380f257aac933f18cda9694 Mon Sep 17 00:00:00 2001 From: Pavel Savchenko Date: Wed, 23 Sep 2015 11:01:25 +0200 Subject: [PATCH 0636/1103] Clearly explain how to use memory cache backend Admittedly I am slow, but it shouldn't have taken me 10 minutes to figure out how to set up cache/memory (very useful for `unittest`s). --- docs/configuration.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/configuration.rst b/docs/configuration.rst index 73b38a5ab..094390ec5 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -396,6 +396,9 @@ Using multiple memcached servers: The "memory" backend stores the cache in memory only: +.. code-block:: python + + CELERY_RESULT_BACKEND = 'cache' CELERY_CACHE_BACKEND = 'memory' CELERY_CACHE_BACKEND_OPTIONS From 0308ce626bad5f7e05e0dea446abd060da7281bc Mon Sep 17 00:00:00 2001 From: Bryan Helmig Date: Thu, 24 Sep 2015 15:25:48 -0700 Subject: [PATCH 0637/1103] explicitly drain events when gossip/heartbeat will not - fix for #1847 kudos for @sabw8217 who's fix i copied verbatim. --- celery/tests/worker/test_loops.py | 4 ++++ celery/worker/loops.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/celery/tests/worker/test_loops.py b/celery/tests/worker/test_loops.py index aa92f66d1..496cffc60 100644 --- a/celery/tests/worker/test_loops.py +++ b/celery/tests/worker/test_loops.py @@ -120,6 +120,10 @@ def add(x, y): return x + y self.add = add + def test_drain_after_consume(self): + x, _ = get_task_callback(self.app) + x.connection.drain_events.assert_called_with() + def test_setup_heartbeat(self): x = X(self.app, heartbeat=10) x.hub.call_repeatedly = Mock(name='x.hub.call_repeatedly()') diff --git a/celery/worker/loops.py b/celery/worker/loops.py index adfd99d04..2605fda6c 100644 --- a/celery/worker/loops.py +++ b/celery/worker/loops.py @@ -47,6 +47,11 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos, if not obj.restart_count and not obj.pool.did_start_ok(): raise WorkerLostError('Could not start worker processes') + # consumer.consume() may have prefetched up to our + # limit - drain an event so we are in a clean state + # prior to starting our event loop. + connection.drain_events() + # FIXME: Use loop.run_forever # Tried and works, but no time to test properly before release. hub.propagate_errors = errors From 5b5e48ab8ee01627d84506496e5a745e6f6dcc42 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 12:37:39 -0700 Subject: [PATCH 0638/1103] Trying to fix Readthedocs build --- requirements/default.txt | 4 ++-- requirements/docs.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/default.txt b/requirements/default.txt index da64babcf..e2bc340a2 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,3 +1,3 @@ pytz>dev -billiard>=3.3.0.17,<3.4 -kombu>=3.0.15,<4.0 +billiard>dev +kombu>dev diff --git a/requirements/docs.txt b/requirements/docs.txt index e9da93cb3..5d5649214 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,5 +1,5 @@ -billiard Sphinx SQLAlchemy +https://github.com/celery/billiard/zipball/master https://github.com/celery/py-amqp/zipball/master https://github.com/celery/kombu/zipball/master From 62383df29b99e4d375bb7fe79b195f6b23880be9 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 13:16:37 -0700 Subject: [PATCH 0639/1103] [Py3] Fixes docs build on py3 --- celery/app/task.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/celery/app/task.py b/celery/app/task.py index f2fe11fae..e0779da11 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -230,6 +230,9 @@ class Task(object): #: Default task expiry time. expires = None + #: Task request stack, the current request will be the topmost. + request_stack = None + #: Some may expect a request to exist even if the task has not been #: called. This should probably be deprecated. _default_request = None @@ -466,7 +469,7 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, except AttributeError: pass else: - check_arguments(*args or (), **kwargs or {}) + check_arguments(*(args or ()), **(kwargs or {})) app = self._get_app() if app.conf.CELERY_ALWAYS_EAGER: From d4d37b002c4195d0bb7a63e55575ccff31568fac Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 13:51:50 -0700 Subject: [PATCH 0640/1103] Cleanup requirements --- requirements/deps/mock.txt | 1 + requirements/deps/nose.txt | 1 + requirements/docs.txt | 6 ++---- requirements/extras/beanstalk.rst | 1 - requirements/jython.txt | 2 +- requirements/security.txt | 2 +- requirements/test-ci.txt | 1 + requirements/test-pypy3.txt | 2 +- requirements/test.txt | 4 ++-- requirements/test3.txt | 2 +- setup.py | 17 +++++++++++++++-- 11 files changed, 26 insertions(+), 13 deletions(-) create mode 100644 requirements/deps/mock.txt create mode 100644 requirements/deps/nose.txt delete mode 100644 requirements/extras/beanstalk.rst diff --git a/requirements/deps/mock.txt b/requirements/deps/mock.txt new file mode 100644 index 000000000..fc5a38307 --- /dev/null +++ b/requirements/deps/mock.txt @@ -0,0 +1 @@ +mock>=1.3 diff --git a/requirements/deps/nose.txt b/requirements/deps/nose.txt new file mode 100644 index 000000000..7331c33c1 --- /dev/null +++ b/requirements/deps/nose.txt @@ -0,0 +1 @@ +nose>=1.3.7 diff --git a/requirements/docs.txt b/requirements/docs.txt index 5d5649214..a1f3b72de 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,5 +1,3 @@ Sphinx -SQLAlchemy -https://github.com/celery/billiard/zipball/master -https://github.com/celery/py-amqp/zipball/master -https://github.com/celery/kombu/zipball/master +-r requirements/extras/sqlalchemy.txt +-r requirements/dev.txt diff --git a/requirements/extras/beanstalk.rst b/requirements/extras/beanstalk.rst deleted file mode 100644 index c62c81bd2..000000000 --- a/requirements/extras/beanstalk.rst +++ /dev/null @@ -1 +0,0 @@ -beanstalkc diff --git a/requirements/jython.txt b/requirements/jython.txt index 4427a9a5f..16a2ad15f 100644 --- a/requirements/jython.txt +++ b/requirements/jython.txt @@ -1,2 +1,2 @@ -threadpool multiprocessing +-r extras/threads.txt diff --git a/requirements/security.txt b/requirements/security.txt index 9292484f9..9ae559b69 100644 --- a/requirements/security.txt +++ b/requirements/security.txt @@ -1 +1 @@ -PyOpenSSL +-r extras/auth.txt diff --git a/requirements/test-ci.txt b/requirements/test-ci.txt index e71d640f2..0814d86ba 100644 --- a/requirements/test-ci.txt +++ b/requirements/test-ci.txt @@ -1,2 +1,3 @@ coverage>=3.0 coveralls +-r extras/redis.txt diff --git a/requirements/test-pypy3.txt b/requirements/test-pypy3.txt index 932a8957f..dc9901d75 100644 --- a/requirements/test-pypy3.txt +++ b/requirements/test-pypy3.txt @@ -1 +1 @@ -mock +-r deps/mock.txt diff --git a/requirements/test.txt b/requirements/test.txt index 0d0b3c697..8ad2e9a3c 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,3 +1,3 @@ +-r deps/mock.txt +-r deps/nose.txt unittest2>=0.5.1 -nose -mock>=1.0.1 diff --git a/requirements/test3.txt b/requirements/test3.txt index f3c7e8e6f..881384714 100644 --- a/requirements/test3.txt +++ b/requirements/test3.txt @@ -1 +1 @@ -nose +-r deps/nose.txt diff --git a/setup.py b/setup.py index 136318076..9a86098ca 100644 --- a/setup.py +++ b/setup.py @@ -116,13 +116,24 @@ def strip_comments(l): return l.split('#', 1)[0].strip() -def reqs(*f): +def _pip_requirement(req): + if req.startswith('-r '): + _, path = req.split() + return reqs(*path.split('/')) + return [req] + + +def _reqs(*f): return [ - r for r in ( + _pip_requirement(r) for r in ( strip_comments(l) for l in open( os.path.join(os.getcwd(), 'requirements', *f)).readlines() ) if r] + +def reqs(*f): + return [req for subreq in _reqs(*f) for req in subreq] + install_requires = reqs('default.txt') if JYTHON: install_requires.extend(reqs('jython.txt')) @@ -164,6 +175,8 @@ def reqs(*f): extras_require = {x: extras(x + '.txt') for x in features} extra['extras_require'] = extras_require +print(tests_require) + # -*- %%% -*- setup( From ef1abb218d9b9957f7c25f253e43e2e7f9f39491 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 13:53:50 -0700 Subject: [PATCH 0641/1103] Fixes typo --- requirements/docs.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/docs.txt b/requirements/docs.txt index a1f3b72de..b0bdf1c0c 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,3 +1,3 @@ Sphinx --r requirements/extras/sqlalchemy.txt --r requirements/dev.txt +-r extras/sqlalchemy.txt +-r dev.txt From 98d9adb3f8a266832091196482acc8e1f2f759b5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 13:55:59 -0700 Subject: [PATCH 0642/1103] [ci] Specified redis dependency twice --- tox.ini | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tox.ini b/tox.ini index f852ec136..1fdf32714 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,6 @@ deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt -r{toxinidir}/requirements/test-ci.txt -r{toxinidir}/requirements/extras/auth.txt - -r{toxinidir}/requirements/extras/redis.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -U -r{toxinidir}/requirements/dev.txt @@ -28,7 +27,6 @@ deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt -r{toxinidir}/requirements/test-ci.txt -r{toxinidir}/requirements/extras/auth.txt - -r{toxinidir}/requirements/extras/redis.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -U -r{toxinidir}/requirements/dev.txt @@ -40,7 +38,6 @@ deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/test-ci.txt -r{toxinidir}/requirements/extras/auth.txt - -r{toxinidir}/requirements/extras/redis.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -U -r{toxinidir}/requirements/dev.txt @@ -52,7 +49,6 @@ deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/test-ci.txt -r{toxinidir}/requirements/dev.txt - -r{toxinidir}/requirements/extras/redis.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -q -U -r{toxinidir}/requirements/dev.txt @@ -65,7 +61,6 @@ deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test-pypy3.txt -r{toxinidir}/requirements/test-ci.txt -r{toxinidir}/requirements/dev.txt - -r{toxinidir}/requirements/extras/redis.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -q -U -r{toxinidir}/requirements/dev.txt From fd15f1d001a54e92f4ee843c32e4a261a0148f31 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 13:58:13 -0700 Subject: [PATCH 0643/1103] CI everything --- requirements/test-ci.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/requirements/test-ci.txt b/requirements/test-ci.txt index 0814d86ba..3a3c87896 100644 --- a/requirements/test-ci.txt +++ b/requirements/test-ci.txt @@ -1,3 +1,10 @@ coverage>=3.0 coveralls -r extras/redis.txt +-r extras/cassandra.txt +-r extras/couchbase.txt +-r extras/couchdb.txt +-r extras/mongodb.txt +-r extras/redis.txt +-r extras/riak.txt +-r extras/sqlalchemy.txt From 97120fc2ed118b547892d965c59c6afc9799a55a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 13:59:34 -0700 Subject: [PATCH 0644/1103] Another attempt at fixing docs build --- requirements/default.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements/default.txt b/requirements/default.txt index e2bc340a2..ce285a811 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,3 +1,6 @@ pytz>dev billiard>dev kombu>dev + +# remove before release +amqp>dev From 621daa74a34a3bb27cdaa3bcee94e8ea60558654 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 14:02:57 -0700 Subject: [PATCH 0645/1103] Oops, redis twice again --- requirements/test-ci.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/test-ci.txt b/requirements/test-ci.txt index 3a3c87896..f6c1699e3 100644 --- a/requirements/test-ci.txt +++ b/requirements/test-ci.txt @@ -5,6 +5,5 @@ coveralls -r extras/couchbase.txt -r extras/couchdb.txt -r extras/mongodb.txt --r extras/redis.txt -r extras/riak.txt -r extras/sqlalchemy.txt From 88e73348657cdd681007e0922cb08eb227327f7d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 14:07:12 -0700 Subject: [PATCH 0646/1103] [CI] Cannot CI couchdb as pycouchdb package is broken: Using cached requests-2.7.0-py2.py3-none-any.whl Collecting pycouchdb (from -r /opt/devel/celery/requirements/extras/couchdb.txt (line 2)) Using cached pycouchdb-1.13.tar.gz Complete output from command python setup.py egg_info: Traceback (most recent call last): File "", line 20, in File "/private/var/folders/t_/8b21_y5j4mdc1r8cslkyr9580000gn/T/pip-build-eSLwrL/pycouchdb/setup.py", line 10, in import pycouchdb File "pycouchdb/__init__.py", line 10, in from .client import Server File "pycouchdb/client.py", line 13, in from .resource import Resource File "pycouchdb/resource.py", line 5, in import requests ImportError: No module named requests --- requirements/test-ci.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/test-ci.txt b/requirements/test-ci.txt index f6c1699e3..a0b25d2aa 100644 --- a/requirements/test-ci.txt +++ b/requirements/test-ci.txt @@ -3,7 +3,6 @@ coveralls -r extras/redis.txt -r extras/cassandra.txt -r extras/couchbase.txt --r extras/couchdb.txt -r extras/mongodb.txt -r extras/riak.txt -r extras/sqlalchemy.txt From 0bcca440c1e36b8e929aa56fab0fd3ac746efd3a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 14:07:52 -0700 Subject: [PATCH 0647/1103] [ci] cannot CI couchbase, requires libcouchbase --- requirements/test-ci.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/test-ci.txt b/requirements/test-ci.txt index a0b25d2aa..8fbbc2564 100644 --- a/requirements/test-ci.txt +++ b/requirements/test-ci.txt @@ -2,7 +2,6 @@ coverage>=3.0 coveralls -r extras/redis.txt -r extras/cassandra.txt --r extras/couchbase.txt -r extras/mongodb.txt -r extras/riak.txt -r extras/sqlalchemy.txt From 85950be50170e02b51a837c62a97c99bfd937729 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 14:15:05 -0700 Subject: [PATCH 0648/1103] [ci] More requirements cleanup --- requirements/{test-ci.txt => test-ci-base.txt} | 1 + requirements/test-ci-default.txt | 2 ++ tox.ini | 15 +++++---------- 3 files changed, 8 insertions(+), 10 deletions(-) rename requirements/{test-ci.txt => test-ci-base.txt} (92%) create mode 100644 requirements/test-ci-default.txt diff --git a/requirements/test-ci.txt b/requirements/test-ci-base.txt similarity index 92% rename from requirements/test-ci.txt rename to requirements/test-ci-base.txt index 8fbbc2564..a115498cc 100644 --- a/requirements/test-ci.txt +++ b/requirements/test-ci-base.txt @@ -5,3 +5,4 @@ coveralls -r extras/mongodb.txt -r extras/riak.txt -r extras/sqlalchemy.txt +-r dev.txt diff --git a/requirements/test-ci-default.txt b/requirements/test-ci-default.txt new file mode 100644 index 000000000..74c02f5fe --- /dev/null +++ b/requirements/test-ci-default.txt @@ -0,0 +1,2 @@ +-r test-ci-base.txt +-r extras/auth.txt diff --git a/tox.ini b/tox.ini index 1fdf32714..d77decf0b 100644 --- a/tox.ini +++ b/tox.ini @@ -14,8 +14,7 @@ commands = nosetests basepython = python3.4 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt - -r{toxinidir}/requirements/test-ci.txt - -r{toxinidir}/requirements/extras/auth.txt + -r{toxinidir}/requirements/test-ci-default.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -U -r{toxinidir}/requirements/dev.txt @@ -25,8 +24,7 @@ commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} basepython = python3.3 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt - -r{toxinidir}/requirements/test-ci.txt - -r{toxinidir}/requirements/extras/auth.txt + -r{toxinidir}/requirements/test-ci-default.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -U -r{toxinidir}/requirements/dev.txt @@ -36,8 +34,7 @@ commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} basepython = python2.7 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt - -r{toxinidir}/requirements/test-ci.txt - -r{toxinidir}/requirements/extras/auth.txt + -r{toxinidir}/requirements/test-ci-default.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -U -r{toxinidir}/requirements/dev.txt @@ -47,8 +44,7 @@ commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} basepython = pypy deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt - -r{toxinidir}/requirements/test-ci.txt - -r{toxinidir}/requirements/dev.txt + -r{toxinidir}/requirements/test-ci-default.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -q -U -r{toxinidir}/requirements/dev.txt @@ -59,8 +55,7 @@ basepython = pypy3 deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test3.txt -r{toxinidir}/requirements/test-pypy3.txt - -r{toxinidir}/requirements/test-ci.txt - -r{toxinidir}/requirements/dev.txt + -r{toxinidir}/requirements/test-ci-base.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -q -U -r{toxinidir}/requirements/dev.txt From d79fb0a4eb6e3f61c703fa5429d33c77261c699c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 16:01:54 -0700 Subject: [PATCH 0649/1103] [CI] pycassa does not work on Python3 --- requirements/test-ci-base.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/test-ci-base.txt b/requirements/test-ci-base.txt index a115498cc..c29ccd8ba 100644 --- a/requirements/test-ci-base.txt +++ b/requirements/test-ci-base.txt @@ -1,7 +1,6 @@ coverage>=3.0 coveralls -r extras/redis.txt --r extras/cassandra.txt -r extras/mongodb.txt -r extras/riak.txt -r extras/sqlalchemy.txt From 511f0085404273d2251053c2af1d55ff292f6df3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 16:02:37 -0700 Subject: [PATCH 0650/1103] [MongoDB] mongo_host must be a list --- celery/backends/mongodb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 2e48fb3df..36cbb5342 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -86,8 +86,9 @@ def __init__(self, app=None, url=None, **kwargs): if self.url: uri_data = pymongo.uri_parser.parse_uri(self.url) # build the hosts list to create a mongo connection - make_host_str = lambda x: "{0}:{1}".format(x[0], x[1]) - hostslist = map(make_host_str, uri_data['nodelist']) + hostslist = [ + "{0}:{1}".format(x[0], x[1]) for x in uri_data['nodelist'] + ] self.user = uri_data['username'] self.password = uri_data['password'] self.mongo_host = hostslist From 540d1f59954b8a95f841368cd330b16902ae9807 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 16:37:39 -0700 Subject: [PATCH 0651/1103] [CI] Fixes PyPy3 build --- celery/backends/riak.py | 19 ++++++++++++++++++- requirements/test-ci-base.txt | 1 - requirements/test-ci-default.txt | 1 + 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/celery/backends/riak.py b/celery/backends/riak.py index f9bc8cf3a..3b7ff90be 100644 --- a/celery/backends/riak.py +++ b/celery/backends/riak.py @@ -8,6 +8,8 @@ """ from __future__ import absolute_import +import sys + try: import riak from riak import RiakClient @@ -25,10 +27,25 @@ Riak bucket names must be composed of ASCII characters only, not: {0!r}\ """ +if sys.version_info[0] == 3: + + def to_bytes(s): + return s.encode() if isinstance(s, str) else s + + + def str_decode(s, encoding): + return to_bytes(s).decode(encoding) + +else: + + def str_decode(s, encoding): + return s.decode("ascii") + + def is_ascii(s): try: - s.decode('ascii') + str_decode(s, 'ascii') except UnicodeDecodeError: return False return True diff --git a/requirements/test-ci-base.txt b/requirements/test-ci-base.txt index c29ccd8ba..aa4a0c6e2 100644 --- a/requirements/test-ci-base.txt +++ b/requirements/test-ci-base.txt @@ -2,6 +2,5 @@ coverage>=3.0 coveralls -r extras/redis.txt -r extras/mongodb.txt --r extras/riak.txt -r extras/sqlalchemy.txt -r dev.txt diff --git a/requirements/test-ci-default.txt b/requirements/test-ci-default.txt index 74c02f5fe..3b354d8ad 100644 --- a/requirements/test-ci-default.txt +++ b/requirements/test-ci-default.txt @@ -1,2 +1,3 @@ -r test-ci-base.txt -r extras/auth.txt +-r extras/riak.txt From 09bb50046e99db6d027a3cf4fe129ececd337737 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 16:45:47 -0700 Subject: [PATCH 0652/1103] [CI] Fixes PyPy build --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d77decf0b..4aedf239b 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,7 @@ commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} basepython = pypy deps = -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt - -r{toxinidir}/requirements/test-ci-default.txt + -r{toxinidir}/requirements/test-ci-base.txt setenv = C_DEBUG_TEST = 1 commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -q -U -r{toxinidir}/requirements/dev.txt From ddfe2bc19145f44336babdb06953ca52ebd8d8ec Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 17:30:28 -0700 Subject: [PATCH 0653/1103] Try some tox stuff --- .travis.yml | 11 ++++---- tox.ini | 72 ++++++++++++++--------------------------------------- 2 files changed, 25 insertions(+), 58 deletions(-) diff --git a/.travis.yml b/.travis.yml index 365248d2c..6b4b6f3ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,8 @@ sudo: false cache: directories: - $HOME/.cache/pip -python: 2.7 +python: + - '3.5' env: global: PYTHONUNBUFFERED=yes @@ -12,10 +13,10 @@ env: - TOXENV=3.3 - TOXENV=3.4 - TOXENV=pypy -install: - - pip install tox -script: - - tox -v -- -v + - TOXENV=3.5 + - TOXENV=pypy3 +install: pip install -U tox +script: tox -v -- -v after_success: - .tox/$TRAVIS_PYTHON_VERSION/bin/coveralls notifications: diff --git a/tox.ini b/tox.ini index 4aedf239b..1ea87155f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,65 +1,31 @@ [tox] -envlist = - 2.7, - 3.3, - 3.4, - pypy, - pypy3 +envlist = 2.7,pypy,3.3,3.4,3.5,pypy3 [testenv] -sitepackages = False -commands = nosetests +deps= + -r{toxinidir}/requirements/default.txt -[testenv:3.4] -basepython = python3.4 -deps = -r{toxinidir}/requirements/default.txt - -r{toxinidir}/requirements/test3.txt - -r{toxinidir}/requirements/test-ci-default.txt -setenv = C_DEBUG_TEST = 1 -commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - pip install -U -r{toxinidir}/requirements/dev.txt - nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + 2.7,pypy: -r{toxinidir}/requirements/test.txt + 2.7,pypy: -r{toxinidir}/requirements/test-ci-default.txt -[testenv:3.3] -basepython = python3.3 -deps = -r{toxinidir}/requirements/default.txt - -r{toxinidir}/requirements/test3.txt - -r{toxinidir}/requirements/test-ci-default.txt -setenv = C_DEBUG_TEST = 1 -commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - pip install -U -r{toxinidir}/requirements/dev.txt - nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + 3.3,3.4,3.5: -r{toxinidir}/requirements/test3.txt + 3.3,3.4,3.5: -r{toxinidir}/requirements/test-ci-default.txt -[testenv:2.7] -basepython = python2.7 -deps = -r{toxinidir}/requirements/default.txt - -r{toxinidir}/requirements/test.txt - -r{toxinidir}/requirements/test-ci-default.txt -setenv = C_DEBUG_TEST = 1 -commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - pip install -U -r{toxinidir}/requirements/dev.txt - nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + pypy3: -r{toxinidir}/requirements/test3.txt + pypy3: -r{toxinidir}/requirements/test-pypy3.txt + pypy3: -r{toxinidir}/requirements/test-ci-base.txt -[testenv:pypy] -basepython = pypy -deps = -r{toxinidir}/requirements/default.txt - -r{toxinidir}/requirements/test.txt - -r{toxinidir}/requirements/test-ci-base.txt -setenv = C_DEBUG_TEST = 1 -commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - pip install -q -U -r{toxinidir}/requirements/dev.txt - nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] - -[testenv:pypy3] -basepython = pypy3 -deps = -r{toxinidir}/requirements/default.txt - -r{toxinidir}/requirements/test3.txt - -r{toxinidir}/requirements/test-pypy3.txt - -r{toxinidir}/requirements/test-ci-base.txt -setenv = C_DEBUG_TEST = 1 +sitepackages = False +recreate = False commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - pip install -q -U -r{toxinidir}/requirements/dev.txt nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] +basepython = + 2.7: python2.7 + 3.3: python3.3 + 3.4: python3.4 + 3.5: python3.5 + pypy: pypy + pypy3: pypy3 [testenv:docs] deps = -r{toxinidir}/requirements/docs.txt From 931a5f8fd99699e3f5bae605ae7767bbc4c3a6e1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 25 Sep 2015 17:36:20 -0700 Subject: [PATCH 0654/1103] [ci] Tox must install dev requirements last --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 1ea87155f..58e2cd375 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,7 @@ deps= sitepackages = False recreate = False commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} + pip install -q -U -r{toxinidir}/requirements/dev.txt nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] basepython = 2.7: python2.7 From 30f1805b97466457556363136a5a8617af9bd0b9 Mon Sep 17 00:00:00 2001 From: areski Date: Mon, 28 Sep 2015 17:09:58 +0200 Subject: [PATCH 0655/1103] typo doc --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index d76671e02..f7a02bd83 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -586,7 +586,7 @@ To not return a negative exit code when this command fails use the ``flakes`` target instead: :: - $ make flakes§ + $ make flakes API reference ~~~~~~~~~~~~~ From f80dca3982bb9eb899cf996420914579c1fdfc9b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 28 Sep 2015 13:04:14 -0700 Subject: [PATCH 0656/1103] memoize: LRUCache already thread-safe. --- celery/utils/functional.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index b345e283b..c498d211f 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -150,7 +150,6 @@ def items(self): def memoize(maxsize=None, keyfun=None, Cache=LRUCache): def _memoize(fun): - mutex = threading.Lock() cache = Cache(limit=maxsize) @wraps(fun) @@ -160,13 +159,11 @@ def _M(*args, **kwargs): else: key = args + (KEYWORD_MARK,) + tuple(sorted(kwargs.items())) try: - with mutex: - value = cache[key] + value = cache[key] except KeyError: value = fun(*args, **kwargs) _M.misses += 1 - with mutex: - cache[key] = value + cache[key] = value else: _M.hits += 1 return value From 990a04615ac9ea36dbf0526550af7733a2deeaa3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 28 Sep 2015 13:04:56 -0700 Subject: [PATCH 0657/1103] Tests: import_all_modules now ignores OSError --- celery/tests/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/celery/tests/__init__.py b/celery/tests/__init__.py index 24fc92c78..8e8366ab6 100644 --- a/celery/tests/__init__.py +++ b/celery/tests/__init__.py @@ -90,3 +90,8 @@ def import_all_modules(name=__name__, file=__file__, import_module(module) except ImportError: pass + except OSError as exc: + warnings.warn(UserWarning( + 'Ignored error importing module {0}: {1!r}').format( + module, exc, + )) From 31767a5a43ce47bbae1d5648bfb60488dc64ee50 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 28 Sep 2015 13:05:44 -0700 Subject: [PATCH 0658/1103] [ci] Tests passing on Python 3.5 --- celery/tests/utils/test_functional.py | 5 ++++- celery/utils/functional.py | 21 ++++++++++----------- tox.ini | 7 +++---- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/celery/tests/utils/test_functional.py b/celery/tests/utils/test_functional.py index e564a4120..99b4f6543 100644 --- a/celery/tests/utils/test_functional.py +++ b/celery/tests/utils/test_functional.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import pickle +import sys from kombu.utils.functional import lazy @@ -14,7 +15,7 @@ maybe_list, ) -from celery.tests.case import Case +from celery.tests.case import Case, SkipTest class test_LRUCache(Case): @@ -63,6 +64,8 @@ def test_least_recently_used(self): self.assertEqual(list(x.keys()), [3, 6, 7]) def assertSafeIter(self, method, interval=0.01, size=10000): + if sys.version_info >= (3,5): + raise SkipTest('Fails on Py3.5') from threading import Thread, Event from time import sleep x = LRUCache(size) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index c498d211f..578b31a47 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -25,7 +25,7 @@ 'first', 'firstmethod', 'chunks', 'padlist', 'mattrgetter', 'uniq', 'regen', 'dictfilter', 'lazy', 'maybe_evaluate', 'head_from_fun'] -IS_PYPY = hasattr(sys, 'pypy_version_info') +IS_PY3 = sys.version_info[0] == 3 KEYWORD_MARK = object() @@ -62,7 +62,7 @@ def __init__(self, limit=None): def __getitem__(self, key): with self.mutex: value = self[key] = self.data.pop(key) - return value + return value def update(self, *args, **kwargs): with self.mutex: @@ -74,9 +74,7 @@ def update(self, *args, **kwargs): for item in islice(iter(data), len(data) - limit): data.pop(item) - def popitem(self, last=True, _needs_lock=IS_PYPY): - if not _needs_lock: - return self.data.popitem(last) + def popitem(self, last=True): with self.mutex: return self.data.popitem(last) @@ -90,8 +88,8 @@ def __setitem__(self, key, value): def __iter__(self): return iter(self.data) - def _iterate_items(self, _need_lock=IS_PYPY): - with self.mutex if _need_lock else DummyContext(): + def _iterate_items(self): + with self.mutex: for k in self: try: yield (k, self.data[k]) @@ -99,8 +97,8 @@ def _iterate_items(self, _need_lock=IS_PYPY): pass iteritems = _iterate_items - def _iterate_values(self, _need_lock=IS_PYPY): - with self.mutex if _need_lock else DummyContext(): + def _iterate_values(self): + with self.mutex: for k in self: try: yield self.data[k] @@ -111,7 +109,8 @@ def _iterate_values(self, _need_lock=IS_PYPY): def _iterate_keys(self): # userdict.keys in py3k calls __getitem__ - return keys(self.data) + with self.mutex: + return keys(self.data) iterkeys = _iterate_keys def incr(self, key, delta=1): @@ -120,7 +119,7 @@ def incr(self, key, delta=1): # integer as long as it exists and we can cast it newval = int(self.data.pop(key)) + delta self[key] = str(newval) - return newval + return newval def __getstate__(self): d = dict(vars(self)) diff --git a/tox.ini b/tox.ini index 58e2cd375..6c86d8064 100644 --- a/tox.ini +++ b/tox.ini @@ -6,14 +6,13 @@ deps= -r{toxinidir}/requirements/default.txt 2.7,pypy: -r{toxinidir}/requirements/test.txt - 2.7,pypy: -r{toxinidir}/requirements/test-ci-default.txt + 2.7: -r{toxinidir}/requirements/test-ci-default.txt - 3.3,3.4,3.5: -r{toxinidir}/requirements/test3.txt + 3.3,3.4,3.5,pypy3: -r{toxinidir}/requirements/test3.txt 3.3,3.4,3.5: -r{toxinidir}/requirements/test-ci-default.txt - pypy3: -r{toxinidir}/requirements/test3.txt + pypy,pypy3: -r{toxinidir}/requirements/test-ci-base.txt pypy3: -r{toxinidir}/requirements/test-pypy3.txt - pypy3: -r{toxinidir}/requirements/test-ci-base.txt sitepackages = False recreate = False From 24b9857311c38161016617e951e5c7e32cab2857 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 28 Sep 2015 13:37:46 -0700 Subject: [PATCH 0659/1103] Fixes typo --- celery/tests/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/tests/__init__.py b/celery/tests/__init__.py index 8e8366ab6..629e9279e 100644 --- a/celery/tests/__init__.py +++ b/celery/tests/__init__.py @@ -92,6 +92,6 @@ def import_all_modules(name=__name__, file=__file__, pass except OSError as exc: warnings.warn(UserWarning( - 'Ignored error importing module {0}: {1!r}').format( + 'Ignored error importing module {0}: {1!r}'.format( module, exc, - )) + ))) From 5106352570e56926d6d0efb5d57516d1d63ec751 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 28 Sep 2015 13:37:54 -0700 Subject: [PATCH 0660/1103] flakes --- celery/__init__.py | 7 +- celery/app/base.py | 3 +- celery/apps/worker.py | 3 +- celery/backends/database/__init__.py | 22 +-- celery/backends/mongodb.py | 18 +- celery/backends/riak.py | 6 +- celery/beat.py | 14 +- celery/bin/beat.py | 6 +- celery/bin/celery.py | 3 +- celery/bin/events.py | 6 +- celery/concurrency/eventlet.py | 9 +- celery/concurrency/prefork.py | 7 +- celery/events/cursesmon.py | 4 +- celery/five.py | 25 +-- celery/fixups/django.py | 11 +- celery/local.py | 182 +++++++++++++++------ celery/tests/app/test_app.py | 15 +- celery/tests/app/test_loaders.py | 3 +- celery/tests/backends/test_database.py | 7 - celery/tests/case.py | 4 +- celery/tests/compat_modules/test_compat.py | 5 +- celery/tests/events/test_state.py | 3 +- celery/tests/tasks/test_chord.py | 4 +- celery/tests/utils/test_functional.py | 2 +- celery/tests/utils/test_platforms.py | 3 +- celery/tests/utils/test_threads.py | 3 +- celery/tests/worker/test_hub.py | 3 +- celery/utils/term.py | 5 +- celery/worker/request.py | 4 +- 29 files changed, 240 insertions(+), 147 deletions(-) diff --git a/celery/__init__.py b/celery/__init__.py index 1fc03e81a..65ef1446c 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -8,6 +8,9 @@ from __future__ import absolute_import, print_function, unicode_literals +import os +import sys + from collections import namedtuple version_info_t = namedtuple( @@ -31,8 +34,6 @@ # -eof meta- -import os -import sys if os.environ.get('C_IMPDEBUG'): # pragma: no cover from .five import builtins real_import = builtins.__import__ @@ -128,7 +129,7 @@ def maybe_patch_concurrency(argv=sys.argv, concurrency.get_implementation(pool) # Lazy loading -from celery import five +from celery import five # noqa old_module, new_module = five.recreate_module( # pragma: no cover __name__, diff --git a/celery/app/base.py b/celery/app/base.py index 5042a6d1c..ac845c565 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -247,7 +247,8 @@ def inner_create_task_cls(shared=True, filter=None, lazy=True, **opts): def _create_task_cls(fun): if shared: - cons = lambda app: app._task_from_fun(fun, **opts) + def cons(app): + return app._task_from_fun(fun, **opts) cons.__name__ = fun.__name__ connect_on_app_finalize(cons) if not lazy or self.finalized: diff --git a/celery/apps/worker.py b/celery/apps/worker.py index a9436b8fa..27b419d78 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -318,7 +318,8 @@ def on_SIGINT(worker): exitcode=EX_FAILURE, ) else: # pragma: no cover - install_worker_int_handler = lambda *a, **kw: None + def install_worker_int_handler(*args, **kwargs): + pass def _reload_current_worker(): diff --git a/celery/backends/database/__init__.py b/celery/backends/database/__init__.py index 96dbb0a0d..86bb4189c 100644 --- a/celery/backends/database/__init__.py +++ b/celery/backends/database/__init__.py @@ -22,25 +22,19 @@ from .models import TaskSet from .session import SessionManager +try: + from sqlalchemy.exc import DatabaseError, InvalidRequestError + from sqlalchemy.orm.exc import StaleDataError +except ImportError: + raise ImproperlyConfigured( + 'The database result backend requires SQLAlchemy to be installed.' + 'See http://pypi.python.org/pypi/SQLAlchemy') + logger = logging.getLogger(__name__) __all__ = ['DatabaseBackend'] -def _sqlalchemy_installed(): - try: - import sqlalchemy - except ImportError: - raise ImproperlyConfigured( - 'The database result backend requires SQLAlchemy to be installed.' - 'See http://pypi.python.org/pypi/SQLAlchemy') - return sqlalchemy -_sqlalchemy_installed() - -from sqlalchemy.exc import DatabaseError, InvalidRequestError -from sqlalchemy.orm.exc import StaleDataError - - @contextmanager def session_cleanup(session): try: diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 36cbb5342..adf535c43 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -10,6 +10,15 @@ from datetime import datetime, timedelta +from kombu.syn import detect_environment +from kombu.utils import cached_property +from kombu.exceptions import EncodeError +from celery import states +from celery.exceptions import ImproperlyConfigured +from celery.five import string_t, items + +from .base import BaseBackend + try: import pymongo except ImportError: # pragma: no cover @@ -25,15 +34,6 @@ Binary = None # noqa InvalidDocument = None # noqa -from kombu.syn import detect_environment -from kombu.utils import cached_property -from kombu.exceptions import EncodeError -from celery import states -from celery.exceptions import ImproperlyConfigured -from celery.five import string_t, items - -from .base import BaseBackend - __all__ = ['MongoBackend'] diff --git a/celery/backends/riak.py b/celery/backends/riak.py index 3b7ff90be..5e4565ede 100644 --- a/celery/backends/riak.py +++ b/celery/backends/riak.py @@ -32,7 +32,6 @@ def to_bytes(s): return s.encode() if isinstance(s, str) else s - def str_decode(s, encoding): return to_bytes(s).decode(encoding) @@ -42,7 +41,6 @@ def str_decode(s, encoding): return s.decode("ascii") - def is_ascii(s): try: str_decode(s, 'ascii') @@ -118,8 +116,8 @@ def _get_client(self): def _get_bucket(self): """Connect to our bucket.""" if ( - self._client is None or not self._client.is_alive() - or not self._bucket + self._client is None or not self._client.is_alive() or + not self._bucket ): self._bucket = self.client.bucket(self.bucket_name) return self._bucket diff --git a/celery/beat.py b/celery/beat.py index 8ba112127..045b65a72 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -185,9 +185,9 @@ def __init__(self, app, schedule=None, max_interval=None, Producer=None, lazy=False, sync_every_tasks=None, **kwargs): self.app = app self.data = maybe_evaluate({} if schedule is None else schedule) - self.max_interval = (max_interval - or app.conf.CELERYBEAT_MAX_LOOP_INTERVAL - or self.max_interval) + self.max_interval = (max_interval or + app.conf.CELERYBEAT_MAX_LOOP_INTERVAL or + self.max_interval) self.Producer = Producer or app.amqp.Producer self._heap = None self.sync_every_tasks = ( @@ -236,8 +236,8 @@ def tick(self, event_t=event_t, min=min, """ def _when(entry, next_time_to_run): - return (mktime(entry.schedule.now().timetuple()) - + (adjust(next_time_to_run) or 0)) + return (mktime(entry.schedule.now().timetuple()) + + (adjust(next_time_to_run) or 0)) adjust = self.adjust max_interval = self.max_interval @@ -474,8 +474,8 @@ class Service(object): def __init__(self, app, max_interval=None, schedule_filename=None, scheduler_cls=None): self.app = app - self.max_interval = (max_interval - or app.conf.CELERYBEAT_MAX_LOOP_INTERVAL) + self.max_interval = (max_interval or + app.conf.CELERYBEAT_MAX_LOOP_INTERVAL) self.scheduler_cls = scheduler_cls or self.scheduler_cls self.schedule_filename = ( schedule_filename or app.conf.CELERYBEAT_SCHEDULE_FILENAME) diff --git a/celery/bin/beat.py b/celery/bin/beat.py index 6b5b73468..4bcbc626b 100644 --- a/celery/bin/beat.py +++ b/celery/bin/beat.py @@ -87,9 +87,9 @@ def get_options(self): default=c.CELERYBEAT_SCHEDULE_FILENAME), Option('--max-interval', type='float'), Option('-S', '--scheduler', dest='scheduler_cls'), - Option('-l', '--loglevel', default=c.CELERYBEAT_LOG_LEVEL)) - + daemon_options(default_pidfile='celerybeat.pid') - + tuple(self.app.user_options['beat']) + Option('-l', '--loglevel', default=c.CELERYBEAT_LOG_LEVEL)) + + daemon_options(default_pidfile='celerybeat.pid') + + tuple(self.app.user_options['beat']) ) diff --git a/celery/bin/celery.py b/celery/bin/celery.py index d558dd8ac..54a9f5bb8 100644 --- a/celery/bin/celery.py +++ b/celery/bin/celery.py @@ -117,7 +117,8 @@ def list_bindings(self, management): except NotImplementedError: raise self.Error('Your transport cannot list bindings.') - fmt = lambda q, e, r: self.out('{0:<28} {1:<28} {2}'.format(q, e, r)) + def fmt(q, e, r): + return self.out('{0:<28} {1:<28} {2}'.format(q, e, r)) fmt('Queue', 'Exchange', 'Routing Key') fmt('-' * 16, '-' * 16, '-' * 16) for b in bindings: diff --git a/celery/bin/events.py b/celery/bin/events.py index d98750504..8cc61b6d6 100644 --- a/celery/bin/events.py +++ b/celery/bin/events.py @@ -125,9 +125,9 @@ def get_options(self): Option('-F', '--frequency', '--freq', type='float', default=1.0), Option('-r', '--maxrate'), - Option('-l', '--loglevel', default='INFO')) - + daemon_options(default_pidfile='celeryev.pid') - + tuple(self.app.user_options['events']) + Option('-l', '--loglevel', default='INFO')) + + daemon_options(default_pidfile='celeryev.pid') + + tuple(self.app.user_options['events']) ) diff --git a/celery/concurrency/eventlet.py b/celery/concurrency/eventlet.py index c501985fa..7a8c9ae1b 100644 --- a/celery/concurrency/eventlet.py +++ b/celery/concurrency/eventlet.py @@ -28,12 +28,13 @@ import warnings warnings.warn(RuntimeWarning(W_RACE % side)) -from kombu.async import timer as _timer +# idiotic pep8.py does not allow expressions before imports +# so have to silence errors here +from kombu.async import timer as _timer # noqa +from celery import signals # noqa -from celery import signals - -from . import base +from . import base # noqa def apply_target(target, args=(), kwargs={}, callback=None, diff --git a/celery/concurrency/prefork.py b/celery/concurrency/prefork.py index c2f99caad..dac9f2111 100644 --- a/celery/concurrency/prefork.py +++ b/celery/concurrency/prefork.py @@ -154,10 +154,7 @@ def on_close(self): self._pool.close() def _get_info(self): - try: - write_stats = self._pool.human_write_stats - except AttributeError: - write_stats = lambda: 'N/A' # only supported by asynpool + write_stats = getattr(self._pool, 'human_write_stats', None) return { 'max-concurrency': self.limit, 'processes': [p.pid for p in self._pool._pool], @@ -165,7 +162,7 @@ def _get_info(self): 'put-guarded-by-semaphore': self.putlocks, 'timeouts': (self._pool.soft_timeout or 0, self._pool.timeout or 0), - 'writes': write_stats() + 'writes': write_stats() if write_stats is not None else 'N/A', } @property diff --git a/celery/events/cursesmon.py b/celery/events/cursesmon.py index 4f34a66de..3ac164fa7 100644 --- a/celery/events/cursesmon.py +++ b/celery/events/cursesmon.py @@ -318,8 +318,8 @@ def selection_result(self): def alert_callback(my, mx, xs): y = count(xs) task = self.state.tasks[self.selected_task] - result = (getattr(task, 'result', None) - or getattr(task, 'exception', None)) + result = (getattr(task, 'result', None) or + getattr(task, 'exception', None)) for line in wrap(result, mx - 2): self.win.addstr(next(y), 3, line) diff --git a/celery/five.py b/celery/five.py index 6c5d9b007..d6ec040cc 100644 --- a/celery/five.py +++ b/celery/five.py @@ -10,14 +10,24 @@ """ from __future__ import absolute_import -__all__ = [ - 'class_property', 'reclassmethod', 'create_module', 'recreate_module', -] +import operator +import sys + +from importlib import import_module +from types import ModuleType # extends amqp.five from amqp.five import * # noqa from amqp.five import __all__ as _all_five +try: + from functools import reduce +except ImportError: + pass + +__all__ = [ + 'class_property', 'reclassmethod', 'create_module', 'recreate_module', +] __all__ += _all_five # ############# Module Generation ########################## @@ -26,17 +36,8 @@ # recreate modules, either for lazy loading or # to create old modules at runtime instead of # having them litter the source tree. -import operator -import sys # import fails in python 2.5. fallback to reduce in stdlib -try: - from functools import reduce -except ImportError: - pass - -from importlib import import_module -from types import ModuleType MODULE_DEPRECATED = """ The module %s is deprecated and will be removed in a future version. diff --git a/celery/fixups/django.py b/celery/fixups/django.py index d67eb3d55..379ce34b9 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -4,11 +4,6 @@ import sys import warnings -if sys.version_info[0] < 3 and not hasattr(sys, 'pypy_version_info'): - from StringIO import StringIO -else: - from io import StringIO - from kombu.utils import cached_property, symbol_by_name from datetime import datetime @@ -18,6 +13,12 @@ from celery.app import default_app from celery.exceptions import FixupWarning +if sys.version_info[0] < 3 and not hasattr(sys, 'pypy_version_info'): + from StringIO import StringIO +else: + from io import StringIO + + __all__ = ['DjangoFixup', 'fixup'] ERR_NOT_INSTALLED = """\ diff --git a/celery/local.py b/celery/local.py index 70f7df72d..2e4b12bd6 100644 --- a/celery/local.py +++ b/celery/local.py @@ -154,54 +154,144 @@ def __setslice__(self, i, j, seq): def __delslice__(self, i, j): del self._get_current_object()[i:j] - __setattr__ = lambda x, n, v: setattr(x._get_current_object(), n, v) - __delattr__ = lambda x, n: delattr(x._get_current_object(), n) - __str__ = lambda x: str(x._get_current_object()) - __lt__ = lambda x, o: x._get_current_object() < o - __le__ = lambda x, o: x._get_current_object() <= o - __eq__ = lambda x, o: x._get_current_object() == o - __ne__ = lambda x, o: x._get_current_object() != o - __gt__ = lambda x, o: x._get_current_object() > o - __ge__ = lambda x, o: x._get_current_object() >= o - __hash__ = lambda x: hash(x._get_current_object()) - __call__ = lambda x, *a, **kw: x._get_current_object()(*a, **kw) - __len__ = lambda x: len(x._get_current_object()) - __getitem__ = lambda x, i: x._get_current_object()[i] - __iter__ = lambda x: iter(x._get_current_object()) - __contains__ = lambda x, i: i in x._get_current_object() - __getslice__ = lambda x, i, j: x._get_current_object()[i:j] - __add__ = lambda x, o: x._get_current_object() + o - __sub__ = lambda x, o: x._get_current_object() - o - __mul__ = lambda x, o: x._get_current_object() * o - __floordiv__ = lambda x, o: x._get_current_object() // o - __mod__ = lambda x, o: x._get_current_object() % o - __divmod__ = lambda x, o: x._get_current_object().__divmod__(o) - __pow__ = lambda x, o: x._get_current_object() ** o - __lshift__ = lambda x, o: x._get_current_object() << o - __rshift__ = lambda x, o: x._get_current_object() >> o - __and__ = lambda x, o: x._get_current_object() & o - __xor__ = lambda x, o: x._get_current_object() ^ o - __or__ = lambda x, o: x._get_current_object() | o - __div__ = lambda x, o: x._get_current_object().__div__(o) - __truediv__ = lambda x, o: x._get_current_object().__truediv__(o) - __neg__ = lambda x: -(x._get_current_object()) - __pos__ = lambda x: +(x._get_current_object()) - __abs__ = lambda x: abs(x._get_current_object()) - __invert__ = lambda x: ~(x._get_current_object()) - __complex__ = lambda x: complex(x._get_current_object()) - __int__ = lambda x: int(x._get_current_object()) - __float__ = lambda x: float(x._get_current_object()) - __oct__ = lambda x: oct(x._get_current_object()) - __hex__ = lambda x: hex(x._get_current_object()) - __index__ = lambda x: x._get_current_object().__index__() - __coerce__ = lambda x, o: x._get_current_object().__coerce__(o) - __enter__ = lambda x: x._get_current_object().__enter__() - __exit__ = lambda x, *a, **kw: x._get_current_object().__exit__(*a, **kw) - __reduce__ = lambda x: x._get_current_object().__reduce__() + def __setattr__(self, name, value): + setattr(self._get_current_object(), name, value) + + def __delattr__(self, name): + delattr(self._get_current_object(), name) + + def __str__(self): + return str(self._get_current_object()) + + def __lt__(self, other): + return self._get_current_object() < other + + def __le__(self, other): + return self._get_current_object() <= other + + def __eq__(self, other): + return self._get_current_object() == other + + def __ne__(self, other): + return self._get_current_object() != other + + def __gt__(self, other): + return self._get_current_object() > other + + def __ge__(self, other): + return self._get_current_object() >= other + + def __hash__(self): + return hash(self._get_current_object()) + + def __call__(self, *a, **kw): + return self._get_current_object()(*a, **kw) + + def __len__(self): + return len(self._get_current_object()) + + def __getitem__(self, i): + return self._get_current_object()[i] + + def __iter__(self): + return iter(self._get_current_object()) + + def __contains__(self, i): + return i in self._get_current_object() + + def __getslice__(self, i, j): + return self._get_current_object()[i:j] + + def __add__(self, other): + return self._get_current_object() + other + + def __sub__(self, other): + return self._get_current_object() - other + + def __mul__(self, other): + return self._get_current_object() * other + + def __floordiv__(self, other): + return self._get_current_object() // other + + def __mod__(self, other): + return self._get_current_object() % other + + def __divmod__(self, other): + return self._get_current_object().__divmod__(other) + + def __pow__(self, other): + return self._get_current_object() ** other + + def __lshift__(self, other): + return self._get_current_object() << other + + def __rshift__(self, other): + return self._get_current_object() >> other + + def __and__(self, other): + return self._get_current_object() & other + + def __xor__(self, other): + return self._get_current_object() ^ other + + def __or__(self, other): + return self._get_current_object() | other + + def __div__(self, other): + return self._get_current_object().__div__(other) + + def __truediv__(self, other): + return self._get_current_object().__truediv__(other) + + def __neg__(self): + return -(self._get_current_object()) + + def __pos__(self): + return +(self._get_current_object()) + + def __abs__(self): + return abs(self._get_current_object()) + + def __invert__(self): + return ~(self._get_current_object()) + + def __complex__(self): + return complex(self._get_current_object()) + + def __int__(self): + return int(self._get_current_object()) + + def __float__(self): + return float(self._get_current_object()) + + def __oct__(self): + return oct(self._get_current_object()) + + def __hex__(self): + return hex(self._get_current_object()) + + def __index__(self): + return self._get_current_object().__index__() + + def __coerce__(self, other): + return self._get_current_object().__coerce__(other) + + def __enter__(self): + return self._get_current_object().__enter__() + + def __exit__(self, *a, **kw): + return self._get_current_object().__exit__(*a, **kw) + + def __reduce__(self): + return self._get_current_object().__reduce__() if not PY3: - __cmp__ = lambda x, o: cmp(x._get_current_object(), o) # noqa - __long__ = lambda x: long(x._get_current_object()) # noqa + def __cmp__(self, other): + return cmp(self._get_current_object(), other) # noqa + + def __long__(self): + return long(self._get_current_object()) # noqa class PromiseProxy(Proxy): diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index af4dedc02..de0d1f034 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -143,7 +143,10 @@ def foo(): def test_add_defaults(self): self.assertFalse(self.app.configured) _conf = {'FOO': 300} - conf = lambda: _conf + + def conf(): + return _conf + self.app.add_defaults(conf) self.assertIn(conf, self.app._pending_defaults) self.assertFalse(self.app.configured) @@ -196,8 +199,11 @@ def test_autodiscover_tasks_force(self): ['proj.A', 'proj.B'], 'tasks', ) self.app.loader.autodiscover_tasks = Mock() + + def lazy_list(): + return ['proj.A', 'proj.B'] self.app.autodiscover_tasks( - lambda: ['proj.A', 'proj.B'], + lazy_list, related_name='george', force=True, ) @@ -207,8 +213,9 @@ def test_autodiscover_tasks_force(self): def test_autodiscover_tasks_lazy(self): with patch('celery.signals.import_modules') as import_modules: - packages = lambda: [1, 2, 3] - self.app.autodiscover_tasks(packages) + def lazy_list(): + return [1, 2, 3] + self.app.autodiscover_tasks(lazy_list) self.assertTrue(import_modules.connect.called) prom = import_modules.connect.call_args[0][0] self.assertIsInstance(prom, promise) diff --git a/celery/tests/app/test_loaders.py b/celery/tests/app/test_loaders.py index c98582933..cb3d3c337 100644 --- a/celery/tests/app/test_loaders.py +++ b/celery/tests/app/test_loaders.py @@ -99,7 +99,8 @@ def test_conf_property(self): self.assertEqual(self.loader.conf['foo'], 'bar') def test_import_default_modules(self): - modnames = lambda l: [m.__name__ for m in l] + def modnames(l): + return [m.__name__ for m in l] self.app.conf.CELERY_IMPORTS = ('os', 'sys') self.assertEqual( sorted(modnames(self.loader.import_default_modules())), diff --git a/celery/tests/backends/test_database.py b/celery/tests/backends/test_database.py index 70d8339bf..4e3cabfeb 100644 --- a/celery/tests/backends/test_database.py +++ b/celery/tests/backends/test_database.py @@ -12,7 +12,6 @@ AppCase, SkipTest, depends_on_current_app, - mask_modules, skip_if_pypy, skip_if_jython, ) @@ -56,12 +55,6 @@ def raises(): raises(max_retries=5) self.assertEqual(calls[0], 5) - def test_missing_SQLAlchemy_raises_ImproperlyConfigured(self): - with mask_modules('sqlalchemy'): - from celery.backends.database import _sqlalchemy_installed - with self.assertRaises(ImproperlyConfigured): - _sqlalchemy_installed() - def test_missing_dburi_raises_ImproperlyConfigured(self): self.app.conf.CELERY_RESULT_DBURI = None with self.assertRaises(ImproperlyConfigured): diff --git a/celery/tests/case.py b/celery/tests/case.py index 432d206b9..aedd3f4fc 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -103,8 +103,8 @@ 'host': os.environ.get('MONGO_HOST') or 'localhost', 'port': os.environ.get('MONGO_PORT') or 27017, 'database': os.environ.get('MONGO_DB') or 'celery_unittests', - 'taskmeta_collection': (os.environ.get('MONGO_TASKMETA_COLLECTION') - or 'taskmeta_collection'), + 'taskmeta_collection': (os.environ.get('MONGO_TASKMETA_COLLECTION') or + 'taskmeta_collection'), 'user': os.environ.get('MONGO_USER'), 'password': os.environ.get('MONGO_PASSWORD'), } diff --git a/celery/tests/compat_modules/test_compat.py b/celery/tests/compat_modules/test_compat.py index ee9c5cb26..433186950 100644 --- a/celery/tests/compat_modules/test_compat.py +++ b/celery/tests/compat_modules/test_compat.py @@ -2,16 +2,13 @@ from datetime import timedelta -import sys -sys.modules.pop('celery.task', None) - from celery.schedules import schedule from celery.task import ( periodic_task, PeriodicTask ) -from celery.tests.case import AppCase, depends_on_current_app +from celery.tests.case import AppCase, depends_on_current_app # noqa @depends_on_current_app diff --git a/celery/tests/events/test_state.py b/celery/tests/events/test_state.py index e12ae77c9..ad8a041d8 100644 --- a/celery/tests/events/test_state.py +++ b/celery/tests/events/test_state.py @@ -26,7 +26,8 @@ # Py2.6: Must first convert float to str _float_to_decimal = str else: - _float_to_decimal = lambda f: f # noqa + def _float_to_decimal(f): # noqa + return f class replay(object): diff --git a/celery/tests/tasks/test_chord.py b/celery/tests/tasks/test_chord.py index e09211f00..a7cc1d859 100644 --- a/celery/tests/tasks/test_chord.py +++ b/celery/tests/tasks/test_chord.py @@ -10,7 +10,9 @@ from celery.result import AsyncResult, GroupResult, EagerResult from celery.tests.case import AppCase, Mock -passthru = lambda x: x + +def passthru(x): + return x class ChordCase(AppCase): diff --git a/celery/tests/utils/test_functional.py b/celery/tests/utils/test_functional.py index 99b4f6543..c60419d00 100644 --- a/celery/tests/utils/test_functional.py +++ b/celery/tests/utils/test_functional.py @@ -64,7 +64,7 @@ def test_least_recently_used(self): self.assertEqual(list(x.keys()), [3, 6, 7]) def assertSafeIter(self, method, interval=0.01, size=10000): - if sys.version_info >= (3,5): + if sys.version_info >= (3, 5): raise SkipTest('Fails on Py3.5') from threading import Thread, Event from time import sleep diff --git a/celery/tests/utils/test_platforms.py b/celery/tests/utils/test_platforms.py index aae0b38a0..02dd7bece 100644 --- a/celery/tests/utils/test_platforms.py +++ b/celery/tests/utils/test_platforms.py @@ -137,7 +137,8 @@ def test_ignore(self, set): @patch('signal.signal') def test_setitem(self, set): - handle = lambda *a: a + def handle(*args): + return args signals['INT'] = handle set.assert_called_with(signal.SIGINT, handle) diff --git a/celery/tests/utils/test_threads.py b/celery/tests/utils/test_threads.py index 4c85b2338..7eaa51e16 100644 --- a/celery/tests/utils/test_threads.py +++ b/celery/tests/utils/test_threads.py @@ -90,7 +90,8 @@ def test_init(self): self.assertListEqual(x.locals, []) self.assertTrue(x.ident_func) - ident = lambda: 1 + def ident(): + return 1 loc = Local() x = LocalManager([loc], ident_func=ident) self.assertListEqual(x.locals, [loc]) diff --git a/celery/tests/worker/test_hub.py b/celery/tests/worker/test_hub.py index 3909e9a2e..4f6b5dfa0 100644 --- a/celery/tests/worker/test_hub.py +++ b/celery/tests/worker/test_hub.py @@ -162,7 +162,8 @@ def test_fire_timers(self): e1, e2, e3 = Mock(), Mock(), Mock() entries = [e1, e2, e3] - reset = lambda: [m.reset() for m in [e1, e2, e3]] + def reset(): + return [m.reset() for m in [e1, e2, e3]] def se(): while 1: diff --git a/celery/utils/term.py b/celery/utils/term.py index f6f08d44c..a71be76b5 100644 --- a/celery/utils/term.py +++ b/celery/utils/term.py @@ -21,11 +21,14 @@ OP_SEQ = '\033[%dm' RESET_SEQ = '\033[0m' COLOR_SEQ = '\033[1;%dm' -fg = lambda s: COLOR_SEQ % s IS_WINDOWS = platform.system() == 'Windows' +def fg(s): + return COLOR_SEQ % s + + class colored(object): """Terminal colored text. diff --git a/celery/worker/request.py b/celery/worker/request.py index 0388a0970..fded7597c 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -409,8 +409,8 @@ def tzlocal(self): @property def store_errors(self): - return (not self.task.ignore_result - or self.task.store_errors_even_if_ignored) + return (not self.task.ignore_result or + self.task.store_errors_even_if_ignored) @property def task_id(self): From e489f3cf1aacf864479e19fde46361f79c073d1c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 28 Sep 2015 13:51:54 -0700 Subject: [PATCH 0661/1103] [CI] Attempt to fix pypy3 build --- celery/tests/bin/test_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/tests/bin/test_events.py b/celery/tests/bin/test_events.py index a6e79f75a..80e17609d 100644 --- a/celery/tests/bin/test_events.py +++ b/celery/tests/bin/test_events.py @@ -32,7 +32,7 @@ def test_run_dump(self): def test_run_top(self): try: import curses # noqa - except ImportError: + except (ImportError, OSError): raise SkipTest('curses monitor requires curses') @_old_patch('celery.events.cursesmon', 'evtop', From 5efd77f843063283e85f1d004d37fc4d85358f34 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 28 Sep 2015 15:08:26 -0700 Subject: [PATCH 0662/1103] [CI] Another curses import, breaking on pypy3 --- celery/tests/events/test_cursesmon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/tests/events/test_cursesmon.py b/celery/tests/events/test_cursesmon.py index c8e615167..d5c10953a 100644 --- a/celery/tests/events/test_cursesmon.py +++ b/celery/tests/events/test_cursesmon.py @@ -14,7 +14,7 @@ class test_CursesDisplay(AppCase): def setup(self): try: import curses # noqa - except ImportError: + except (ImportError, OSError): raise SkipTest('curses monitor requires curses') from celery.events import cursesmon From fe33f16e014611ee7267081b3380b5a0003faf78 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 28 Sep 2015 15:17:55 -0700 Subject: [PATCH 0663/1103] [CI] do not use pip cache, and use travis_retry when installing tox --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6b4b6f3ee..f9cb0a0e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,6 @@ language: python sudo: false -cache: - directories: - - $HOME/.cache/pip +cache: false python: - '3.5' env: @@ -15,7 +13,7 @@ env: - TOXENV=pypy - TOXENV=3.5 - TOXENV=pypy3 -install: pip install -U tox +install: travis_retry pip install -U tox script: tox -v -- -v after_success: - .tox/$TRAVIS_PYTHON_VERSION/bin/coveralls From ba75fa0eae79daec62e593af2b98743a2d5e7f4d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 29 Sep 2015 14:43:09 -0700 Subject: [PATCH 0664/1103] Fix for #1847 cannot drain events for Redis. Closes #2827 --- celery/worker/loops.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/worker/loops.py b/celery/worker/loops.py index 2605fda6c..223c15378 100644 --- a/celery/worker/loops.py +++ b/celery/worker/loops.py @@ -50,7 +50,8 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos, # consumer.consume() may have prefetched up to our # limit - drain an event so we are in a clean state # prior to starting our event loop. - connection.drain_events() + if connection.transport.driver_type == 'amqp': + hub.call_soon(connection.drain_events) # FIXME: Use loop.run_forever # Tried and works, but no time to test properly before release. From 6c39ebb82a1b90097f2cf6880b7bb282469dc573 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 29 Sep 2015 17:11:46 -0700 Subject: [PATCH 0665/1103] Tests passing --- celery/tests/worker/test_loops.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/celery/tests/worker/test_loops.py b/celery/tests/worker/test_loops.py index 496cffc60..f70ccf41b 100644 --- a/celery/tests/worker/test_loops.py +++ b/celery/tests/worker/test_loops.py @@ -17,7 +17,8 @@ class X(object): - def __init__(self, app, heartbeat=None, on_task_message=None): + def __init__(self, app, heartbeat=None, on_task_message=None, + transport_driver_type=None): hub = Hub() ( self.obj, @@ -43,6 +44,8 @@ def __init__(self, app, heartbeat=None, on_task_message=None): self.consumer.callbacks = [] self.obj.strategies = {} self.connection.connection_errors = (socket.error,) + if transport_driver_type: + self.connection.transport.driver_type = transport_driver_type self.hub.readers = {} self.hub.writers = {} self.hub.consolidate = set() @@ -121,8 +124,10 @@ def add(x, y): self.add = add def test_drain_after_consume(self): - x, _ = get_task_callback(self.app) - x.connection.drain_events.assert_called_with() + x, _ = get_task_callback(self.app, transport_driver_type='amqp') + self.assertIn( + x.connection.drain_events, [p.fun for p in x.hub._ready], + ) def test_setup_heartbeat(self): x = X(self.app, heartbeat=10) From 1a953d6aa2d25c4a7b84a133c1560b45e168b539 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 29 Sep 2015 18:57:04 -0700 Subject: [PATCH 0666/1103] Docs: Use better Pygments highlighters --- celery/app/task.py | 2 +- celery/bin/multi.py | 54 ++-- celery/platforms.py | 2 +- celery/result.py | 2 +- celery/tests/security/test_security.py | 2 +- celery/utils/serialization.py | 2 +- docs/configuration.rst | 14 +- docs/contributing.rst | 54 ++-- docs/django/first-steps-with-django.rst | 16 +- docs/faq.rst | 16 +- docs/getting-started/brokers/beanstalk.rst | 2 +- docs/getting-started/brokers/couchdb.rst | 2 +- docs/getting-started/brokers/django.rst | 2 +- docs/getting-started/brokers/ironmq.rst | 2 +- docs/getting-started/brokers/mongodb.rst | 2 +- docs/getting-started/brokers/rabbitmq.rst | 30 +- docs/getting-started/brokers/redis.rst | 2 +- docs/getting-started/brokers/sqs.rst | 2 +- .../first-steps-with-celery.rst | 16 +- docs/getting-started/next-steps.rst | 135 ++++++--- docs/history/changelog-1.0.rst | 18 +- docs/history/changelog-2.0.rst | 24 +- docs/history/changelog-2.1.rst | 26 +- docs/history/changelog-2.2.rst | 8 +- docs/history/changelog-2.3.rst | 2 +- docs/history/changelog-2.4.rst | 2 +- docs/history/changelog-2.5.rst | 4 +- docs/history/changelog-3.0.rst | 16 +- docs/history/changelog-3.1.rst | 10 +- docs/includes/installation.txt | 2 +- docs/internals/guide.rst | 2 +- docs/internals/protocol.rst | 34 ++- docs/reference/celery.rst | 4 +- docs/tutorials/daemonizing.rst | 8 +- docs/tutorials/debugging.rst | 2 +- docs/userguide/application.rst | 20 +- docs/userguide/calling.rst | 16 +- docs/userguide/canvas.rst | 280 +++++++++++++----- docs/userguide/concurrency/eventlet.rst | 2 +- docs/userguide/extending.rst | 12 +- docs/userguide/monitoring.rst | 62 ++-- docs/userguide/optimizing.rst | 4 +- docs/userguide/periodic-tasks.rst | 22 +- docs/userguide/remote-tasks.rst | 26 +- docs/userguide/routing.rst | 42 ++- docs/userguide/tasks.rst | 28 +- docs/userguide/workers.rst | 120 +++++--- docs/whatsnew-2.5.rst | 10 +- docs/whatsnew-3.0.rst | 104 ++++--- docs/whatsnew-3.1.rst | 46 +-- 50 files changed, 791 insertions(+), 524 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index e0779da11..c07ff2729 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -580,7 +580,7 @@ def retry(self, args=None, kwargs=None, exc=None, throw=True, **Example** - .. code-block:: python + .. code-block:: pycon >>> from imaginary_twitter_lib import Twitter >>> from proj.celery import app diff --git a/celery/bin/multi.py b/celery/bin/multi.py index 7429619df..03f9e79b3 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -6,79 +6,79 @@ Examples ======== -.. code-block:: bash +.. code-block:: console - # Single worker with explicit name and events enabled. + $ # Single worker with explicit name and events enabled. $ celery multi start Leslie -E - # Pidfiles and logfiles are stored in the current directory - # by default. Use --pidfile and --logfile argument to change - # this. The abbreviation %n will be expanded to the current - # node name. + $ # Pidfiles and logfiles are stored in the current directory + $ # by default. Use --pidfile and --logfile argument to change + $ # this. The abbreviation %n will be expanded to the current + $ # node name. $ celery multi start Leslie -E --pidfile=/var/run/celery/%n.pid --logfile=/var/log/celery/%n%I.log - # You need to add the same arguments when you restart, - # as these are not persisted anywhere. + $ # You need to add the same arguments when you restart, + $ # as these are not persisted anywhere. $ celery multi restart Leslie -E --pidfile=/var/run/celery/%n.pid --logfile=/var/run/celery/%n%I.log - # To stop the node, you need to specify the same pidfile. + $ # To stop the node, you need to specify the same pidfile. $ celery multi stop Leslie --pidfile=/var/run/celery/%n.pid - # 3 workers, with 3 processes each + $ # 3 workers, with 3 processes each $ celery multi start 3 -c 3 celery worker -n celery1@myhost -c 3 celery worker -n celery2@myhost -c 3 celery worker -n celery3@myhost -c 3 - # start 3 named workers + $ # start 3 named workers $ celery multi start image video data -c 3 celery worker -n image@myhost -c 3 celery worker -n video@myhost -c 3 celery worker -n data@myhost -c 3 - # specify custom hostname + $ # specify custom hostname $ celery multi start 2 --hostname=worker.example.com -c 3 celery worker -n celery1@worker.example.com -c 3 celery worker -n celery2@worker.example.com -c 3 - # specify fully qualified nodenames + $ # specify fully qualified nodenames $ celery multi start foo@worker.example.com bar@worker.example.com -c 3 - # fully qualified nodenames but using the current hostname + $ # fully qualified nodenames but using the current hostname $ celery multi start foo@%h bar@%h - # Advanced example starting 10 workers in the background: - # * Three of the workers processes the images and video queue - # * Two of the workers processes the data queue with loglevel DEBUG - # * the rest processes the default' queue. + $ # Advanced example starting 10 workers in the background: + $ # * Three of the workers processes the images and video queue + $ # * Two of the workers processes the data queue with loglevel DEBUG + $ # * the rest processes the default' queue. $ celery multi start 10 -l INFO -Q:1-3 images,video -Q:4,5 data -Q default -L:4,5 DEBUG - # You can show the commands necessary to start the workers with - # the 'show' command: + $ # You can show the commands necessary to start the workers with + $ # the 'show' command: $ celery multi show 10 -l INFO -Q:1-3 images,video -Q:4,5 data -Q default -L:4,5 DEBUG - # Additional options are added to each celery worker' comamnd, - # but you can also modify the options for ranges of, or specific workers + $ # Additional options are added to each celery worker' comamnd, + $ # but you can also modify the options for ranges of, or specific workers - # 3 workers: Two with 3 processes, and one with 10 processes. + $ # 3 workers: Two with 3 processes, and one with 10 processes. $ celery multi start 3 -c 3 -c:1 10 celery worker -n celery1@myhost -c 10 celery worker -n celery2@myhost -c 3 celery worker -n celery3@myhost -c 3 - # can also specify options for named workers + $ # can also specify options for named workers $ celery multi start image video data -c 3 -c:image 10 celery worker -n image@myhost -c 10 celery worker -n video@myhost -c 3 celery worker -n data@myhost -c 3 - # ranges and lists of workers in options is also allowed: - # (-c:1-3 can also be written as -c:1,2,3) + $ # ranges and lists of workers in options is also allowed: + $ # (-c:1-3 can also be written as -c:1,2,3) $ celery multi start 5 -c 3 -c:1-3 10 celery worker -n celery1@myhost -c 10 celery worker -n celery2@myhost -c 10 @@ -86,7 +86,7 @@ celery worker -n celery4@myhost -c 3 celery worker -n celery5@myhost -c 3 - # lists also works with named workers + $ # lists also works with named workers $ celery multi start foo bar baz xuzzy -c 3 -c:foo,bar,baz 10 celery worker -n foo@myhost -c 10 celery worker -n bar@myhost -c 10 diff --git a/celery/platforms.py b/celery/platforms.py index a665e7f48..047270406 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -515,7 +515,7 @@ class Signals(object): **Examples**: - .. code-block:: python + .. code-block:: pycon >>> from celery.platforms import signals diff --git a/celery/result.py b/celery/result.py index df8880d11..12c01d121 100644 --- a/celery/result.py +++ b/celery/result.py @@ -219,7 +219,7 @@ def pow2(i): Calling :meth:`collect` would return: - .. code-block:: python + .. code-block:: pycon >>> from celery.result import ResultBase >>> from proj.tasks import A diff --git a/celery/tests/security/test_security.py b/celery/tests/security/test_security.py index 07d594d0a..9cc49e5f6 100644 --- a/celery/tests/security/test_security.py +++ b/celery/tests/security/test_security.py @@ -3,7 +3,7 @@ Generated with: -.. code-block:: bash +.. code-block:: console $ openssl genrsa -des3 -passout pass:test -out key1.key 1024 $ openssl req -new -key key1.key -out key1.csr -passin pass:test diff --git a/celery/utils/serialization.py b/celery/utils/serialization.py index 598e058a4..91a79fc88 100644 --- a/celery/utils/serialization.py +++ b/celery/utils/serialization.py @@ -86,7 +86,7 @@ class UnpickleableExceptionWrapper(Exception): **Example** - .. code-block:: python + .. code-block:: pycon >>> def pickle_it(raising_function): ... try: diff --git a/docs/configuration.rst b/docs/configuration.rst index 73b38a5ab..04cd08dfd 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -434,7 +434,7 @@ Configuring the backend URL To install the redis package use `pip` or `easy_install`: - .. code-block:: bash + .. code-block:: console $ pip install redis @@ -540,7 +540,7 @@ Cassandra backend settings To install the pycassa package use `pip` or `easy_install`: - .. code-block:: bash + .. code-block:: console $ pip install pycassa @@ -636,7 +636,7 @@ Riak backend settings To install the riak package use `pip` or `easy_install`: - .. code-block:: bash + .. code-block:: console $ pip install riak @@ -702,7 +702,7 @@ IronCache backend settings To install the iron_celery package use `pip` or `easy_install`: - .. code-block:: bash + .. code-block:: console $ pip install iron_celery @@ -729,7 +729,7 @@ Couchbase backend settings To install the couchbase package use `pip` or `easy_install`: - .. code-block:: bash + .. code-block:: console $ pip install couchbase @@ -775,7 +775,7 @@ CouchDB backend settings To install the couchbase package use `pip` or `easy_install`: - .. code-block:: bash + .. code-block:: console $ pip install pycouchdb @@ -967,7 +967,7 @@ With the follow settings: The final routing options for ``tasks.add`` will become: -.. code-block:: python +.. code-block:: javascript {"exchange": "cpubound", "routing_key": "tasks.add", diff --git a/docs/contributing.rst b/docs/contributing.rst index 26cc0f04b..a51c54e75 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -214,7 +214,7 @@ spelling or other errors on the website/docs/code. D) Include the output from the `celery report` command: - .. code-block:: bash + .. code-block:: console $ celery -A proj report @@ -402,14 +402,14 @@ is in the Github Guide: `Fork a Repo`_. After you have cloned the repository you should checkout your copy to a directory on your machine: -.. code-block:: bash +.. code-block:: console $ git clone git@github.com:username/celery.git When the repository is cloned enter the directory to set up easy access to upstream changes: -.. code-block:: bash +.. code-block:: console $ cd celery $ git remote add upstream git://github.com/celery/celery.git @@ -418,7 +418,7 @@ to upstream changes: If you need to pull in new changes from upstream you should always use the :option:`--rebase` option to ``git pull``: -.. code-block:: bash +.. code-block:: console git pull --rebase upstream master @@ -448,14 +448,14 @@ A complete list of the dependencies needed are located in Installing the test requirements: -.. code-block:: bash +.. code-block:: console $ pip install -U -r requirements/test.txt When installation of dependencies is complete you can execute the test suite by calling ``nosetests``: -.. code-block:: bash +.. code-block:: console $ nosetests @@ -480,7 +480,7 @@ Some useful options to :program:`nosetests` are: If you want to run the tests for a single test file only you can do so like this: -.. code-block:: bash +.. code-block:: console $ nosetests celery.tests.test_worker.test_worker_job @@ -510,13 +510,13 @@ To calculate test coverage you must first install the :mod:`coverage` module. Installing the :mod:`coverage` module: -.. code-block:: bash +.. code-block:: console $ pip install -U coverage Code coverage in HTML: -.. code-block:: bash +.. code-block:: console $ nosetests --with-coverage --cover-html @@ -525,7 +525,7 @@ The coverage output will then be located at Code coverage in XML (Cobertura-style): -.. code-block:: bash +.. code-block:: console $ nosetests --with-coverage --cover-xml --cover-xml-file=coverage.xml @@ -541,16 +541,16 @@ distribution. To run the tests for all supported Python versions simply execute: -.. code-block:: bash +.. code-block:: console $ tox If you only want to test specific Python versions use the :option:`-e` option: -.. code-block:: bash +.. code-block:: console - $ tox -e py26 + $ tox -e 2.7 Building the documentation -------------------------- @@ -558,14 +558,14 @@ Building the documentation To build the documentation you need to install the dependencies listed in :file:`requirements/docs.txt`: -.. code-block:: bash +.. code-block:: console $ pip install -U -r requirements/docs.txt After these dependencies are installed you should be able to build the docs by running: -.. code-block:: bash +.. code-block:: console $ cd docs $ rm -rf .build @@ -584,7 +584,7 @@ can be found in :file:`requirements/pkgutils.txt`. Installing the dependencies: -.. code-block:: bash +.. code-block:: console $ pip install -U -r requirements/pkgutils.txt @@ -594,14 +594,14 @@ pyflakes & PEP8 To ensure that your changes conform to PEP8 and to run pyflakes execute: -.. code-block:: bash +.. code-block:: console $ make flakecheck To not return a negative exit code when this command fails use the ``flakes`` target instead: -.. code-block:: bash +.. code-block:: console $ make flakes§ @@ -611,7 +611,7 @@ API reference To make sure that all modules have a corresponding section in the API reference please execute: -.. code-block:: bash +.. code-block:: console $ make apicheck $ make indexcheck @@ -628,14 +628,14 @@ and this module is considered part of the public API, use the following steps: Use an existing file as a template: -.. code-block:: bash +.. code-block:: console $ cd docs/reference/ $ cp celery.schedules.rst celery.worker.awesome.rst Edit the file using your favorite editor: -.. code-block:: bash +.. code-block:: console $ vim celery.worker.awesome.rst @@ -645,7 +645,7 @@ Edit the file using your favorite editor: Edit the index using your favorite editor: -.. code-block:: bash +.. code-block:: console $ vim index.rst @@ -654,7 +654,7 @@ Edit the index using your favorite editor: Commit your changes: -.. code-block:: bash +.. code-block:: console # Add the file to git $ git add celery.worker.awesome.rst @@ -838,7 +838,7 @@ that require 3rd party libraries must be added. After you've made changes to this file you need to render the distro :file:`README` file: - .. code-block:: bash + .. code-block:: console $ pip install -U requirements/pkgutils.txt $ make readme @@ -1045,19 +1045,19 @@ the :file:`README` files. There is a script to convert sphinx syntax to generic reStructured Text syntax, and the make target `readme` does this for you: -.. code-block:: bash +.. code-block:: console $ make readme Now commit the changes: -.. code-block:: bash +.. code-block:: console $ git commit -a -m "Bumps version to X.Y.Z" and make a new version tag: -.. code-block:: bash +.. code-block:: console $ git tag vX.Y.Z $ git push --tags diff --git a/docs/django/first-steps-with-django.rst b/docs/django/first-steps-with-django.rst index 10879bc45..4fb551487 100644 --- a/docs/django/first-steps-with-django.rst +++ b/docs/django/first-steps-with-django.rst @@ -55,7 +55,7 @@ first we import absolute imports from the future, so that our from __future__ import absolute_import -Then we set the default :envvar:`DJANGO_SETTINGS_MODULE` +Then we set the default :envvar:`DJANGO_SETTINGS_MODULE` environment variable for the :program:`celery` command-line program: .. code-block:: python @@ -137,14 +137,14 @@ concrete app instance: Using the Django ORM/Cache as a result backend. ----------------------------------------------- -The [``django-celery``](https://github.com/celery/django-celery) library defines result backends that -uses the Django ORM and Django Cache frameworks. +The [``django-celery``](https://github.com/celery/django-celery) library defines +result backends that uses the Django ORM and Django Cache frameworks. To use this with your project you need to follow these four steps: 1. Install the ``django-celery`` library: - .. code-block:: bash + .. code-block:: console $ pip install django-celery @@ -159,13 +159,13 @@ To use this with your project you need to follow these four steps: If you are using south_ for schema migrations, you'll want to: - .. code-block:: bash + .. code-block:: console $ python manage.py migrate djcelery For those who are not using south, a normal ``syncdb`` will work: - .. code-block:: bash + .. code-block:: console $ python manage.py syncdb @@ -212,7 +212,7 @@ as a daemon - see :ref:`daemonizing` - but for testing and development it is useful to be able to start a worker instance by using the ``celery worker`` manage command, much as you would use Django's runserver: -.. code-block:: bash +.. code-block:: console $ celery -A proj worker -l info @@ -220,7 +220,7 @@ development it is useful to be able to start a worker instance by using the For a complete listing of the command-line options available, use the help command: -.. code-block:: bash +.. code-block:: console $ celery help diff --git a/docs/faq.rst b/docs/faq.rst index 84598faa8..4ca99c601 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -306,7 +306,7 @@ Why aren't my tasks processed? **Answer:** With RabbitMQ you can see how many consumers are currently receiving tasks by running the following command: -.. code-block:: bash +.. code-block:: console $ rabbitmqctl list_queues -p name messages consumers Listing queues ... @@ -366,13 +366,13 @@ How do I purge all waiting tasks? **Answer:** You can use the ``celery purge`` command to purge all configured task queues: -.. code-block:: bash +.. code-block:: console $ celery -A proj purge or programatically: -.. code-block:: python +.. code-block:: pycon >>> from proj.celery import app >>> app.control.purge() @@ -381,7 +381,7 @@ or programatically: If you only want to purge messages from a specific queue you have to use the AMQP API or the :program:`celery amqp` utility: -.. code-block:: bash +.. code-block:: console $ celery -A proj amqp queue.purge @@ -523,7 +523,7 @@ setting. If you don't use the results for a task, make sure you set the `ignore_result` option: -.. code-block python +.. code-block:: python @app.task(ignore_result=True) def mytask(): @@ -705,7 +705,7 @@ control commands will be received in round-robin between them. To work around this you can explicitly set the nodename for every worker using the :option:`-n` argument to :mod:`~celery.bin.worker`: -.. code-block:: bash +.. code-block:: console $ celery -A proj worker -n worker1@%h $ celery -A proj worker -n worker2@%h @@ -842,9 +842,9 @@ task so the task will not run again. Identifying the type of process is easier if you have installed the ``setproctitle`` module: -.. code-block:: bash +.. code-block:: console - pip install setproctitle + $ pip install setproctitle With this library installed you will be able to see the type of process in ps listings, but the worker must be restarted for this to take effect. diff --git a/docs/getting-started/brokers/beanstalk.rst b/docs/getting-started/brokers/beanstalk.rst index 4854310a0..4f0ed7df5 100644 --- a/docs/getting-started/brokers/beanstalk.rst +++ b/docs/getting-started/brokers/beanstalk.rst @@ -22,7 +22,7 @@ For the Beanstalk support you have to install additional dependencies. You can install both Celery and these dependencies in one go using the ``celery[beanstalk]`` :ref:`bundle `: -.. code-block:: bash +.. code-block:: console $ pip install -U celery[beanstalk] diff --git a/docs/getting-started/brokers/couchdb.rst b/docs/getting-started/brokers/couchdb.rst index d731ef061..8708fbcf7 100644 --- a/docs/getting-started/brokers/couchdb.rst +++ b/docs/getting-started/brokers/couchdb.rst @@ -20,7 +20,7 @@ For the CouchDB support you have to install additional dependencies. You can install both Celery and these dependencies in one go using the ``celery[couchdb]`` :ref:`bundle `: -.. code-block:: bash +.. code-block:: console $ pip install -U celery[couchdb] diff --git a/docs/getting-started/brokers/django.rst b/docs/getting-started/brokers/django.rst index b36f40687..f6c0d6b2b 100644 --- a/docs/getting-started/brokers/django.rst +++ b/docs/getting-started/brokers/django.rst @@ -34,7 +34,7 @@ configuration values. #. Sync your database schema: -.. code-block:: bash +.. code-block:: console $ python manage.py syncdb diff --git a/docs/getting-started/brokers/ironmq.rst b/docs/getting-started/brokers/ironmq.rst index 49ddcf46f..7fa8e2f31 100644 --- a/docs/getting-started/brokers/ironmq.rst +++ b/docs/getting-started/brokers/ironmq.rst @@ -11,7 +11,7 @@ Installation For IronMQ support, you'll need the [iron_celery](http://github.com/iron-io/iron_celery) library: -.. code-block:: bash +.. code-block:: console $ pip install iron_celery diff --git a/docs/getting-started/brokers/mongodb.rst b/docs/getting-started/brokers/mongodb.rst index 394736893..96c396c94 100644 --- a/docs/getting-started/brokers/mongodb.rst +++ b/docs/getting-started/brokers/mongodb.rst @@ -20,7 +20,7 @@ For the MongoDB support you have to install additional dependencies. You can install both Celery and these dependencies in one go using the ``celery[mongodb]`` :ref:`bundle `: -.. code-block:: bash +.. code-block:: console $ pip install -U celery[mongodb] diff --git a/docs/getting-started/brokers/rabbitmq.rst b/docs/getting-started/brokers/rabbitmq.rst index 2b55670ce..f5c077493 100644 --- a/docs/getting-started/brokers/rabbitmq.rst +++ b/docs/getting-started/brokers/rabbitmq.rst @@ -12,9 +12,11 @@ Installation & Configuration RabbitMQ is the default broker so it does not require any additional dependencies or initial configuration, other than the URL location of -the broker instance you want to use:: +the broker instance you want to use: - >>> BROKER_URL = 'amqp://guest:guest@localhost:5672//' +.. code-block:: python + + BROKER_URL = 'amqp://guest:guest@localhost:5672//' For a description of broker URLs and a full list of the various broker configuration options available to Celery, @@ -46,19 +48,19 @@ Setting up RabbitMQ To use celery we need to create a RabbitMQ user, a virtual host and allow that user access to that virtual host: -.. code-block:: bash +.. code-block:: console $ sudo rabbitmqctl add_user myuser mypassword -.. code-block:: bash +.. code-block:: console $ sudo rabbitmqctl add_vhost myvhost -.. code-block:: bash +.. code-block:: console $ sudo rabbitmqctl set_user_tags myuser mytag -.. code-block:: bash +.. code-block:: console $ sudo rabbitmqctl set_permissions -p myvhost myuser ".*" ".*" ".*" @@ -79,13 +81,13 @@ shiny package management system for OS X. First, install homebrew using the one-line command provided by the `Homebrew documentation`_: -.. code-block:: bash +.. code-block:: console ruby -e "$(curl -fsSL https://raw.github.com/Homebrew/homebrew/go/install)" Finally, we can install rabbitmq using :program:`brew`: -.. code-block:: bash +.. code-block:: console $ brew install rabbitmq @@ -96,7 +98,7 @@ Finally, we can install rabbitmq using :program:`brew`: After you have installed rabbitmq with brew you need to add the following to your path to be able to start and stop the broker. Add it to your .bash_profile or .profile -.. code-block:: bash +.. code-block:: console `PATH=$PATH:/usr/local/sbin` @@ -109,7 +111,7 @@ to communicate with nodes. Use the :program:`scutil` command to permanently set your host name: -.. code-block:: bash +.. code-block:: console $ sudo scutil --set HostName myhost.local @@ -121,7 +123,7 @@ back into an IP address:: If you start the rabbitmq server, your rabbit node should now be `rabbit@myhost`, as verified by :program:`rabbitmqctl`: -.. code-block:: bash +.. code-block:: console $ sudo rabbitmqctl status Status of node rabbit@myhost ... @@ -146,21 +148,21 @@ Starting/Stopping the RabbitMQ server To start the server: -.. code-block:: bash +.. code-block:: console $ sudo rabbitmq-server you can also run it in the background by adding the :option:`-detached` option (note: only one dash): -.. code-block:: bash +.. code-block:: console $ sudo rabbitmq-server -detached Never use :program:`kill` to stop the RabbitMQ server, but rather use the :program:`rabbitmqctl` command: -.. code-block:: bash +.. code-block:: console $ sudo rabbitmqctl stop diff --git a/docs/getting-started/brokers/redis.rst b/docs/getting-started/brokers/redis.rst index 485d15abb..21726b6d1 100644 --- a/docs/getting-started/brokers/redis.rst +++ b/docs/getting-started/brokers/redis.rst @@ -13,7 +13,7 @@ For the Redis support you have to install additional dependencies. You can install both Celery and these dependencies in one go using the ``celery[redis]`` :ref:`bundle `: -.. code-block:: bash +.. code-block:: console $ pip install -U celery[redis] diff --git a/docs/getting-started/brokers/sqs.rst b/docs/getting-started/brokers/sqs.rst index 9f2331471..b9ec699cf 100644 --- a/docs/getting-started/brokers/sqs.rst +++ b/docs/getting-started/brokers/sqs.rst @@ -18,7 +18,7 @@ Installation For the Amazon SQS support you have to install the `boto`_ library: -.. code-block:: bash +.. code-block:: console $ pip install -U boto diff --git a/docs/getting-started/first-steps-with-celery.rst b/docs/getting-started/first-steps-with-celery.rst index fd152df73..23d1df848 100644 --- a/docs/getting-started/first-steps-with-celery.rst +++ b/docs/getting-started/first-steps-with-celery.rst @@ -56,7 +56,7 @@ Detailed information about using RabbitMQ with Celery: If you are using Ubuntu or Debian install RabbitMQ by executing this command: -.. code-block:: bash +.. code-block:: console $ sudo apt-get install rabbitmq-server @@ -111,7 +111,7 @@ Installing Celery Celery is on the Python Package Index (PyPI), so it can be installed with standard Python tools like ``pip`` or ``easy_install``: -.. code-block:: bash +.. code-block:: console $ pip install celery @@ -157,7 +157,7 @@ Running the celery worker server You now run the worker by executing our program with the ``worker`` argument: -.. code-block:: bash +.. code-block:: console $ celery -A tasks worker --loglevel=info @@ -173,13 +173,13 @@ for more information). For a complete listing of the command-line options available, do: -.. code-block:: bash +.. code-block:: console $ celery worker --help There are also several other commands available, and help is also available: -.. code-block:: bash +.. code-block:: console $ celery help @@ -344,7 +344,7 @@ current directory or on the Python path, it could look like this: To verify that your configuration file works properly, and doesn't contain any syntax errors, you can try to import it: -.. code-block:: bash +.. code-block:: console $ python -m celeryconfig @@ -377,7 +377,7 @@ If you are using RabbitMQ or Redis as the broker then you can also direct the workers to set a new rate limit for the task at runtime: -.. code-block:: bash +.. code-block:: console $ celery -A tasks control rate_limit tasks.add 10/m worker@example.com: OK @@ -411,7 +411,7 @@ Worker does not start: Permission Error A simple workaround is to create a symbolic link: - .. code-block:: bash + .. code-block:: console # ln -s /run/shm /dev/shm diff --git a/docs/getting-started/next-steps.rst b/docs/getting-started/next-steps.rst index 25a2de336..d93ec6e98 100644 --- a/docs/getting-started/next-steps.rst +++ b/docs/getting-started/next-steps.rst @@ -72,7 +72,7 @@ Starting the worker The :program:`celery` program can be used to start the worker (you need to run the worker in the directory above proj): -.. code-block:: bash +.. code-block:: console $ celery -A proj worker -l info @@ -128,7 +128,7 @@ and emulating priorities, all described in the :ref:`Routing Guide You can get a complete list of command-line arguments by passing in the `--help` flag: -.. code-block:: bash +.. code-block:: console $ celery worker --help @@ -149,7 +149,7 @@ described in detail in the :ref:`daemonization tutorial `. The daemonization scripts uses the :program:`celery multi` command to start one or more workers in the background: -.. code-block:: bash +.. code-block:: console $ celery multi start w1 -A proj -l info celery multi v3.1.1 (Cipater) @@ -158,7 +158,7 @@ start one or more workers in the background: You can restart it too: -.. code-block:: bash +.. code-block:: console $ celery multi restart w1 -A proj -l info celery multi v3.1.1 (Cipater) @@ -173,7 +173,7 @@ You can restart it too: or stop it: -.. code-block:: bash +.. code-block:: console $ celery multi stop w1 -A proj -l info @@ -181,7 +181,7 @@ The ``stop`` command is asynchronous so it will not wait for the worker to shutdown. You will probably want to use the ``stopwait`` command instead which will ensure all currently executing tasks is completed: -.. code-block:: bash +.. code-block:: console $ celery multi stopwait w1 -A proj -l info @@ -196,7 +196,7 @@ By default it will create pid and log files in the current directory, to protect against multiple workers launching on top of each other you are encouraged to put these in a dedicated directory: -.. code-block:: bash +.. code-block:: console $ mkdir -p /var/run/celery $ mkdir -p /var/log/celery @@ -207,7 +207,7 @@ With the multi command you can start multiple workers, and there is a powerful command-line syntax to specify arguments for different workers too, e.g: -.. code-block:: bash +.. code-block:: console $ celery multi start 10 -A proj -l info -Q:1-3 images,video -Q:4,5 data \ -Q default -L:4,5 debug @@ -250,17 +250,23 @@ for larger projects. Calling Tasks ============= -You can call a task using the :meth:`delay` method:: +You can call a task using the :meth:`delay` method: + +.. code-block:: pycon >>> add.delay(2, 2) This method is actually a star-argument shortcut to another method called -:meth:`apply_async`:: +:meth:`apply_async`: + +.. code-block:: pycon >>> add.apply_async((2, 2)) The latter enables you to specify execution options like the time to run -(countdown), the queue it should be sent to and so on:: +(countdown), the queue it should be sent to and so on: + +.. code-block:: pycon >>> add.apply_async((2, 2), queue='lopri', countdown=10) @@ -268,7 +274,9 @@ In the above example the task will be sent to a queue named ``lopri`` and the task will execute, at the earliest, 10 seconds after the message was sent. Applying the task directly will execute the task in the current process, -so that no message is sent:: +so that no message is sent: + +.. code-block:: pycon >>> add(2, 2) 4 @@ -296,22 +304,31 @@ have. Also note that result backends are not used for monitoring tasks and work for that Celery uses dedicated event messages (see :ref:`guide-monitoring`). If you have a result backend configured you can retrieve the return -value of a task:: +value of a task: + +.. code-block:: pycon >>> res = add.delay(2, 2) >>> res.get(timeout=1) 4 -You can find the task's id by looking at the :attr:`id` attribute:: +You can find the task's id by looking at the :attr:`id` attribute: + +.. code-block:: pycon >>> res.id d6b3aea2-fb9b-4ebc-8da4-848818db9114 You can also inspect the exception and traceback if the task raised an -exception, in fact ``result.get()`` will propagate any errors by default:: +exception, in fact ``result.get()`` will propagate any errors by default: + +.. code-block:: pycon >>> res = add.delay(2) >>> res.get(timeout=1) + +.. code-block:: pytb + Traceback (most recent call last): File "", line 1, in File "/opt/devel/celery/celery/result.py", line 113, in get @@ -321,7 +338,9 @@ exception, in fact ``result.get()`` will propagate any errors by default:: TypeError: add() takes exactly 2 arguments (1 given) If you don't wish for the errors to propagate then you can disable that -by passing the ``propagate`` argument:: +by passing the ``propagate`` argument: + +.. code-block:: pycon >>> res.get(propagate=False) TypeError('add() takes exactly 2 arguments (1 given)',) @@ -337,7 +356,9 @@ use the corresponding methods on the result instance:: False So how does it know if the task has failed or not? It can find out by looking -at the tasks *state*:: +at the tasks *state*: + +.. code-block:: pycon >>> res.state 'FAILURE' @@ -353,7 +374,9 @@ The started state is a special state that is only recorded if the The pending state is actually not a recorded state, but rather the default state for any task id that is unknown, which you can see -from this example:: +from this example: + +.. code-block:: pycon >>> from proj.celery import app @@ -387,12 +410,16 @@ invocation in a way such that it can be passed to functions or even serialized and sent across the wire. You can create a signature for the ``add`` task using the arguments ``(2, 2)``, -and a countdown of 10 seconds like this:: +and a countdown of 10 seconds like this: + +.. code-block:: pycon >>> add.signature((2, 2), countdown=10) tasks.add(2, 2) -There is also a shortcut using star arguments:: +There is also a shortcut using star arguments: + +.. code-block:: pycon >>> add.s(2, 2) tasks.add(2, 2) @@ -405,7 +432,9 @@ have the ``delay`` and ``apply_async`` methods. But there is a difference in that the signature may already have an argument signature specified. The ``add`` task takes two arguments, -so a signature specifying two arguments would make a complete signature:: +so a signature specifying two arguments would make a complete signature: + +.. code-block:: pycon >>> s1 = add.s(2, 2) >>> res = s1.delay() @@ -413,13 +442,17 @@ so a signature specifying two arguments would make a complete signature:: 4 But, you can also make incomplete signatures to create what we call -*partials*:: +*partials*: + +.. code-block:: pycon # incomplete partial: add(?, 2) >>> s2 = add.s(2) ``s2`` is now a partial signature that needs another argument to be complete, -and this can be resolved when calling the signature:: +and this can be resolved when calling the signature: + +.. code-block:: pycon # resolves the partial: add(8, 2) >>> res = s2.delay(8) @@ -430,7 +463,9 @@ Here you added the argument 8, which was prepended to the existing argument 2 forming a complete signature of ``add(8, 2)``. Keyword arguments can also be added later, these are then merged with any -existing keyword arguments, but with new arguments taking precedence:: +existing keyword arguments, but with new arguments taking precedence: + +.. code-block:: pycon >>> s3 = add.s(2, 2, debug=True) >>> s3.delay(debug=False) # debug is now False. @@ -484,7 +519,7 @@ A :class:`~celery.group` calls a list of tasks in parallel, and it returns a special result instance that lets you inspect the results as a group, and retrieve the return values in order. -.. code-block:: python +.. code-block:: pycon >>> from celery import group >>> from proj.tasks import add @@ -494,7 +529,7 @@ as a group, and retrieve the return values in order. - Partial group -.. code-block:: python +.. code-block:: pycon >>> g = group(add.s(i) for i in xrange(10)) >>> g(10).get() @@ -506,7 +541,7 @@ Chains Tasks can be linked together so that after one task returns the other is called: -.. code-block:: python +.. code-block:: pycon >>> from celery import chain >>> from proj.tasks import add, mul @@ -518,9 +553,9 @@ is called: or a partial chain: -.. code-block:: python +.. code-block:: pycon - # (? + 4) * 8 + >>> # (? + 4) * 8 >>> g = chain(add.s(4) | mul.s(8)) >>> g(4).get() 64 @@ -528,7 +563,7 @@ or a partial chain: Chains can also be written like this: -.. code-block:: python +.. code-block:: pycon >>> (add.s(4, 4) | mul.s(8))().get() 64 @@ -538,7 +573,7 @@ Chords A chord is a group with a callback: -.. code-block:: python +.. code-block:: pycon >>> from celery import chord >>> from proj.tasks import add, xsum @@ -550,7 +585,7 @@ A chord is a group with a callback: A group chained to another task will be automatically converted to a chord: -.. code-block:: python +.. code-block:: pycon >>> (group(add.s(i, i) for i in xrange(10)) | xsum.s())().get() 90 @@ -571,7 +606,9 @@ Celery supports all of the routing facilities provided by AMQP, but it also supports simple routing where messages are sent to named queues. The :setting:`CELERY_ROUTES` setting enables you to route tasks by name -and keep everything centralized in one location:: +and keep everything centralized in one location: + +.. code-block:: python app.conf.update( CELERY_ROUTES = { @@ -580,7 +617,9 @@ and keep everything centralized in one location:: ) You can also specify the queue at runtime -with the ``queue`` argument to ``apply_async``:: +with the ``queue`` argument to ``apply_async``: + +.. code-block:: pycon >>> from proj.tasks import add >>> add.apply_async((2, 2), queue='hipri') @@ -588,7 +627,7 @@ with the ``queue`` argument to ``apply_async``:: You can then make a worker consume from this queue by specifying the :option:`-Q` option: -.. code-block:: bash +.. code-block:: console $ celery -A proj worker -Q hipri @@ -597,7 +636,7 @@ for example you can make the worker consume from both the default queue, and the ``hipri`` queue, where the default queue is named ``celery`` for historical reasons: -.. code-block:: bash +.. code-block:: console $ celery -A proj worker -Q hipri,celery @@ -615,7 +654,7 @@ you can control and inspect the worker at runtime. For example you can see what tasks the worker is currently working on: -.. code-block:: bash +.. code-block:: console $ celery -A proj inspect active @@ -626,7 +665,7 @@ You can also specify one or more workers to act on the request using the :option:`--destination` option, which is a comma separated list of worker host names: -.. code-block:: bash +.. code-block:: console $ celery -A proj inspect active --destination=celery@example.com @@ -638,47 +677,47 @@ does not change anything in the worker, it only replies information and statistics about what is going on inside the worker. For a list of inspect commands you can execute: -.. code-block:: bash +.. code-block:: console $ celery -A proj inspect --help Then there is the :program:`celery control` command, which contains commands that actually changes things in the worker at runtime: -.. code-block:: bash +.. code-block:: console $ celery -A proj control --help For example you can force workers to enable event messages (used for monitoring tasks and workers): -.. code-block:: bash +.. code-block:: console $ celery -A proj control enable_events When events are enabled you can then start the event dumper to see what the workers are doing: -.. code-block:: bash +.. code-block:: console $ celery -A proj events --dump or you can start the curses interface: -.. code-block:: bash +.. code-block:: console $ celery -A proj events when you're finished monitoring you can disable events again: -.. code-block:: bash +.. code-block:: console $ celery -A proj control disable_events The :program:`celery status` command also uses remote control commands and shows a list of online workers in the cluster: -.. code-block:: bash +.. code-block:: console $ celery -A proj status @@ -693,7 +732,9 @@ All times and dates, internally and in messages uses the UTC timezone. When the worker receives a message, for example with a countdown set it converts that UTC time to local time. If you wish to use a different timezone than the system timezone then you must -configure that using the :setting:`CELERY_TIMEZONE` setting:: +configure that using the :setting:`CELERY_TIMEZONE` setting: + +.. code-block:: python app.conf.CELERY_TIMEZONE = 'Europe/London' @@ -711,7 +752,7 @@ for throughput then you should read the :ref:`Optimizing Guide If you're using RabbitMQ then you should install the :mod:`librabbitmq` module, which is an AMQP client implemented in C: -.. code-block:: bash +.. code-block:: console $ pip install librabbitmq diff --git a/docs/history/changelog-1.0.rst b/docs/history/changelog-1.0.rst index f10ff9451..cf0fdf143 100644 --- a/docs/history/changelog-1.0.rst +++ b/docs/history/changelog-1.0.rst @@ -20,13 +20,13 @@ If you've already used the AMQP backend this means you have to delete the previous definitions: - .. code-block:: bash + .. code-block:: console $ camqadm exchange.delete celeryresults or: - .. code-block:: bash + .. code-block:: console $ python manage.py camqadm exchange.delete celeryresults @@ -506,7 +506,7 @@ Fixes If you're using Celery with Django, you can't use `project.settings` as the settings module name, but the following should work: - .. code-block:: bash + .. code-block:: console $ python manage.py celeryd --settings=settings @@ -534,7 +534,7 @@ Fixes Excellent for deleting queues/bindings/exchanges, experimentation and testing: - .. code-block:: bash + .. code-block:: console $ camqadm 1> help @@ -543,7 +543,7 @@ Fixes When using Django, use the management command instead: - .. code-block:: bash + .. code-block:: console $ python manage.py camqadm 1> help @@ -711,7 +711,7 @@ Backward incompatible changes To launch the periodic task scheduler you have to run celerybeat: - .. code-block:: bash + .. code-block:: console $ celerybeat @@ -720,7 +720,7 @@ Backward incompatible changes If you only have one worker server you can embed it into the worker like this: - .. code-block:: bash + .. code-block:: console $ celeryd --beat # Embed celerybeat in celeryd. @@ -1552,7 +1552,7 @@ arguments, so be sure to flush your task queue before you upgrade. * You can now run the celery daemon by using `manage.py`: - .. code-block:: bash + .. code-block:: console $ python manage.py celeryd @@ -1693,7 +1693,7 @@ arguments, so be sure to flush your task queue before you upgrade. * Now using the Sphinx documentation system, you can build the html documentation by doing: - .. code-block:: bash + .. code-block:: console $ cd docs $ make html diff --git a/docs/history/changelog-2.0.rst b/docs/history/changelog-2.0.rst index 93f7d5a6a..b55afa688 100644 --- a/docs/history/changelog-2.0.rst +++ b/docs/history/changelog-2.0.rst @@ -278,13 +278,13 @@ Documentation If you've already hit this problem you may have to delete the declaration: - .. code-block:: bash + .. code-block:: console $ camqadm exchange.delete celerycrq or: - .. code-block:: bash + .. code-block:: console $ python manage.py camqadm exchange.delete celerycrq @@ -387,7 +387,7 @@ Documentation Use the `-S|--statedb` argument to the worker to enable it: - .. code-block:: bash + .. code-block:: console $ celeryd --statedb=/var/run/celeryd @@ -599,7 +599,7 @@ Backward incompatible changes If you've already used celery with this backend chances are you have to delete the previous declaration: - .. code-block:: bash + .. code-block:: console $ camqadm exchange.delete celeryresults @@ -638,7 +638,7 @@ News If you run `celeryev` with the `-d` switch it will act as an event dumper, simply dumping the events it receives to standard out: - .. code-block:: bash + .. code-block:: console $ celeryev -d -> celeryev: starting capture... @@ -742,7 +742,7 @@ News This feature is added for easily setting up routing using the `-Q` option to the worker: - .. code-block:: bash + .. code-block:: console $ celeryd -Q video, image @@ -887,7 +887,7 @@ News command would make the worker only consume from the `image` and `video` queues: - .. code-block:: bash + .. code-block:: console $ celeryd -Q image,video @@ -916,25 +916,25 @@ News Before you run the tests you need to install the test requirements: - .. code-block:: bash + .. code-block:: console $ pip install -r requirements/test.txt Running all tests: - .. code-block:: bash + .. code-block:: console $ nosetests Specifying the tests to run: - .. code-block:: bash + .. code-block:: console $ nosetests celery.tests.test_task Producing HTML coverage: - .. code-block:: bash + .. code-block:: console $ nosetests --with-coverage3 @@ -947,7 +947,7 @@ News Some examples: - .. code-block:: bash + .. code-block:: console # Advanced example with 10 workers: # * Three of the workers processes the images and video queue diff --git a/docs/history/changelog-2.1.rst b/docs/history/changelog-2.1.rst index 57b898fcd..5d4856c00 100644 --- a/docs/history/changelog-2.1.rst +++ b/docs/history/changelog-2.1.rst @@ -223,7 +223,7 @@ News Example using celeryctl to start consuming from queue "queue", in exchange "exchange", of type "direct" using binding key "key": - .. code-block:: bash + .. code-block:: console $ celeryctl inspect add_consumer queue exchange direct key $ celeryctl inspect cancel_consumer queue @@ -234,7 +234,7 @@ News Another example using :class:`~celery.task.control.inspect`: - .. code-block:: python + .. code-block:: pycon >>> from celery.task.control import inspect >>> inspect.add_consumer(queue="queue", exchange="exchange", @@ -296,7 +296,7 @@ Important Notes To do this use :program:`python` to find the location of this module: - .. code-block:: bash + .. code-block:: console $ python >>> import celery.platform @@ -306,7 +306,7 @@ Important Notes Here the compiled module is in :file:`/opt/devel/celery/celery/`, to remove the offending files do: - .. code-block:: bash + .. code-block:: console $ rm -f /opt/devel/celery/celery/platform.py* @@ -345,13 +345,13 @@ News 1. Create the new database tables: - .. code-block:: bash + .. code-block:: console $ python manage.py syncdb 2. Start the django-celery snapshot camera: - .. code-block:: bash + .. code-block:: console $ python manage.py celerycam @@ -403,7 +403,7 @@ News Some examples: - .. code-block:: bash + .. code-block:: console $ celeryctl apply tasks.add -a '[2, 2]' --countdown=10 @@ -482,7 +482,7 @@ News Example: - .. code-block:: bash + .. code-block:: console $ celeryd -I app1.tasks,app2.tasks @@ -692,7 +692,7 @@ Experimental multi can now be used to start, stop and restart worker nodes: - .. code-block:: bash + .. code-block:: console $ celeryd-multi start jerry elaine george kramer @@ -701,7 +701,7 @@ Experimental use the `--pidfile` and `--logfile` arguments with the `%n` format: - .. code-block:: bash + .. code-block:: console $ celeryd-multi start jerry elaine george kramer \ --logfile=/var/log/celeryd@%n.log \ @@ -709,20 +709,20 @@ Experimental Stopping: - .. code-block:: bash + .. code-block:: console $ celeryd-multi stop jerry elaine george kramer Restarting. The nodes will be restarted one by one as the old ones are shutdown: - .. code-block:: bash + .. code-block:: console $ celeryd-multi restart jerry elaine george kramer Killing the nodes (**WARNING**: Will discard currently executing tasks): - .. code-block:: bash + .. code-block:: console $ celeryd-multi kill jerry elaine george kramer diff --git a/docs/history/changelog-2.2.rst b/docs/history/changelog-2.2.rst index 5db27d0a7..a93613bf7 100644 --- a/docs/history/changelog-2.2.rst +++ b/docs/history/changelog-2.2.rst @@ -666,7 +666,7 @@ Important Notes If you telnet the port specified you will be presented with a ``pdb`` shell: - .. code-block:: bash + .. code-block:: console $ telnet localhost 6900 Connected to localhost. @@ -711,7 +711,7 @@ Important Notes If you would like to remove the old exchange you can do so by executing the following command: - .. code-block:: bash + .. code-block:: console $ camqadm exchange.delete celeryevent @@ -721,7 +721,7 @@ Important Notes Configuration options must appear after the last argument, separated by two dashes: - .. code-block:: bash + .. code-block:: console $ celery worker -l info -I tasks -- broker.host=localhost broker.vhost=/app @@ -924,7 +924,7 @@ News For example: - .. code-block:: bash + .. code-block:: console $ celery worker --config=celeryconfig.py --loader=myloader.Loader diff --git a/docs/history/changelog-2.3.rst b/docs/history/changelog-2.3.rst index 90a4454f5..d38dd51c9 100644 --- a/docs/history/changelog-2.3.rst +++ b/docs/history/changelog-2.3.rst @@ -287,7 +287,7 @@ News Example use: - .. code-block:: bash + .. code-block:: console $ celery multi start 4 -c 2 -- broker.host=amqp.example.com \ broker.vhost=/ \ diff --git a/docs/history/changelog-2.4.rst b/docs/history/changelog-2.4.rst index 64866b87c..1cfbd7f4e 100644 --- a/docs/history/changelog-2.4.rst +++ b/docs/history/changelog-2.4.rst @@ -205,7 +205,7 @@ Important Notes Also, programs now support the :option:`-b|--broker` option to specify a broker URL on the command-line: - .. code-block:: bash + .. code-block:: console $ celery worker -b redis://localhost diff --git a/docs/history/changelog-2.5.rst b/docs/history/changelog-2.5.rst index 133ee8742..77936ab34 100644 --- a/docs/history/changelog-2.5.rst +++ b/docs/history/changelog-2.5.rst @@ -94,7 +94,7 @@ News Example: - .. code-block:: python + .. code-block:: pycon >>> s = add.subtask((5,)) >>> new = s.clone(args=(10,), countdown=5}) @@ -145,7 +145,7 @@ Fixes Like with the worker it is now possible to configure celery settings on the command-line for celery control|inspect - .. code-block:: bash + .. code-block:: console $ celery inspect -- broker.pool_limit=30 diff --git a/docs/history/changelog-3.0.rst b/docs/history/changelog-3.0.rst index 25ee5cebb..0dee20c78 100644 --- a/docs/history/changelog-3.0.rst +++ b/docs/history/changelog-3.0.rst @@ -596,7 +596,7 @@ If you're looking for versions prior to 3.0.x you should go to :ref:`history`. - ``subtask.id`` added as an alias to ``subtask['options'].id`` - .. code-block:: python + .. code-block:: pycon >>> s = add.s(2, 2) >>> s.id = 'my-id' @@ -690,9 +690,9 @@ If you're looking for versions prior to 3.0.x you should go to :ref:`history`. Previously it would incorrectly add a regular result instead of a group result, but now this works: - .. code-block:: python + .. code-block:: pycon - # [4 + 4, 4 + 8, 16 + 8] + >>> # [4 + 4, 4 + 8, 16 + 8] >>> res = (add.s(2, 2) | group(add.s(4), add.s(8), add.s(16)))() >>> res >> c1 = (add.s(2) | add.s(4)) >>> c2 = (add.s(8) | add.s(16)) >>> c3 = (c1 | c2) - # 8 + 2 + 4 + 8 + 16 + >>> # 8 + 2 + 4 + 8 + 16 >>> assert c3(8).get() == 38 - Subtasks can now be used with unregistered tasks. @@ -891,7 +891,7 @@ If you're looking for versions prior to 3.0.x you should go to :ref:`history`. Users can force paths to be created by calling the ``create-paths`` subcommand: - .. code-block:: bash + .. code-block:: console $ sudo /etc/init.d/celeryd create-paths @@ -971,7 +971,7 @@ If you're looking for versions prior to 3.0.x you should go to :ref:`history`. Previously calling a chord/group/chain would modify the ids of subtasks so that: - .. code-block:: python + .. code-block:: pycon >>> c = chord([add.s(2, 2), add.s(4, 4)], xsum.s()) >>> c() @@ -1077,7 +1077,7 @@ If you're looking for versions prior to 3.0.x you should go to :ref:`history`. You can do this by executing the following command: - .. code-block:: bash + .. code-block:: console $ python manage.py shell >>> from djcelery.models import PeriodicTask diff --git a/docs/history/changelog-3.1.rst b/docs/history/changelog-3.1.rst index 6e748025d..860580250 100644 --- a/docs/history/changelog-3.1.rst +++ b/docs/history/changelog-3.1.rst @@ -371,7 +371,7 @@ News and if you use the ``librabbitmq`` module you also have to upgrade to librabbitmq 1.5.0: - .. code-block:: bash + .. code-block:: console $ pip install -U librabbitmq @@ -507,9 +507,9 @@ News This means that referring to a number will work when specifying a list of node names and not just for a number range: - .. code-block:: bash + .. code-block:: console - celery multi start A B C D -c:1 4 -c:2-4 8 + $ celery multi start A B C D -c:1 4 -c:2-4 8 In this example ``1`` refers to node A (as it's the first node in the list). @@ -735,7 +735,7 @@ News Example using command-line configuration to set a broker heartbeat from :program:`celery multi`: - .. code-block:: bash + .. code-block:: console $ celery multi start 1 -c3 -- broker.heartbeat=30 @@ -915,7 +915,7 @@ Fixes Example: - .. code-block:: bash + .. code-block:: console $ celery -A proj worker -n foo@%h --logfile=%n.log --statedb=%n.db diff --git a/docs/includes/installation.txt b/docs/includes/installation.txt index 2ab46ab35..a9113ea29 100644 --- a/docs/includes/installation.txt +++ b/docs/includes/installation.txt @@ -26,7 +26,7 @@ You can specify these in your requirements or on the ``pip`` comand-line by using brackets. Multiple bundles can be specified by separating them by commas. -.. code-block:: bash +.. code-block:: console $ pip install "celery[librabbitmq]" diff --git a/docs/internals/guide.rst b/docs/internals/guide.rst index 36e053864..ae35f6347 100644 --- a/docs/internals/guide.rst +++ b/docs/internals/guide.rst @@ -108,7 +108,7 @@ A subclass can change the default value: and the value can be set at instantiation: -.. code-block:: python +.. code-block:: pycon >>> producer = TaskProducer(serializer='msgpack') diff --git a/docs/internals/protocol.rst b/docs/internals/protocol.rst index 285ed9b06..9e6ffd7f8 100644 --- a/docs/internals/protocol.rst +++ b/docs/internals/protocol.rst @@ -125,7 +125,9 @@ Changes from version 1 This is fixed in the new message protocol by specifying a list of signatures, each task will then pop a task off the list - when sending the next message:: + when sending the next message: + + .. code-block:: python execute_task(message) chain = embed['chain'] @@ -138,25 +140,27 @@ Changes from version 1 - ``root_id`` and ``parent_id`` fields helps keep track of workflows. - ``shadow`` lets you specify a different name for logs, monitors - can be used for e.g. meta tasks that calls any function:: + can be used for e.g. meta tasks that calls any function: + + .. code-block:: python - from celery.utils.imports import qualname + from celery.utils.imports import qualname - class PickleTask(Task): - abstract = True + class PickleTask(Task): + abstract = True - def unpack_args(self, fun, args=()): - return fun, args + def unpack_args(self, fun, args=()): + return fun, args - def apply_async(self, args, kwargs, **options): - fun, real_args = self.unpack_args(*args) - return super(PickleTask, self).apply_async( - (fun, real_args, kwargs), shadow=qualname(fun), **options - ) + def apply_async(self, args, kwargs, **options): + fun, real_args = self.unpack_args(*args) + return super(PickleTask, self).apply_async( + (fun, real_args, kwargs), shadow=qualname(fun), **options + ) - @app.task(base=PickleTask) - def call(fun, args, kwargs): - return fun(*args, **kwargs) + @app.task(base=PickleTask) + def call(fun, args, kwargs): + return fun(*args, **kwargs) .. _message-protocol-task-v1: diff --git a/docs/reference/celery.rst b/docs/reference/celery.rst index d244e95e8..449479cfb 100644 --- a/docs/reference/celery.rst +++ b/docs/reference/celery.rst @@ -154,7 +154,7 @@ and creating Celery applications. :keyword force: Force reading configuration immediately. By default the configuration will be read only when required. - .. code-block:: python + .. code-block:: pycon >>> celery.config_from_object("myapp.celeryconfig") @@ -169,7 +169,7 @@ and creating Celery applications. The value of the environment variable must be the name of a module to import. - .. code-block:: python + .. code-block:: pycon >>> os.environ["CELERY_CONFIG_MODULE"] = "myapp.celeryconfig" >>> celery.config_from_envvar("CELERY_CONFIG_MODULE") diff --git a/docs/tutorials/daemonizing.rst b/docs/tutorials/daemonizing.rst index 776de1987..edb7e80b3 100644 --- a/docs/tutorials/daemonizing.rst +++ b/docs/tutorials/daemonizing.rst @@ -52,7 +52,7 @@ must also export them (e.g. ``export DISPLAY=":0"``) instead they can use the :program:`celery multi` utility (or :program:`celery worker --detach`): - .. code-block:: bash + .. code-block:: console $ celery multi start worker1 \ -A proj \ @@ -368,7 +368,7 @@ Troubleshooting If you can't get the init scripts to work, you should try running them in *verbose mode*: -.. code-block:: bash +.. code-block:: console # sh -x /etc/init.d/celeryd start @@ -381,9 +381,9 @@ not be able to see them anywhere. For this situation you can use the :envvar:`C_FAKEFORK` environment variable to skip the daemonization step: -.. code-block:: bash +.. code-block:: console - C_FAKEFORK=1 sh -x /etc/init.d/celeryd start + # C_FAKEFORK=1 sh -x /etc/init.d/celeryd start and now you should be able to see the errors. diff --git a/docs/tutorials/debugging.rst b/docs/tutorials/debugging.rst index 7eb8e5cc9..942d565d8 100644 --- a/docs/tutorials/debugging.rst +++ b/docs/tutorials/debugging.rst @@ -52,7 +52,7 @@ information:: If you telnet the port specified you will be presented with a `pdb` shell: -.. code-block:: bash +.. code-block:: console $ telnet localhost 6900 Connected to localhost. diff --git a/docs/userguide/application.rst b/docs/userguide/application.rst index 5c080ffbe..4f7dcff2d 100644 --- a/docs/userguide/application.rst +++ b/docs/userguide/application.rst @@ -17,7 +17,7 @@ same process space. Let's create one now: -.. code-block:: python +.. code-block:: pycon >>> from celery import Celery >>> app = Celery() @@ -43,7 +43,7 @@ registry*. Whenever you define a task, that task will also be added to the local registry: -.. code-block:: python +.. code-block:: pycon >>> @app.task ... def add(x, y): @@ -93,7 +93,7 @@ the tasks will be named starting with "``tasks``" (the real name of the module): You can specify another name for the main module: -.. code-block:: python +.. code-block:: pycon >>> app = Celery('tasks') >>> app.main @@ -236,7 +236,7 @@ environment variable named :envvar:`CELERY_CONFIG_MODULE`: You can then specify the configuration module to use via the environment: -.. code-block:: bash +.. code-block:: console $ CELERY_CONFIG_MODULE="celeryconfig.prod" celery worker -l info @@ -252,7 +252,7 @@ passwords and API keys. Celery comes with several utilities used for presenting the configuration, one is :meth:`~celery.app.utils.Settings.humanize`: -.. code-block:: python +.. code-block:: pycon >>> app.conf.humanize(with_defaults=False, censored=True) @@ -263,7 +263,7 @@ default keys and values by changing the ``with_defaults`` argument. If you instead want to work with the configuration as a dictionary, then you can use the :meth:`~celery.app.utils.Settings.table` method: -.. code-block:: python +.. code-block:: pycon >>> app.conf.table(with_defaults=False, censored=True) @@ -299,7 +299,7 @@ application has been *finalized*, This example shows how the task is not created until you use the task, or access an attribute (in this case :meth:`repr`): -.. code-block:: python +.. code-block:: pycon >>> @app.task >>> def add(x, y): @@ -410,7 +410,7 @@ In development you can set the :envvar:`CELERY_TRACE_APP` environment variable to raise an exception if the app chain breaks: -.. code-block:: bash +.. code-block:: console $ CELERY_TRACE_APP=1 celery worker -l info @@ -423,7 +423,7 @@ chain breaks: For example, in the beginning it was possible to use any callable as a task: - .. code-block:: python + .. code-block:: pycon def hello(to): return 'hello {0}'.format(to) @@ -507,7 +507,7 @@ and so on. It's also possible to change the default base class for an application by changing its :meth:`@Task` attribute: -.. code-block:: python +.. code-block:: pycon >>> from celery import Celery, Task diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index 36cefe9aa..8042379e3 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -160,7 +160,9 @@ option: In addition, both the ``link`` and ``link_error`` options can be expressed -as a list:: +as a list: + +.. code-block:: python add.apply_async((2, 2), link=[add.s(16), other_task.s()]) @@ -177,7 +179,7 @@ The ETA (estimated time of arrival) lets you set a specific date and time that is the earliest time at which your task will be executed. `countdown` is a shortcut to set eta by seconds into the future. -.. code-block:: python +.. code-block:: pycon >>> result = add.apply_async((2, 2), countdown=3) >>> result.get() # this takes at least 3 seconds to return @@ -195,7 +197,7 @@ While `countdown` is an integer, `eta` must be a :class:`~datetime.datetime` object, specifying an exact date and time (including millisecond precision, and timezone information): -.. code-block:: python +.. code-block:: pycon >>> from datetime import datetime, timedelta @@ -211,7 +213,7 @@ The `expires` argument defines an optional expiry time, either as seconds after task publish, or a specific date and time using :class:`~datetime.datetime`: -.. code-block:: python +.. code-block:: pycon >>> # Task expires after one minute from now. >>> add.apply_async((10, 10), expires=60) @@ -385,7 +387,7 @@ to use when sending a task: Example setting a custom serializer for a single task invocation: -.. code-block:: python +.. code-block:: pycon >>> add.apply_async((10, 10), serializer='json') @@ -442,7 +444,7 @@ publisher: Though this particular example is much better expressed as a group: -.. code-block:: python +.. code-block:: pycon >>> from celery import group @@ -466,7 +468,7 @@ Simple routing (name <-> name) is accomplished using the ``queue`` option:: You can then assign workers to the ``priority.high`` queue by using the workers :option:`-Q` argument: -.. code-block:: bash +.. code-block:: console $ celery -A proj worker -l info -Q celery,priority.high diff --git a/docs/userguide/canvas.rst b/docs/userguide/canvas.rst index 59d19c951..75f7581d8 100644 --- a/docs/userguide/canvas.rst +++ b/docs/userguide/canvas.rst @@ -26,7 +26,9 @@ A :func:`~celery.signature` wraps the arguments, keyword arguments, and executio of a single task invocation in a way such that it can be passed to functions or even serialized and sent across the wire. -- You can create a signature for the ``add`` task using its name like this:: +- You can create a signature for the ``add`` task using its name like this: + + .. code-block:: pycon >>> from celery import signature >>> signature('tasks.add', args=(2, 2), countdown=10) @@ -35,22 +37,30 @@ or even serialized and sent across the wire. This task has a signature of arity 2 (two arguments): ``(2, 2)``, and sets the countdown execution option to 10. -- or you can create one using the task's ``signature`` method:: +- or you can create one using the task's ``signature`` method: + + .. code-block:: pycon >>> add.signature((2, 2), countdown=10) tasks.add(2, 2) -- There is also a shortcut using star arguments:: +- There is also a shortcut using star arguments: + + .. code-block:: pycon >>> add.s(2, 2) tasks.add(2, 2) -- Keyword arguments are also supported:: +- Keyword arguments are also supported: + + .. code-block:: pycon >>> add.s(2, 2, debug=True) tasks.add(2, 2, debug=True) -- From any signature instance you can inspect the different fields:: +- From any signature instance you can inspect the different fields: + + .. code-block:: pycon >>> s = add.signature((2, 2), {'debug': True}, countdown=10) >>> s.args @@ -63,20 +73,27 @@ or even serialized and sent across the wire. - It supports the "Calling API" which means it supports ``delay`` and ``apply_async`` or being called directly. - Calling the signature will execute the task inline in the current process:: + Calling the signature will execute the task inline in the current process: + + .. code-block:: pycon >>> add(2, 2) 4 >>> add.s(2, 2)() 4 - ``delay`` is our beloved shortcut to ``apply_async`` taking star-arguments:: + ``delay`` is our beloved shortcut to ``apply_async`` taking star-arguments: + + .. code-block:: pycon >>> result = add.delay(2, 2) >>> result.get() 4 - ``apply_async`` takes the same arguments as the :meth:`Task.apply_async <@Task.apply_async>` method:: + ``apply_async`` takes the same arguments as the + :meth:`Task.apply_async <@Task.apply_async>` method: + + .. code-block:: pycon >>> add.apply_async(args, kwargs, **options) >>> add.signature(args, kwargs, **options).apply_async() @@ -85,20 +102,26 @@ or even serialized and sent across the wire. >>> add.signature((2, 2), countdown=1).apply_async() - You can't define options with :meth:`~@Task.s`, but a chaining - ``set`` call takes care of that:: + ``set`` call takes care of that: + + .. code-block:: pycon - >>> add.s(2, 2).set(countdown=1) - proj.tasks.add(2, 2) + >>> add.s(2, 2).set(countdown=1) + proj.tasks.add(2, 2) Partials -------- -With a signature, you can execute the task in a worker:: +With a signature, you can execute the task in a worker: + +.. code-block:: pycon >>> add.s(2, 2).delay() >>> add.s(2, 2).apply_async(countdown=1) -Or you can call it directly in the current process:: +Or you can call it directly in the current process: + +.. code-block:: pycon >>> add.s(2, 2)() 4 @@ -106,27 +129,35 @@ Or you can call it directly in the current process:: Specifying additional args, kwargs or options to ``apply_async``/``delay`` creates partials: -- Any arguments added will be prepended to the args in the signature:: +- Any arguments added will be prepended to the args in the signature: + + .. code-block:: pycon - >>> partial = add.s(2) # incomplete signature - >>> partial.delay(4) # 4 + 2 - >>> partial.apply_async((4,)) # same + >>> partial = add.s(2) # incomplete signature + >>> partial.delay(4) # 4 + 2 + >>> partial.apply_async((4,)) # same - Any keyword arguments added will be merged with the kwargs in the signature, - with the new keyword arguments taking precedence:: + with the new keyword arguments taking precedence: - >>> s = add.s(2, 2) - >>> s.delay(debug=True) # -> add(2, 2, debug=True) - >>> s.apply_async(kwargs={'debug': True}) # same + .. code-block:: pycon + + >>> s = add.s(2, 2) + >>> s.delay(debug=True) # -> add(2, 2, debug=True) + >>> s.apply_async(kwargs={'debug': True}) # same - Any options added will be merged with the options in the signature, - with the new options taking precedence:: + with the new options taking precedence: - >>> s = add.signature((2, 2), countdown=10) - >>> s.apply_async(countdown=1) # countdown is now 1 + .. code-block:: pycon + + >>> s = add.signature((2, 2), countdown=10) + >>> s.apply_async(countdown=1) # countdown is now 1 You can also clone signatures to create derivatives: +.. code-block:: pycon + >>> s = add.s(2) proj.tasks.add(2) @@ -142,11 +173,15 @@ Partials are meant to be used with callbacks, any tasks linked or chord callbacks will be applied with the result of the parent task. Sometimes you want to specify a callback that does not take additional arguments, and in that case you can set the signature -to be immutable:: +to be immutable: + +.. code-block:: pycon >>> add.apply_async((2, 2), link=reset_buffers.signature(immutable=True)) -The ``.si()`` shortcut can also be used to create immutable signatures:: +The ``.si()`` shortcut can also be used to create immutable signatures: + +.. code-block:: pycon >>> add.apply_async((2, 2), link=reset_buffers.si()) @@ -157,7 +192,9 @@ so it's not possible to call the signature with partial args/kwargs. In this tutorial I sometimes use the prefix operator `~` to signatures. You probably shouldn't use it in your production code, but it's a handy shortcut - when experimenting in the Python shell:: + when experimenting in the Python shell: + + .. code-block:: pycon >>> ~sig @@ -173,7 +210,9 @@ Callbacks .. versionadded:: 3.0 Callbacks can be added to any task using the ``link`` argument -to ``apply_async``:: +to ``apply_async``: + +.. code-block:: pycon add.apply_async((2, 2), link=other_task.s()) @@ -183,18 +222,24 @@ and it will be applied with the return value of the parent task as argument. As I mentioned earlier, any arguments you add to a signature, will be prepended to the arguments specified by the signature itself! -If you have the signature:: +If you have the signature: + +.. code-block:: pycon >>> sig = add.s(10) -then `sig.delay(result)` becomes:: +then `sig.delay(result)` becomes: + +.. code-block:: pycon >>> add.apply_async(args=(result, 10)) ... Now let's call our ``add`` task with a callback using partial -arguments:: +arguments: + +.. code-block:: pycon >>> add.apply_async((2, 2), link=add.s(8)) @@ -230,7 +275,9 @@ The Primitives a temporary task where a list of arguments is applied to the task. E.g. ``task.map([1, 2])`` results in a single task being called, applying the arguments in order to the task function so - that the result is:: + that the result is: + + .. code-block:: python res = [task(1), task(2)] @@ -238,13 +285,17 @@ The Primitives Works exactly like map except the arguments are applied as ``*args``. For example ``add.starmap([(2, 2), (4, 4)])`` results in a single - task calling:: + task calling: + + .. code-block:: python res = [add(2, 2), add(4, 4)] - ``chunks`` - Chunking splits a long list of arguments into parts, e.g the operation:: + Chunking splits a long list of arguments into parts, e.g the operation: + + .. code-block:: pycon >>> items = zip(xrange(1000), xrange(1000)) # 1000 items >>> add.chunks(items, 10) @@ -263,16 +314,18 @@ Here's some examples: Here's a simple chain, the first task executes passing its return value to the next task in the chain, and so on. - .. code-block:: python + .. code-block:: pycon >>> from celery import chain - # 2 + 2 + 4 + 8 + >>> # 2 + 2 + 4 + 8 >>> res = chain(add.s(2, 2), add.s(4), add.s(8))() >>> res.get() 16 - This can also be written using pipes:: + This can also be written using pipes: + + .. code-block:: pycon >>> (add.s(2, 2) | add.s(4) | add.s(8))().get() 16 @@ -284,15 +337,21 @@ Here's some examples: for example if you don't want the result of the previous task in a chain. In that case you can mark the signature as immutable, so that the arguments - cannot be changed:: + cannot be changed: + + .. code-block:: pycon >>> add.signature((2, 2), immutable=True) - There's also an ``.si`` shortcut for this:: + There's also an ``.si`` shortcut for this: + + .. code-block:: pycon >>> add.si(2, 2) - Now you can create a chain of independent tasks instead:: + Now you can create a chain of independent tasks instead: + + .. code-block:: pycon >>> res = (add.si(2, 2) | add.si(4, 4) | add.s(8, 8))() >>> res.get() @@ -306,7 +365,9 @@ Here's some examples: - Simple group - You can easily create a group of tasks to execute in parallel:: + You can easily create a group of tasks to execute in parallel: + + .. code-block:: pycon >>> from celery import group >>> res = group(add.s(i, i) for i in xrange(10))() @@ -317,7 +378,9 @@ Here's some examples: The chord primitive enables us to add callback to be called when all of the tasks in a group have finished executing, which is often - required for algorithms that aren't embarrassingly parallel:: + required for algorithms that aren't embarrassingly parallel: + + .. code-block:: pycon >>> from celery import chord >>> res = chord((add.s(i, i) for i in xrange(10)), xsum.s())() @@ -329,7 +392,9 @@ Here's some examples: into a list and sent to the ``xsum`` task. The body of a chord can also be immutable, so that the return value - of the group is not passed on to the callback:: + of the group is not passed on to the callback: + + .. code-block:: pycon >>> chord((import_contact.s(c) for c in contacts), ... notify_complete.si(import_id)).apply_async() @@ -338,7 +403,9 @@ Here's some examples: - Blow your mind by combining - Chains can be partial too:: + Chains can be partial too: + + .. code-block:: pycon >>> c1 = (add.s(4) | mul.s(8)) @@ -347,7 +414,9 @@ Here's some examples: >>> res.get() 160 - Which means that you can combine chains:: + Which means that you can combine chains: + + .. code-block:: pycon # ((4 + 16) * 2 + 4) * 8 >>> c2 = (add.s(4, 16) | mul.s(2) | (add.s(4) | mul.s(8))) @@ -357,7 +426,9 @@ Here's some examples: 352 Chaining a group together with another task will automatically - upgrade it to be a chord:: + upgrade it to be a chord: + + .. code-block:: pycon >>> c3 = (group(add.s(i, i) for i in xrange(10)) | xsum.s()) >>> res = c3() @@ -365,7 +436,9 @@ Here's some examples: 90 Groups and chords accepts partial arguments too, so in a chain - the return value of the previous task is forwarded to all tasks in the group:: + the return value of the previous task is forwarded to all tasks in the group: + + .. code-block:: pycon >>> new_user_workflow = (create_user.s() | group( @@ -378,7 +451,9 @@ Here's some examples: If you don't want to forward arguments to the group then - you can make the signatures in the group immutable:: + you can make the signatures in the group immutable: + + .. code-block:: pycon >>> res = (add.s(4, 4) | group(add.si(i, i) for i in xrange(10)))() >>> res.get() @@ -406,7 +481,9 @@ Chains .. versionadded:: 3.0 Tasks can be linked together, which in practice means adding -a callback task:: +a callback task: + +.. code-block:: pycon >>> res = add.apply_async((2, 2), link=mul.s(16)) >>> res.get() @@ -417,7 +494,9 @@ task as the first argument, which in the above case will result in ``mul(4, 16)`` since the result is 4. The results will keep track of any subtasks called by the original task, -and this can be accessed from the result instance:: +and this can be accessed from the result instance: + +.. code-block:: pycon >>> res.children [] @@ -427,7 +506,9 @@ and this can be accessed from the result instance:: The result instance also has a :meth:`~@AsyncResult.collect` method that treats the result as a graph, enabling you to iterate over -the results:: +the results: + +.. code-block:: pycon >>> list(res.collect()) [(, 4), @@ -437,19 +518,25 @@ By default :meth:`~@AsyncResult.collect` will raise an :exc:`~@IncompleteStream` exception if the graph is not fully formed (one of the tasks has not completed yet), but you can get an intermediate representation of the graph -too:: +too: + +.. code-block:: pycon >>> for result, value in res.collect(intermediate=True)): .... You can link together as many tasks as you like, -and signatures can be linked too:: +and signatures can be linked too: + +.. code-block:: pycon >>> s = add.s(2, 2) >>> s.link(mul.s(4)) >>> s.link(log_result.s()) -You can also add *error callbacks* using the ``link_error`` argument:: +You can also add *error callbacks* using the ``link_error`` argument: + +.. code-block:: pycon >>> add.apply_async((2, 2), link_error=log_error.s()) @@ -476,25 +563,29 @@ To make it even easier to link tasks together there is a special signature called :class:`~celery.chain` that lets you chain tasks together: -.. code-block:: python +.. code-block:: pycon >>> from celery import chain >>> from proj.tasks import add, mul - # (4 + 4) * 8 * 10 + >>> # (4 + 4) * 8 * 10 >>> res = chain(add.s(4, 4), mul.s(8), mul.s(10)) proj.tasks.add(4, 4) | proj.tasks.mul(8) | proj.tasks.mul(10) Calling the chain will call the tasks in the current process -and return the result of the last task in the chain:: +and return the result of the last task in the chain: + +.. code-block:: pycon >>> res = chain(add.s(4, 4), mul.s(8), mul.s(10))() >>> res.get() 640 It also sets ``parent`` attributes so that you can -work your way up the chain to get intermediate results:: +work your way up the chain to get intermediate results: + +.. code-block:: pycon >>> res.parent.get() 64 @@ -506,7 +597,9 @@ work your way up the chain to get intermediate results:: -Chains can also be made using the ``|`` (pipe) operator:: +Chains can also be made using the ``|`` (pipe) operator: + +.. code-block:: pycon >>> (add.s(2, 2) | mul.s(8) | mul.s(10)).apply_async() @@ -516,7 +609,7 @@ Graphs In addition you can work with the result graph as a :class:`~celery.datastructures.DependencyGraph`: -.. code-block:: python +.. code-block:: pycon >>> res = chain(add.s(4, 4), mul.s(8), mul.s(10))() @@ -527,7 +620,9 @@ In addition you can work with the result graph as a 285fa253-fcf8-42ef-8b95-0078897e83e6(1) 463afec2-5ed4-4036-b22d-ba067ec64f52(0) -You can even convert these graphs to *dot* format:: +You can even convert these graphs to *dot* format: + +.. code-block:: pycon >>> with open('graph.dot', 'w') as fh: ... res.parent.parent.graph.to_dot(fh) @@ -535,7 +630,7 @@ You can even convert these graphs to *dot* format:: and create images: -.. code-block:: bash +.. code-block:: console $ dot -Tpng graph.dot -o graph.png @@ -550,7 +645,9 @@ Groups A group can be used to execute several tasks in parallel. -The :class:`~celery.group` function takes a list of signatures:: +The :class:`~celery.group` function takes a list of signatures: + +.. code-block:: pycon >>> from celery import group >>> from proj.tasks import add @@ -561,14 +658,18 @@ The :class:`~celery.group` function takes a list of signatures:: If you **call** the group, the tasks will be applied one after one in the current process, and a :class:`~celery.result.GroupResult` instance is returned which can be used to keep track of the results, -or tell how many tasks are ready and so on:: +or tell how many tasks are ready and so on: + +.. code-block:: pycon >>> g = group(add.s(2, 2), add.s(4, 4)) >>> res = g() >>> res.get() [4, 8] -Group also supports iterators:: +Group also supports iterators: + +.. code-block:: pycon >>> group(add.s(i, i) for i in xrange(100))() @@ -580,7 +681,9 @@ Group Results The group task returns a special result too, this result works just like normal task results, except -that it works on the group as a whole:: +that it works on the group as a whole: + +.. code-block:: pycon >>> from celery import group >>> from tasks import add @@ -653,7 +756,7 @@ Chords Tasks used within a chord must *not* ignore their results. If the result backend is disabled for *any* task (header or body) in your chord you should read ":ref:`chord-important-notes`". - + A chord is a task that only executes after all of the tasks in a group have finished executing. @@ -677,7 +780,9 @@ already a standard function): Now you can use a chord to calculate each addition step in parallel, and then -get the sum of the resulting numbers:: +get the sum of the resulting numbers: + +.. code-block:: pycon >>> from celery import chord >>> from tasks import add, tsum @@ -688,9 +793,11 @@ get the sum of the resulting numbers:: This is obviously a very contrived example, the overhead of messaging and -synchronization makes this a lot slower than its Python counterpart:: +synchronization makes this a lot slower than its Python counterpart: + +.. code-block:: pycon - sum(i + i for i in xrange(100)) + >>> sum(i + i for i in xrange(100)) The synchronization step is costly, so you should avoid using chords as much as possible. Still, the chord is a powerful primitive to have in your toolbox @@ -698,7 +805,7 @@ as synchronization is a required step for many parallel algorithms. Let's break the chord expression down: -.. code-block:: python +.. code-block:: pycon >>> callback = tsum.s() >>> header = [add.s(i, i) for i in range(100)] @@ -725,11 +832,14 @@ Errors will propagate to the callback, so the callback will not be executed instead the callback changes to failure state, and the error is set to the :exc:`~@ChordError` exception: -.. code-block:: python +.. code-block:: pycon >>> c = chord([add.s(4, 4), raising_task.s(), add.s(8, 8)]) >>> result = c() >>> result.get() + +.. code-block:: pytb + Traceback (most recent call last): File "", line 1, in File "*/celery/result.py", line 120, in get @@ -833,7 +943,7 @@ They differ from group in that For example using ``map``: -.. code-block:: python +.. code-block:: pycon >>> from proj.tasks import add @@ -848,7 +958,9 @@ is the same as having a task doing: def temp(): return [xsum(range(10)), xsum(range(100))] -and using ``starmap``:: +and using ``starmap``: + +.. code-block:: pycon >>> ~add.starmap(zip(range(10), range(10))) [0, 2, 4, 6, 8, 10, 12, 14, 16, 18] @@ -863,7 +975,9 @@ is the same as having a task doing: Both ``map`` and ``starmap`` are signature objects, so they can be used as other signatures and combined in groups etc., for example -to call the starmap after 10 seconds:: +to call the starmap after 10 seconds: + +.. code-block:: pycon >>> add.starmap(zip(range(10), range(10))).apply_async(countdown=10) @@ -883,14 +997,14 @@ it may considerably increase performance. To create a chunks signature you can use :meth:`@Task.chunks`: -.. code-block:: python +.. code-block:: pycon >>> add.chunks(zip(range(100), range(100)), 10) As with :class:`~celery.group` the act of sending the messages for the chunks will happen in the current process when called: -.. code-block:: python +.. code-block:: pycon >>> from proj.tasks import add @@ -909,16 +1023,22 @@ the chunks will happen in the current process when called: while calling ``.apply_async`` will create a dedicated task so that the individual tasks are applied in a worker -instead:: +instead: + +.. code-block:: pycon >>> add.chunks(zip(range(100), range(100)), 10).apply_async() -You can also convert chunks to a group:: +You can also convert chunks to a group: + +.. code-block:: pycon >>> group = add.chunks(zip(range(100), range(100)), 10).group() and with the group skew the countdown of each task by increments -of one:: +of one: + +.. code-block:: pycon >>> group.skew(start=1, stop=10)() diff --git a/docs/userguide/concurrency/eventlet.rst b/docs/userguide/concurrency/eventlet.rst index aec95fd33..01f98bfb3 100644 --- a/docs/userguide/concurrency/eventlet.rst +++ b/docs/userguide/concurrency/eventlet.rst @@ -42,7 +42,7 @@ Enabling Eventlet You can enable the Eventlet pool by using the ``-P`` option to :program:`celery worker`: -.. code-block:: bash +.. code-block:: console $ celery -A proj worker -P eventlet -c 1000 diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index 1ed9786f0..831532504 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -396,7 +396,9 @@ Attributes Every registered task type has an entry in this mapping, where the value is used to execute an incoming message of this task type (the task execution strategy). This mapping is generated by the Tasks - bootstep when the consumer starts:: + bootstep when the consumer starts: + + .. code-block:: python for name, task in app.tasks.items(): strategies[name] = task.start_strategy(app, consumer) @@ -429,7 +431,9 @@ Attributes .. attribute:: qos The :class:`~kombu.common.QoS` object can be used to change the - task channels current prefetch_count value, e.g:: + task channels current prefetch_count value, e.g: + + .. code-block:: python # increment at next cycle consumer.qos.increment_eventually(1) @@ -473,7 +477,9 @@ Installing Bootsteps ==================== ``app.steps['worker']`` and ``app.steps['consumer']`` can be modified -to add new bootsteps:: +to add new bootsteps: + +.. code-block:: pycon >>> app = Celery() >>> app.steps['worker'].add(MyWorkerStep) # < add class, do not instantiate diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index 2618ab897..1cf04eaca 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -31,13 +31,13 @@ and manage worker nodes (and to some degree tasks). To list all the commands available do: -.. code-block:: bash +.. code-block:: console $ celery help or to get help for a specific command do: -.. code-block:: bash +.. code-block:: console $ celery --help @@ -56,13 +56,13 @@ Commands * **status**: List active nodes in this cluster - .. code-block:: bash + .. code-block:: console $ celery -A proj status * **result**: Show the result of a task - .. code-block:: bash + .. code-block:: console $ celery -A proj result -t tasks.add 4e196aa4-0141-4601-8138-7aa33db0f577 @@ -75,14 +75,14 @@ Commands There is no undo for this operation, and messages will be permanently deleted! - .. code-block:: bash + .. code-block:: console $ celery -A proj purge * **inspect active**: List active tasks - .. code-block:: bash + .. code-block:: console $ celery -A proj inspect active @@ -90,7 +90,7 @@ Commands * **inspect scheduled**: List scheduled ETA tasks - .. code-block:: bash + .. code-block:: console $ celery -A proj inspect scheduled @@ -99,7 +99,7 @@ Commands * **inspect reserved**: List reserved tasks - .. code-block:: bash + .. code-block:: console $ celery -A proj inspect reserved @@ -109,37 +109,37 @@ Commands * **inspect revoked**: List history of revoked tasks - .. code-block:: bash + .. code-block:: console $ celery -A proj inspect revoked * **inspect registered**: List registered tasks - .. code-block:: bash + .. code-block:: console $ celery -A proj inspect registered * **inspect stats**: Show worker statistics (see :ref:`worker-statistics`) - .. code-block:: bash + .. code-block:: console $ celery -A proj inspect stats * **control enable_events**: Enable events - .. code-block:: bash + .. code-block:: console $ celery -A proj control enable_events * **control disable_events**: Disable events - .. code-block:: bash + .. code-block:: console $ celery -A proj control disable_events * **migrate**: Migrate tasks from one broker to another (**EXPERIMENTAL**). - .. code-block:: bash + .. code-block:: console $ celery -A proj migrate redis://localhost amqp://localhost @@ -163,7 +163,7 @@ By default the inspect and control commands operates on all workers. You can specify a single, or a list of workers by using the `--destination` argument: -.. code-block:: bash +.. code-block:: console $ celery -A proj inspect -d w1,w2 reserved @@ -244,25 +244,25 @@ Usage You can use pip to install Flower: -.. code-block:: bash +.. code-block:: console $ pip install flower Running the flower command will start a web-server that you can visit: -.. code-block:: bash +.. code-block:: console $ celery -A proj flower The default port is http://localhost:5555, but you can change this using the `--port` argument: -.. code-block:: bash +.. code-block:: console $ celery -A proj flower --port=5555 Broker URL can also be passed through the `--broker` argument : -.. code-block:: bash +.. code-block:: console $ celery flower --broker=amqp://guest:guest@localhost:5672// or @@ -270,7 +270,7 @@ Broker URL can also be passed through the `--broker` argument : Then, you can visit flower in your web browser : -.. code-block:: bash +.. code-block:: console $ open http://localhost:5555 @@ -296,7 +296,7 @@ probably want to use Flower instead. Starting: -.. code-block:: bash +.. code-block:: console $ celery -A proj events @@ -308,19 +308,19 @@ You should see a screen like: `celery events` is also used to start snapshot cameras (see :ref:`monitoring-snapshots`: -.. code-block:: bash +.. code-block:: console $ celery -A proj events --camera= --frequency=1.0 and it includes a tool to dump events to :file:`stdout`: -.. code-block:: bash +.. code-block:: console $ celery -A proj events --dump For a complete list of options use ``--help``: -.. code-block:: bash +.. code-block:: console $ celery events --help @@ -355,7 +355,7 @@ Inspecting queues Finding the number of tasks in a queue: -.. code-block:: bash +.. code-block:: console $ rabbitmqctl list_queues name messages messages_ready \ messages_unacknowledged @@ -370,13 +370,13 @@ not acknowledged yet (meaning it is in progress, or has been reserved). Finding the number of workers currently consuming from a queue: -.. code-block:: bash +.. code-block:: console $ rabbitmqctl list_queues name consumers Finding the amount of memory allocated to a queue: -.. code-block:: bash +.. code-block:: console $ rabbitmqctl list_queues name memory @@ -399,13 +399,13 @@ Inspecting queues Finding the number of tasks in a queue: -.. code-block:: bash +.. code-block:: console $ redis-cli -h HOST -p PORT -n DATABASE_NUMBER llen QUEUE_NAME The default queue is named `celery`. To get all available queues, invoke: -.. code-block:: bash +.. code-block:: console $ redis-cli -h HOST -p PORT -n DATABASE_NUMBER keys \* @@ -480,7 +480,7 @@ for example if you want to capture state every 2 seconds using the camera ``myapp.Camera`` you run :program:`celery events` with the following arguments: -.. code-block:: bash +.. code-block:: console $ celery -A proj events -c myapp.Camera --frequency=2.0 @@ -520,7 +520,7 @@ about state objects. Now you can use this cam with :program:`celery events` by specifying it with the :option:`-c` option: -.. code-block:: bash +.. code-block:: console $ celery -A proj events -c myapp.DumpCam --frequency=2.0 diff --git a/docs/userguide/optimizing.rst b/docs/userguide/optimizing.rst index e5ab4b312..673951083 100644 --- a/docs/userguide/optimizing.rst +++ b/docs/userguide/optimizing.rst @@ -60,7 +60,7 @@ librabbitmq If you're using RabbitMQ (AMQP) as the broker then you can install the :mod:`librabbitmq` module to use an optimized client written in C: -.. code-block:: bash +.. code-block:: console $ pip install librabbitmq @@ -228,7 +228,7 @@ size is 1MB (can only be changed system wide). You can disable this prefetching behavior by enabling the :option:`-Ofair` worker option: -.. code-block:: bash +.. code-block:: console $ celery -A proj worker -l info -Ofair diff --git a/docs/userguide/periodic-tasks.rst b/docs/userguide/periodic-tasks.rst index a1546bdf5..e103a938c 100644 --- a/docs/userguide/periodic-tasks.rst +++ b/docs/userguide/periodic-tasks.rst @@ -63,7 +63,7 @@ schedule manually. The database scheduler will not reset when timezone related settings change, so you must do this manually: - .. code-block:: bash + .. code-block:: console $ python manage.py shell >>> from djcelery.models import PeriodicTask @@ -283,12 +283,12 @@ sunset, dawn or dusk, you can use the from celery.schedules import solar CELERYBEAT_SCHEDULE = { - # Executes at sunset in Melbourne - 'add-at-melbourne-sunset': { - 'task': 'tasks.add', - 'schedule': solar('sunset', -37.81753, 144.96715), - 'args': (16, 16), - }, + # Executes at sunset in Melbourne + 'add-at-melbourne-sunset': { + 'task': 'tasks.add', + 'schedule': solar('sunset', -37.81753, 144.96715), + 'args': (16, 16), + }, } The arguments are simply: ``solar(event, latitude, longitude)`` @@ -378,7 +378,7 @@ Starting the Scheduler To start the :program:`celery beat` service: -.. code-block:: bash +.. code-block:: console $ celery -A proj beat @@ -387,7 +387,7 @@ workers `-B` option, this is convenient if you will never run more than one worker node, but it's not commonly used and for that reason is not recommended for production use: -.. code-block:: bash +.. code-block:: console $ celery -A proj worker -B @@ -396,7 +396,7 @@ file (named `celerybeat-schedule` by default), so it needs access to write in the current directory, or alternatively you can specify a custom location for this file: -.. code-block:: bash +.. code-block:: console $ celery -A proj beat -s /home/celery/var/run/celerybeat-schedule @@ -418,7 +418,7 @@ which is simply keeping track of the last run times in a local database file `django-celery` also ships with a scheduler that stores the schedule in the Django database: -.. code-block:: bash +.. code-block:: console $ celery -A proj beat -S djcelery.schedulers.DatabaseScheduler diff --git a/docs/userguide/remote-tasks.rst b/docs/userguide/remote-tasks.rst index f9cfa76fb..d8fe3587a 100644 --- a/docs/userguide/remote-tasks.rst +++ b/docs/userguide/remote-tasks.rst @@ -18,13 +18,17 @@ If you need to call into another language, framework or similar, you can do so by using HTTP callback tasks. The HTTP callback tasks uses GET/POST data to pass arguments and returns -result as a JSON response. The scheme to call a task is:: +result as a JSON response. The scheme to call a task is: - GET http://example.com/mytask/?arg1=a&arg2=b&arg3=c +.. code-block:: http -or using POST:: + GET HTTP/1.1 http://example.com/mytask/?arg1=a&arg2=b&arg3=c - POST http://example.com/mytask +or using POST: + +.. code-block:: http + + POST HTTP/1.1 http://example.com/mytask .. note:: @@ -33,11 +37,15 @@ or using POST:: Whether to use GET or POST is up to you and your requirements. The web page should then return a response in the following format -if the execution was successful:: +if the execution was successful: + +.. code-block:: javascript {'status': 'success', 'retval': …} -or if there was an error:: +or if there was an error: + +.. code-block:: javascript {'status': 'failure', 'reason': 'Invalid moon alignment.'} @@ -97,13 +105,17 @@ Calling webhook tasks To call a task you can use the :class:`~celery.task.http.URL` class: +.. code-block:: pycon + >>> from celery.task.http import URL >>> res = URL('http://example.com/multiply').get_async(x=10, y=10) :class:`~celery.task.http.URL` is a shortcut to the :class:`HttpDispatchTask`. You can subclass this to extend the -functionality. +functionality: + +.. code-block:: pycon >>> from celery.task.http import HttpDispatchTask >>> res = HttpDispatchTask.delay( diff --git a/docs/userguide/routing.rst b/docs/userguide/routing.rst index 8b0705436..485a93269 100644 --- a/docs/userguide/routing.rst +++ b/docs/userguide/routing.rst @@ -43,14 +43,14 @@ With this route enabled import feed tasks will be routed to the Now you can start server `z` to only process the feeds queue like this: -.. code-block:: bash +.. code-block:: console user@z:/$ celery -A proj worker -Q feeds You can specify as many queues as you want, so you can make this server process the default queue as well: -.. code-block:: bash +.. code-block:: console user@z:/$ celery -A proj worker -Q feeds,celery @@ -82,7 +82,7 @@ are declared. A queue named `"video"` will be created with the following settings: -.. code-block:: python +.. code-block:: javascript {'exchange': 'video', 'exchange_type': 'direct', @@ -145,13 +145,13 @@ You can also override this using the `routing_key` argument to To make server `z` consume from the feed queue exclusively you can start it with the ``-Q`` option: -.. code-block:: bash +.. code-block:: console user@z:/$ celery -A proj worker -Q feed_tasks --hostname=z@%h Servers `x` and `y` must be configured to consume from the default queue: -.. code-block:: bash +.. code-block:: console user@x:/$ celery -A proj worker -Q default --hostname=x@%h user@y:/$ celery -A proj worker -Q default --hostname=y@%h @@ -159,7 +159,7 @@ Servers `x` and `y` must be configured to consume from the default queue: If you want, you can even have your feed processing worker handle regular tasks as well, maybe in times when there's a lot of work to do: -.. code-block:: python +.. code-block:: console user@z:/$ celery -A proj worker -Q feed_tasks,default --hostname=z@%h @@ -209,7 +209,7 @@ metadata -- like the number of retries or an ETA. This is an example task message represented as a Python dictionary: -.. code-block:: python +.. code-block:: javascript {'task': 'myapp.tasks.add', 'id': '54086c5e-6193-4575-8308-dbab76798756', @@ -365,7 +365,7 @@ but different implementation may not implement all commands. You can write commands directly in the arguments to :program:`celery amqp`, or just start with no arguments to start it in shell-mode: -.. code-block:: bash +.. code-block:: console $ celery -A proj amqp -> connecting to amqp://guest@localhost:5672/. @@ -379,7 +379,7 @@ hit the `tab` key to show a list of possible matches. Let's create a queue you can send messages to: -.. code-block:: bash +.. code-block:: console $ celery -A proj amqp 1> exchange.declare testexchange direct @@ -395,7 +395,9 @@ the routing key ``testkey``. From now on all messages sent to the exchange ``testexchange`` with routing key ``testkey`` will be moved to this queue. You can send a message by -using the ``basic.publish`` command:: +using the ``basic.publish`` command: + +.. code-block:: console 4> basic.publish 'This is a message!' testexchange testkey ok. @@ -405,7 +407,9 @@ Now that the message is sent you can retrieve it again. You can use the (which is alright for maintenance tasks, for services you'd want to use ``basic.consume`` instead) -Pop a message off the queue:: +Pop a message off the queue: + +.. code-block:: console 5> basic.get testqueue {'body': 'This is a message!', @@ -428,12 +432,16 @@ This tag is used to acknowledge the message. Also note that delivery tags are not unique across connections, so in another client the delivery tag `1` might point to a different message than in this channel. -You can acknowledge the message you received using ``basic.ack``:: +You can acknowledge the message you received using ``basic.ack``: + +.. code-block:: console 6> basic.ack 1 ok. -To clean up after our test session you should delete the entities you created:: +To clean up after our test session you should delete the entities you created: + +.. code-block:: console 7> queue.delete testqueue ok. 0 messages deleted. @@ -533,11 +541,15 @@ becomes --> You install router classes by adding them to the :setting:`CELERY_ROUTES` -setting:: +setting: + +.. code-block:: python CELERY_ROUTES = (MyRouter(),) -Router classes can also be added by name:: +Router classes can also be added by name: + +.. code-block:: python CELERY_ROUTES = ('myapp.routers.MyRouter',) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 4a4d22788..9c04b3775 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -73,7 +73,9 @@ these can be specified as arguments to the decorator: if you don't know what that is then please read :ref:`first-steps`. If you're using Django or are still using the "old" module based celery API, - then you can import the task decorator like this:: + then you can import the task decorator like this: + + .. code-block:: python from celery import task @@ -106,7 +108,7 @@ will be generated out of the function name if a custom name is not provided. For example: -.. code-block:: python +.. code-block:: pycon >>> @app.task(name='sum-of-two-numbers') >>> def add(x, y): @@ -119,13 +121,15 @@ A best practice is to use the module name as a namespace, this way names won't collide if there's already a task with that name defined in another module. -.. code-block:: python +.. code-block:: pycon >>> @app.task(name='tasks.add') >>> def add(x, y): ... return x + y -You can tell the name of the task by investigating its name attribute:: +You can tell the name of the task by investigating its name attribute: + +.. code-block:: pycon >>> add.name 'tasks.add' @@ -168,7 +172,7 @@ If you install the app under the name ``project.myapp`` then the tasks module will be imported as ``project.myapp.tasks``, so you must make sure you always import the tasks using the same name: -.. code-block:: python +.. code-block:: pycon >>> from project.myapp.tasks import mytask # << GOOD @@ -177,7 +181,7 @@ so you must make sure you always import the tasks using the same name: The second example will cause the task to be named differently since the worker and the client imports the modules under different names: -.. code-block:: python +.. code-block:: pycon >>> from project.myapp.tasks import mytask >>> mytask.name @@ -894,7 +898,9 @@ The name of the state is usually an uppercase string. As an example you could have a look at :mod:`abortable tasks <~celery.contrib.abortable>` which defines its own custom :state:`ABORTED` state. -Use :meth:`~@Task.update_state` to update a task's state:: +Use :meth:`~@Task.update_state` to update a task's state:. + +.. code-block:: python @app.task(bind=True) def upload_files(self, filenames): @@ -1268,7 +1274,7 @@ All defined tasks are listed in a registry. The registry contains a list of task names and their task classes. You can investigate this registry yourself: -.. code-block:: python +.. code-block:: pycon >>> from proj.celery import app >>> app.tasks @@ -1503,7 +1509,9 @@ that automatically expands some abbreviations in it: article.save() First, an author creates an article and saves it, then the author -clicks on a button that initiates the abbreviation task:: +clicks on a button that initiates the abbreviation task: + +.. code-block:: pycon >>> article = Article.objects.get(id=102) >>> expand_abbreviations.delay(article) @@ -1524,6 +1532,8 @@ re-fetch the article in the task body: article.body.replace('MyCorp', 'My Corporation') article.save() +.. code-block:: pycon + >>> expand_abbreviations(article_id) There might even be performance benefits to this approach, as sending large diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index 75cdf72f6..b12852a8d 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -21,14 +21,14 @@ Starting the worker You can start the worker in the foreground by executing the command: -.. code-block:: bash +.. code-block:: console $ celery -A proj worker -l info For a full list of available command-line options see :mod:`~celery.bin.worker`, or simply do: -.. code-block:: bash +.. code-block:: console $ celery worker --help @@ -36,7 +36,7 @@ You can also start multiple workers on the same machine. If you do so be sure to give a unique name to each individual worker by specifying a host name with the :option:`--hostname|-n` argument: -.. code-block:: bash +.. code-block:: console $ celery -A proj worker --loglevel=INFO --concurrency=10 -n worker1.%h $ celery -A proj worker --loglevel=INFO --concurrency=10 -n worker2.%h @@ -81,7 +81,7 @@ Also as processes can't override the :sig:`KILL` signal, the worker will not be able to reap its children, so make sure to do so manually. This command usually does the trick: -.. code-block:: bash +.. code-block:: console $ ps auxww | grep 'celery worker' | awk '{print $2}' | xargs kill -9 @@ -94,10 +94,10 @@ To restart the worker you should send the `TERM` signal and start a new instance. The easiest way to manage workers for development is by using `celery multi`: - .. code-block:: bash +.. code-block:: console - $ celery multi start 1 -A proj -l info -c4 --pidfile=/var/run/celery/%n.pid - $ celery multi restart 1 --pidfile=/var/run/celery/%n.pid + $ celery multi start 1 -A proj -l info -c4 --pidfile=/var/run/celery/%n.pid + $ celery multi restart 1 --pidfile=/var/run/celery/%n.pid For production deployments you should be using init scripts or other process supervision systems (see :ref:`daemonizing`). @@ -107,7 +107,7 @@ restart the worker using the :sig:`HUP` signal, but note that the worker will be responsible for restarting itself so this is prone to problems and is not recommended in production: -.. code-block:: bash +.. code-block:: console $ kill -HUP $pid @@ -265,14 +265,18 @@ Some remote control commands also have higher-level interfaces using :meth:`~@control.broadcast` in the background, like :meth:`~@control.rate_limit` and :meth:`~@control.ping`. -Sending the :control:`rate_limit` command and keyword arguments:: +Sending the :control:`rate_limit` command and keyword arguments: + +.. code-block:: pycon >>> app.control.broadcast('rate_limit', ... arguments={'task_name': 'myapp.mytask', ... 'rate_limit': '200/m'}) This will send the command asynchronously, without waiting for a reply. -To request a reply you have to use the `reply` argument:: +To request a reply you have to use the `reply` argument: + +.. code-block:: pycon >>> app.control.broadcast('rate_limit', { ... 'task_name': 'myapp.mytask', 'rate_limit': '200/m'}, reply=True) @@ -281,7 +285,9 @@ To request a reply you have to use the `reply` argument:: {'worker3.example.com': 'New rate limit set successfully'}] Using the `destination` argument you can specify a list of workers -to receive the command:: +to receive the command: + +.. code-block:: pycon >>> app.control.broadcast('rate_limit', { ... 'task_name': 'myapp.mytask', @@ -331,7 +337,7 @@ Terminating a task also revokes it. **Example** -:: +.. code-block:: pycon >>> result.revoke() @@ -359,7 +365,7 @@ several tasks at once. **Example** -:: +.. code-block:: pycon >>> app.control.revoke([ ... '7993b0aa-1f0b-4780-9af0-c47c0858b3f2', @@ -385,15 +391,15 @@ of revoked ids will also vanish. If you want to preserve this list between restarts you need to specify a file for these to be stored in by using the `--statedb` argument to :program:`celery worker`: -.. code-block:: bash +.. code-block:: console - celery -A proj worker -l info --statedb=/var/run/celery/worker.state + $ celery -A proj worker -l info --statedb=/var/run/celery/worker.state or if you use :program:`celery multi` you will want to create one file per worker instance so then you can use the `%n` format to expand the current node name: -.. code-block:: bash +.. code-block:: console celery multi start 2 -l info --statedb=/var/run/celery/%n.state @@ -463,7 +469,9 @@ and hard time limits for a task — named ``time_limit``. Example changing the time limit for the ``tasks.crawl_the_web`` task to have a soft time limit of one minute, and a hard time limit of -two minutes:: +two minutes: + +.. code-block:: pycon >>> app.control.time_limit('tasks.crawl_the_web', soft=60, hard=120, reply=True) @@ -484,7 +492,7 @@ Changing rate-limits at runtime Example changing the rate limit for the `myapp.mytask` task to execute at most 200 tasks of that type every minute: -.. code-block:: python +.. code-block:: pycon >>> app.control.rate_limit('myapp.mytask', '200/m') @@ -492,7 +500,7 @@ The above does not specify a destination, so the change request will affect all worker instances in the cluster. If you only want to affect a specific list of workers you can include the ``destination`` argument: -.. code-block:: python +.. code-block:: pycon >>> app.control.rate_limit('myapp.mytask', '200/m', ... destination=['celery@worker1.example.com']) @@ -562,7 +570,7 @@ queue named ``celery``). You can specify what queues to consume from at startup, by giving a comma separated list of queues to the :option:`-Q` option: -.. code-block:: bash +.. code-block:: console $ celery -A proj worker -l info -Q foo,bar,baz @@ -586,7 +594,7 @@ to start consuming from a queue. This operation is idempotent. To tell all workers in the cluster to start consuming from a queue named "``foo``" you can use the :program:`celery control` program: -.. code-block:: bash +.. code-block:: console $ celery -A proj control add_consumer foo -> worker1.local: OK @@ -595,11 +603,13 @@ named "``foo``" you can use the :program:`celery control` program: If you want to specify a specific worker you can use the :option:`--destination`` argument: -.. code-block:: bash +.. code-block:: console $ celery -A proj control add_consumer foo -d worker1.local -The same can be accomplished dynamically using the :meth:`@control.add_consumer` method:: +The same can be accomplished dynamically using the :meth:`@control.add_consumer` method: + +.. code-block:: pycon >>> app.control.add_consumer('foo', reply=True) [{u'worker1.local': {u'ok': u"already consuming from u'foo'"}}] @@ -611,7 +621,9 @@ The same can be accomplished dynamically using the :meth:`@control.add_consumer` By now I have only shown examples using automatic queues, If you need more control you can also specify the exchange, routing_key and -even other options:: +even other options: + +.. code-block:: pycon >>> app.control.add_consumer( ... queue='baz', @@ -637,14 +649,14 @@ control command. To force all workers in the cluster to cancel consuming from a queue you can use the :program:`celery control` program: -.. code-block:: bash +.. code-block:: console $ celery -A proj control cancel_consumer foo The :option:`--destination` argument can be used to specify a worker, or a list of workers, to act on the command: -.. code-block:: bash +.. code-block:: console $ celery -A proj control cancel_consumer foo -d worker1.local @@ -652,7 +664,7 @@ list of workers, to act on the command: You can also cancel consumers programmatically using the :meth:`@control.cancel_consumer` method: -.. code-block:: bash +.. code-block:: console >>> app.control.cancel_consumer('foo', reply=True) [{u'worker1.local': {u'ok': u"no longer consuming from u'foo'"}}] @@ -665,7 +677,7 @@ Queues: List of active queues You can get a list of queues that a worker consumes from by using the :control:`active_queues` control command: -.. code-block:: bash +.. code-block:: console $ celery -A proj inspect active_queues [...] @@ -674,14 +686,16 @@ Like all other remote control commands this also supports the :option:`--destination` argument used to specify which workers should reply to the request: -.. code-block:: bash +.. code-block:: console $ celery -A proj inspect active_queues -d worker1.local [...] This can also be done programmatically by using the -:meth:`@control.inspect.active_queues` method:: +:meth:`@control.inspect.active_queues` method: + +.. code-block:: pycon >>> app.control.inspect().active_queues() [...] @@ -726,7 +740,7 @@ implementations: to install the :mod:`pyinotify` library you have to run the following command: - .. code-block:: bash + .. code-block:: console $ pip install pyinotify @@ -740,7 +754,7 @@ implementations: You can force an implementation by setting the :envvar:`CELERYD_FSNOTIFY` environment variable: -.. code-block:: bash +.. code-block:: console $ env CELERYD_FSNOTIFY=stat celery worker -l info --autoreload @@ -766,14 +780,14 @@ Example Running the following command will result in the `foo` and `bar` modules being imported by the worker processes: -.. code-block:: python +.. code-block:: pycon >>> app.control.broadcast('pool_restart', ... arguments={'modules': ['foo', 'bar']}) Use the ``reload`` argument to reload modules it has already imported: -.. code-block:: python +.. code-block:: pycon >>> app.control.broadcast('pool_restart', ... arguments={'modules': ['foo'], @@ -782,7 +796,7 @@ Use the ``reload`` argument to reload modules it has already imported: If you don't specify any modules then all known tasks modules will be imported/reloaded: -.. code-block:: python +.. code-block:: pycon >>> app.control.broadcast('pool_restart', arguments={'reload': True}) @@ -816,16 +830,16 @@ uses remote control commands under the hood. You can also use the ``celery`` command to inspect workers, and it supports the same commands as the :class:`@control` interface. -.. code-block:: python +.. code-block:: pycon - # Inspect all nodes. + >>> # Inspect all nodes. >>> i = app.control.inspect() - # Specify multiple nodes to inspect. + >>> # Specify multiple nodes to inspect. >>> i = app.control.inspect(['worker1.example.com', 'worker2.example.com']) - # Specify a single node to inspect. + >>> # Specify a single node to inspect. >>> i = app.control.inspect('worker1.example.com') .. _worker-inspect-registered-tasks: @@ -834,7 +848,9 @@ Dump of registered tasks ------------------------ You can get a list of tasks registered in the worker using the -:meth:`~@control.inspect.registered`:: +:meth:`~@control.inspect.registered`: + +.. code-block:: pycon >>> i.registered() [{'worker1.example.com': ['tasks.add', @@ -846,7 +862,9 @@ Dump of currently executing tasks --------------------------------- You can get a list of active tasks using -:meth:`~@control.inspect.active`:: +:meth:`~@control.inspect.active`: + +.. code-block:: pycon >>> i.active() [{'worker1.example.com': @@ -861,7 +879,9 @@ Dump of scheduled (ETA) tasks ----------------------------- You can get a list of tasks waiting to be scheduled by using -:meth:`~@control.inspect.scheduled`:: +:meth:`~@control.inspect.scheduled`: + +.. code-block:: pycon >>> i.scheduled() [{'worker1.example.com': @@ -891,7 +911,9 @@ Reserved tasks are tasks that have been received, but are still waiting to be executed. You can get a list of these using -:meth:`~@control.inspect.reserved`:: +:meth:`~@control.inspect.reserved`: + +.. code-block:: pycon >>> i.reserved() [{'worker1.example.com': @@ -910,7 +932,7 @@ The remote control command ``inspect stats`` (or :meth:`~@control.inspect.stats`) will give you a long list of useful (or not so useful) statistics about the worker: -.. code-block:: bash +.. code-block:: console $ celery -A proj inspect stats @@ -1108,7 +1130,7 @@ Remote shutdown This command will gracefully shut down the worker remotely: -.. code-block:: python +.. code-block:: pycon >>> app.control.broadcast('shutdown') # shutdown all workers >>> app.control.broadcast('shutdown, destination="worker1@example.com") @@ -1123,7 +1145,7 @@ The workers reply with the string 'pong', and that's just about it. It will use the default one second timeout for replies unless you specify a custom timeout: -.. code-block:: python +.. code-block:: pycon >>> app.control.ping(timeout=0.5) [{'worker1.example.com': 'pong'}, @@ -1131,7 +1153,9 @@ a custom timeout: {'worker3.example.com': 'pong'}] :meth:`~@control.ping` also supports the `destination` argument, -so you can specify which workers to ping:: +so you can specify which workers to ping: + +.. code-block:: pycon >>> ping(['worker2.example.com', 'worker3.example.com']) [{'worker2.example.com': 'pong'}, @@ -1149,7 +1173,7 @@ You can enable/disable events by using the `enable_events`, `disable_events` commands. This is useful to temporarily monitor a worker using :program:`celery events`/:program:`celerymon`. -.. code-block:: python +.. code-block:: pycon >>> app.control.enable_events() >>> app.control.disable_events() diff --git a/docs/whatsnew-2.5.rst b/docs/whatsnew-2.5.rst index ec3d2e721..b57ac0d5c 100644 --- a/docs/whatsnew-2.5.rst +++ b/docs/whatsnew-2.5.rst @@ -64,7 +64,7 @@ race condition leading to an annoying warning. The :program:`camqadm` command can be used to delete the previous exchange: - .. code-block:: bash + .. code-block:: console $ camqadm exchange.delete celeryresults @@ -240,7 +240,7 @@ implementations: to install the :mod:`pyinotify` library you have to run the following command: - .. code-block:: bash + .. code-block:: console $ pip install pyinotify @@ -254,7 +254,7 @@ implementations: You can force an implementation by setting the :envvar:`CELERYD_FSNOTIFY` environment variable: -.. code-block:: bash +.. code-block:: console $ env CELERYD_FSNOTIFY=stat celeryd -l info --autoreload @@ -378,7 +378,7 @@ In Other News Additional configuration must be added at the end of the argument list followed by ``--``, for example: - .. code-block:: bash + .. code-block:: console $ celerybeat -l info -- celerybeat.max_loop_interval=10.0 @@ -428,7 +428,7 @@ In Other News **Examples**: - .. code-block:: bash + .. code-block:: console $ celeryctl migrate redis://localhost amqp://localhost $ celeryctl migrate amqp://localhost//v1 amqp://localhost//v2 diff --git a/docs/whatsnew-3.0.rst b/docs/whatsnew-3.0.rst index 24dd072f9..dc1320e27 100644 --- a/docs/whatsnew-3.0.rst +++ b/docs/whatsnew-3.0.rst @@ -96,7 +96,7 @@ has been removed, and that makes it incompatible with earlier versions. You can manually delete the old exchanges if you want, using the :program:`celery amqp` command (previously called ``camqadm``): -.. code-block:: bash +.. code-block:: console $ celery amqp exchange.delete celeryd.pidbox $ celery amqp exchange.delete reply.celeryd.pidbox @@ -128,7 +128,7 @@ All Celery's command-line programs are now available from a single You can see a list of subcommands and options by running: -.. code-block:: bash +.. code-block:: console $ celery help @@ -168,7 +168,7 @@ The setup.py install script will try to remove the old package, but if that doesn't work for some reason you have to remove it manually. This command helps: -.. code-block:: bash +.. code-block:: console $ rm -r $(dirname $(python -c ' import celery;print(celery.__file__)'))/app/task/ @@ -303,13 +303,13 @@ Tasks can now have callbacks and errbacks, and dependencies are recorded which can than be used to produce an image: - .. code-block:: bash + .. code-block:: console $ dot -Tpng graph.dot -o graph.png - A new special subtask called ``chain`` is also included: - .. code-block:: python + .. code-block:: pycon >>> from celery import chain @@ -351,7 +351,9 @@ The priority field is a number in the range of 0 - 9, where The priority range is collapsed into four steps by default, since it is unlikely that nine steps will yield more benefit than using four steps. The number of steps can be configured by setting the ``priority_steps`` -transport option, which must be a list of numbers in **sorted order**:: +transport option, which must be a list of numbers in **sorted order**: + +.. code-block:: pycon >>> BROKER_TRANSPORT_OPTIONS = { ... 'priority_steps': [0, 2, 4, 6, 8, 9], @@ -393,28 +395,34 @@ accidentally changed while switching to using blocking pop. - A new shortcut has been added to tasks: - :: + .. code-block:: pycon >>> task.s(arg1, arg2, kw=1) - as a shortcut to:: + as a shortcut to: + + .. code-block:: pycon >>> task.subtask((arg1, arg2), {'kw': 1}) -- Tasks can be chained by using the ``|`` operator:: +- Tasks can be chained by using the ``|`` operator: + + .. code-block:: pycon >>> (add.s(2, 2), pow.s(2)).apply_async() - Subtasks can be "evaluated" using the ``~`` operator: - :: + .. code-block:: pycon >>> ~add.s(2, 2) 4 >>> ~(add.s(2, 2) | pow.s(2)) - is the same as:: + is the same as: + + .. code-block:: pycon >>> chain(add.s(2, 2), pow.s(2)).apply_async().get() @@ -434,7 +442,9 @@ accidentally changed while switching to using blocking pop. It's now a pure dict subclass with properties for attribute access to the relevant keys. -- The repr's now outputs how the sequence would like imperatively:: +- The repr's now outputs how the sequence would like imperatively: + + .. code-block:: pycon >>> from celery import chord @@ -467,7 +477,7 @@ stable and is now documented as part of the offical API. These commands are available programmatically as :meth:`@control.add_consumer` / :meth:`@control.cancel_consumer`: - .. code-block:: python + .. code-block:: pycon >>> celery.control.add_consumer(queue_name, ... destination=['w1.example.com']) @@ -476,7 +486,7 @@ stable and is now documented as part of the offical API. or using the :program:`celery control` command: - .. code-block:: bash + .. code-block:: console $ celery control -d w1.example.com add_consumer queue $ celery control -d w1.example.com cancel_consumer queue @@ -493,14 +503,14 @@ stable and is now documented as part of the offical API. This command is available programmatically as :meth:`@control.autoscale`: - .. code-block:: python + .. code-block:: pycon >>> celery.control.autoscale(max=10, min=5, ... destination=['w1.example.com']) or using the :program:`celery control` command: - .. code-block:: bash + .. code-block:: console $ celery control -d w1.example.com autoscale 10 5 @@ -511,14 +521,14 @@ stable and is now documented as part of the offical API. These commands are available programmatically as :meth:`@control.pool_grow` / :meth:`@control.pool_shrink`: - .. code-block:: python + .. code-block:: pycon >>> celery.control.pool_grow(2, destination=['w1.example.com']) >>> celery.contorl.pool_shrink(2, destination=['w1.example.com']) or using the :program:`celery control` command: - .. code-block:: bash + .. code-block:: console $ celery control -d w1.example.com pool_grow 2 $ celery control -d w1.example.com pool_shrink 2 @@ -537,12 +547,16 @@ Immutable subtasks ------------------ ``subtask``'s can now be immutable, which means that the arguments -will not be modified when calling callbacks:: +will not be modified when calling callbacks: + +.. code-block:: pycon >>> chain(add.s(2, 2), clear_static_electricity.si()) means it will not receive the argument of the parent task, -and ``.si()`` is a shortcut to:: +and ``.si()`` is a shortcut to: + +.. code-block:: pycon >>> clear_static_electricity.subtask(immutable=True) @@ -602,7 +616,9 @@ Task registry no longer global Every Celery instance now has its own task registry. -You can make apps share registries by specifying it:: +You can make apps share registries by specifying it: + +.. code-block:: pycon >>> app1 = Celery() >>> app2 = Celery(tasks=app1.tasks) @@ -610,7 +626,9 @@ You can make apps share registries by specifying it:: Note that tasks are shared between registries by default, so that tasks will be added to every subsequently created task registry. As an alternative tasks can be private to specific task registries -by setting the ``shared`` argument to the ``@task`` decorator:: +by setting the ``shared`` argument to the ``@task`` decorator: + +.. code-block:: python @celery.task(shared=False) def add(x, y): @@ -625,7 +643,9 @@ by default, it will first be bound (and configured) when a concrete subclass is created. This means that you can safely import and make task base classes, -without also initializing the app environment:: +without also initializing the app environment: + +.. code-block:: python from celery.task import Task @@ -636,6 +656,8 @@ without also initializing the app environment:: print('CALLING %r' % (self,)) return self.run(*args, **kwargs) +.. code-block:: pycon + >>> DebugTask @@ -676,7 +698,7 @@ E.g. if you have a project named 'proj' where the celery app is located in 'from proj.celery import app', then the following will be equivalent: -.. code-block:: bash +.. code-block:: console $ celery worker --app=proj $ celery worker --app=proj.celery: @@ -697,7 +719,9 @@ In Other News descriptors that creates a new subclass on access. This means that e.g. ``app.Worker`` is an actual class - and will work as expected when:: + and will work as expected when: + + .. code-block:: python class Worker(app.Worker): ... @@ -715,7 +739,9 @@ In Other News - Result backends can now be set using an URL - Currently only supported by redis. Example use:: + Currently only supported by redis. Example use: + + .. code-block:: python CELERY_RESULT_BACKEND = 'redis://localhost/1' @@ -754,20 +780,22 @@ In Other News - Bugreport now available as a command and broadcast command - - Get it from a Python repl:: + - Get it from a Python repl: + + .. code-block:: pycon - >>> import celery - >>> print(celery.bugreport()) + >>> import celery + >>> print(celery.bugreport()) - Using the ``celery`` command line program: - .. code-block:: bash + .. code-block:: console $ celery report - Get it from remote workers: - .. code-block:: bash + .. code-block:: console $ celery inspect report @@ -788,7 +816,9 @@ In Other News Returns a list of the results applying the task function to every item in the sequence. - Example:: + Example: + + .. code-block:: pycon >>> from celery import xstarmap @@ -799,12 +829,16 @@ In Other News - ``group.skew(start=, stop=, step=)`` - Skew will skew the countdown for the individual tasks in a group, - e.g. with a group:: + Skew will skew the countdown for the individual tasks in a group, + e.g. with a group: + + .. code-block:: pycon >>> g = group(add.s(i, i) for i in xrange(10)) - Skewing the tasks from 0 seconds to 10 seconds:: + Skewing the tasks from 0 seconds to 10 seconds: + + .. code-block:: pycon >>> g.skew(stop=10) diff --git a/docs/whatsnew-3.1.rst b/docs/whatsnew-3.1.rst index 32bd47d39..da481f743 100644 --- a/docs/whatsnew-3.1.rst +++ b/docs/whatsnew-3.1.rst @@ -159,7 +159,7 @@ in init scripts. The rest will be removed in 3.2. If this is not a new installation then you may want to remove the old commands: -.. code-block:: bash +.. code-block:: console $ pip uninstall celery $ # repeat until it fails @@ -250,7 +250,7 @@ Caveats You can disable this prefetching behavior by enabling the :option:`-Ofair` worker option: - .. code-block:: bash + .. code-block:: console $ celery -A proj worker -l info -Ofair @@ -325,9 +325,9 @@ but if you would like to experiment with it you should know that: Instead you use the :program:`celery` command directly: - .. code-block:: bash + .. code-block:: console - celery -A proj worker -l info + $ celery -A proj worker -l info For this to work your app module must store the :envvar:`DJANGO_SETTINGS_MODULE` environment variable, see the example in the :ref:`Django @@ -410,14 +410,14 @@ If a custom name is not specified then the worker will use the name 'celery' by default, resulting in a fully qualified node name of 'celery@hostname': -.. code-block:: bash +.. code-block:: console $ celery worker -n example.com celery@example.com To also set the name you must include the @: -.. code-block:: bash +.. code-block:: console $ celery worker -n worker1@example.com worker1@example.com @@ -431,7 +431,7 @@ Remember that the ``-n`` argument also supports simple variable substitutions, so if the current hostname is *george.example.com* then the ``%h`` macro will expand into that: -.. code-block:: bash +.. code-block:: console $ celery worker -n worker1@%h worker1@george.example.com @@ -556,7 +556,7 @@ Time limits can now be set by the client Two new options have been added to the Calling API: ``time_limit`` and ``soft_time_limit``: -.. code-block:: python +.. code-block:: pycon >>> res = add.apply_async((2, 2), time_limit=10, soft_time_limit=8) @@ -605,7 +605,7 @@ setuptools extras. You install extras by specifying them inside brackets: -.. code-block:: bash +.. code-block:: console $ pip install celery[redis,mongodb] @@ -659,9 +659,9 @@ This means that: now does the same as calling the task directly: -.. code-block:: python +.. code-block:: pycon - add(2, 2) + >>> add(2, 2) In Other News ------------- @@ -685,7 +685,7 @@ In Other News Regular signature: - .. code-block:: python + .. code-block:: pycon >>> s = add.s(2, 2) >>> result = s.freeze() @@ -696,7 +696,7 @@ In Other News Group: - .. code-block:: python + .. code-block:: pycon >>> g = group(add.s(2, 2), add.s(4, 4)) >>> result = g.freeze() @@ -767,9 +767,9 @@ In Other News A dispatcher instantiated as follows: - .. code-block:: python + .. code-block:: pycon - app.events.Dispatcher(connection, groups=['worker']) + >>> app.events.Dispatcher(connection, groups=['worker']) will only send worker related events and silently drop any attempts to send events related to any other group. @@ -814,7 +814,7 @@ In Other News Example: - .. code-block:: bash + .. code-block:: console $ celery inspect conf @@ -923,7 +923,7 @@ In Other News You can create graphs from the currently installed bootsteps: - .. code-block:: bash + .. code-block:: console # Create graph of currently installed bootsteps in both the worker # and consumer namespaces. @@ -937,7 +937,7 @@ In Other News Or graphs of workers in a cluster: - .. code-block:: bash + .. code-block:: console # Create graph from the current cluster $ celery graph workers | dot -T png -o workers.png @@ -986,11 +986,11 @@ In Other News The :envvar:`C_IMPDEBUG` can be set to trace imports as they occur: - .. code-block:: bash + .. code-block:: console $ C_IMDEBUG=1 celery worker -l info - .. code-block:: bash + .. code-block:: console $ C_IMPDEBUG=1 celery shell @@ -1089,7 +1089,7 @@ In Other News The :option:`-X` argument is the inverse of the :option:`-Q` argument and accepts a list of queues to exclude (not consume from): - .. code-block:: bash + .. code-block:: console # Consume from all queues in CELERY_QUEUES, but not the 'foo' queue. $ celery worker -A proj -l info -X foo @@ -1098,13 +1098,13 @@ In Other News This means that you can now do: - .. code-block:: bash + .. code-block:: console $ C_FAKEFORK=1 celery multi start 10 or: - .. code-block:: bash + .. code-block:: console $ C_FAKEFORK=1 /etc/init.d/celeryd start From 71c8b41dd0e09d244ff628f251be7645dc071bf9 Mon Sep 17 00:00:00 2001 From: Seungha Kim Date: Wed, 30 Sep 2015 17:35:51 +0900 Subject: [PATCH 0667/1103] Update canvas.rst Fix order of partial arguments --- docs/userguide/canvas.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/canvas.rst b/docs/userguide/canvas.rst index 75f7581d8..b55fe5770 100644 --- a/docs/userguide/canvas.rst +++ b/docs/userguide/canvas.rst @@ -162,7 +162,7 @@ You can also clone signatures to create derivatives: proj.tasks.add(2) >>> s.clone(args=(4,), kwargs={'debug': True}) - proj.tasks.add(2, 4, debug=True) + proj.tasks.add(4, 2, debug=True) Immutability ------------ From 89f5f33e7ee90d346a98b2d8629b4f04edab1c5b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 2 Oct 2015 11:44:38 -0700 Subject: [PATCH 0668/1103] Adds abstract classes CallableTask and CallableSignature --- celery/app/task.py | 2 + celery/canvas.py | 2 + celery/utils/abstract.py | 128 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 celery/utils/abstract.py diff --git a/celery/app/task.py b/celery/app/task.py index c07ff2729..5e21e5b1e 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -19,6 +19,7 @@ from celery.exceptions import Ignore, MaxRetriesExceededError, Reject, Retry from celery.five import class_property, items from celery.result import EagerResult +from celery.utils import abstract from celery.utils import uuid, maybe_reraise from celery.utils.functional import mattrgetter, maybe_list from celery.utils.imports import instantiate @@ -923,4 +924,5 @@ def backend(self, value): # noqa @property def __name__(self): return self.__class__.__name__ +abstract.CallableTask.register(Task) BaseTask = Task # compat alias diff --git a/celery/canvas.py b/celery/canvas.py index a2edd3817..4dbb3563b 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -22,6 +22,7 @@ from celery._state import current_app, get_current_worker_task from celery.result import GroupResult +from celery.utils import abstract from celery.utils.functional import ( maybe_list, is_list, regen, chunks as _chunks, @@ -356,6 +357,7 @@ def _apply_async(self): subtask_type = _getitem_property('subtask_type') chord_size = _getitem_property('chord_size') immutable = _getitem_property('immutable') +abstract.CallableSignature.register(Signature) @Signature.register_type diff --git a/celery/utils/abstract.py b/celery/utils/abstract.py new file mode 100644 index 000000000..cf996fc71 --- /dev/null +++ b/celery/utils/abstract.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- +""" + celery.utils.abstract + ~~~~~~~~~~~~~~~~~~~~~ + + Abstract classes. + +""" +from __future__ import absolute_import + +__all__ = ['CallableTask', 'CallableSignature'] + +from abc import ABCMeta, abstractmethod, abstractproperty +from collections import Callable + +from celery.five import with_metaclass + + +def _hasattr(C, attr): + return any(attr in B.__dict__ for B in C.__mro__) + + +@with_metaclass(ABCMeta) +class _AbstractClass(object): + __required_attributes__ = frozenset() + + @classmethod + def __subclasshook__(cls, C): + return ( + cls is AsynCallable and + all(_hasattr(C, attr) for attr in cls.__required_attributes__) + ) or NotImplemented + + +class CallableTask(_AbstractClass, Callable): + __required_attributes__ = frozenset({ + 'delay', 'apply_async', 'apply', + }) + + @abstractmethod + def delay(self, *args, **kwargs): + pass + + @abstractmethod + def apply_async(self, *args, **kwargs): + pass + + @abstractmethod + def apply(self, *args, **kwargs): + pass + + +class CallableSignature(AsynCallable): + __required_attributes__ = frozenset({ + 'clone', 'freeze', 'set', 'link', 'link_error', '__or__', + }) + + @abstractproperty + def name(self): + pass + + @abstractproperty + def type(self): + pass + + @abstractproperty + def app(self): + pass + + @abstractproperty + def id(self): + pass + + @abstractproperty + def task(self): + pass + + @abstractproperty + def args(self): + pass + + @abstractproperty + def kwargs(self): + pass + + @abstractproperty + def options(self): + pass + + @abstractproperty + def subtask_type(self): + pass + + @abstractproperty + def chord_size(self): + pass + + @abstractproperty + def immutable(self): + pass + + @abstractmethod + def clone(self, args=None, kwargs=None): + pass + + @abstractmethod + def freeze(self, id=None, group_id=None, chord=None, root_id=None): + pass + + @abstractmethod + def set(self, immutable=None, **options): + pass + + @abstractmethod + def link(self, callback): + pass + + @abstractmethod + def link_error(self, errback): + pass + + @abstractmethod + def __or__(self, other): + pass + + @abstractmethod + def __invert__(self): + pass From 132d8d94d38f4050db876f56a841d5a5e487b25b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 2 Oct 2015 11:48:58 -0700 Subject: [PATCH 0669/1103] Use repr(signature) for periodic task when no name provided. Closes #2834 --- celery/app/base.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index ac845c565..ed0a97852 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -37,6 +37,7 @@ from celery.five import items, values from celery.loaders import get_loader_cls from celery.local import PromiseProxy, maybe_evaluate +from celery.utils import abstract from celery.utils import gen_task_name from celery.utils.dispatch import Signal from celery.utils.functional import first, maybe_list, head_from_fun @@ -537,8 +538,8 @@ def _load_config(self): # load lazy periodic tasks pending_beat = self._pending_periodic_tasks while pending_beat: - pargs, pkwargs = pending_beat.popleft() - self._add_periodic_task(*pargs, **pkwargs) + self._add_periodic_task(*pending_beat.popleft()) + # Settings.__setitem__ method, set Settings.change if self._preconf: for key, value in items(self._preconf): @@ -562,20 +563,22 @@ def signature(self, *args, **kwargs): kwargs['app'] = self return self.canvas.signature(*args, **kwargs) - def add_periodic_task(self, *args, **kwargs): - if not self.configured: - return self._pending_periodic_tasks.append((args, kwargs)) - return self._add_periodic_task(*args, **kwargs) - - def _add_periodic_task(self, schedule, sig, - args=(), kwargs={}, name=None, **opts): - from .task import Task - - sig = (self.signature(sig.name, args, kwargs) - if isinstance(sig, Task) else sig.clone(args, kwargs)) - - name = name or ':'.join([sig.name, ','.join(map(str, sig.args))]) - self._conf.CELERYBEAT_SCHEDULE[name] = { + def add_periodic_task(self, schedule, sig, + args=(), kwargs=(), name=None, **opts): + key, entry = self._sig_to_periodic_task_entry( + schedule, sig, args, kwargs, name, **opts) + if self.configured: + self._add_periodic_task(key, entry) + else: + self._pending_periodic_tasks.append((key, entry)) + return key + + def _sig_to_periodic_task_entry(self, schedule, sig, + args=(), kwargs={}, name=None, **opts): + sig = (sig.clone(args, kwargs) + if isinstance(sig, abstract.CallableSignature) + else self.signature(sig.name, args, kwargs) + return name or repr(sig), { 'schedule': schedule, 'task': sig.name, 'args': sig.args, @@ -583,6 +586,9 @@ def _add_periodic_task(self, schedule, sig, 'options': dict(sig.options, **opts), } + def _add_periodic_task(self, key, entry): + self._conf.CELERYBEAT_SCHEDULE[key] = entry + def create_task_cls(self): """Creates a base task class using default configuration taken from this app.""" From 964cbce6f82f9daefdc211f13702570b25e8fb66 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 2 Oct 2015 12:01:09 -0700 Subject: [PATCH 0670/1103] Fixes tests --- celery/app/base.py | 2 +- celery/canvas.py | 13 +++++++------ celery/utils/abstract.py | 14 +++++++++++--- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index ed0a97852..32f5ffcd0 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -577,7 +577,7 @@ def _sig_to_periodic_task_entry(self, schedule, sig, args=(), kwargs={}, name=None, **opts): sig = (sig.clone(args, kwargs) if isinstance(sig, abstract.CallableSignature) - else self.signature(sig.name, args, kwargs) + else self.signature(sig.name, args, kwargs)) return name or repr(sig), { 'schedule': schedule, 'task': sig.name, diff --git a/celery/canvas.py b/celery/canvas.py index 4dbb3563b..d012173dc 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -426,7 +426,7 @@ def prepare_steps(self, args, tasks, while steps: task = steps.popleft() - if not isinstance(task, Signature): + if not isinstance(task, abstract.CallableSignature): task = from_dict(task, app=app) if isinstance(task, group): task = maybe_unroll_group(task) @@ -606,7 +606,7 @@ def apply_chunks(cls, task, it, n, app=None): def _maybe_group(tasks): if isinstance(tasks, group): tasks = list(tasks.tasks) - elif isinstance(tasks, Signature): + elif isinstance(tasks, abstract.CallableSignature): tasks = [tasks] else: tasks = [signature(t) for t in regen(tasks)] @@ -632,10 +632,11 @@ def from_dict(self, d, app=None): ) def _prepared(self, tasks, partial_args, group_id, root_id, app, dict=dict, - Signature=Signature, from_dict=Signature.from_dict): + CallableSignature=abstract.CallableSignature, + from_dict=Signature.from_dict): for task in tasks: if isinstance(task, dict): - if isinstance(task, Signature): + if isinstance(task, CallableSignature): # local sigs are always of type Signature, and we # clone them to make sure we do not modify the originals. task = task.clone() @@ -918,7 +919,7 @@ def __repr__(self): def signature(varies, *args, **kwargs): if isinstance(varies, dict): - if isinstance(varies, Signature): + if isinstance(varies, abstract.CallableSignature): return varies.clone() return Signature.from_dict(varies) return Signature(varies, *args, **kwargs) @@ -928,7 +929,7 @@ def signature(varies, *args, **kwargs): def maybe_signature(d, app=None): if d is not None: if isinstance(d, dict): - if not isinstance(d, Signature): + if not isinstance(d, abstract.CallableSignature): d = signature(d) elif isinstance(d, list): return [maybe_signature(s, app=app) for s in d] diff --git a/celery/utils/abstract.py b/celery/utils/abstract.py index cf996fc71..601468942 100644 --- a/celery/utils/abstract.py +++ b/celery/utils/abstract.py @@ -25,9 +25,9 @@ class _AbstractClass(object): __required_attributes__ = frozenset() @classmethod - def __subclasshook__(cls, C): + def _subclasshook_using(cls, parent, C): return ( - cls is AsynCallable and + cls is parent and all(_hasattr(C, attr) for attr in cls.__required_attributes__) ) or NotImplemented @@ -49,8 +49,12 @@ def apply_async(self, *args, **kwargs): def apply(self, *args, **kwargs): pass + @classmethod + def __subclasshook__(cls, C): + return cls._subclasshook_using(CallableTask, C) + -class CallableSignature(AsynCallable): +class CallableSignature(CallableTask): __required_attributes__ = frozenset({ 'clone', 'freeze', 'set', 'link', 'link_error', '__or__', }) @@ -126,3 +130,7 @@ def __or__(self, other): @abstractmethod def __invert__(self): pass + + @classmethod + def __subclasshook__(cls, C): + return cls._subclasshook_using(CallableSignature, C) From f1930712e0148c4b8ccc024b59183bcc407560a4 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 2 Oct 2015 12:01:42 -0700 Subject: [PATCH 0671/1103] Stresstest default heartbeat now 30s --- funtests/stress/stress/templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index 96dd2aa90..14c87f123 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -50,7 +50,7 @@ def template_names(): @template() class default(object): - BROKER_HEARTBEAT=2 + BROKER_HEARTBEAT=30 CELERY_ACCEPT_CONTENT = ['json'] CELERY_DEFAULT_QUEUE = CSTRESS_QUEUE CELERY_TASK_SERIALIZER = 'json' From fc38ae3b2e95578f57602e2d40b2c52637c207c1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 2 Oct 2015 12:03:06 -0700 Subject: [PATCH 0672/1103] Removes worker Queues bootstep --- celery/tests/worker/test_components.py | 26 ++++++++++++-------------- celery/tests/worker/test_worker.py | 12 ++++++------ celery/worker/__init__.py | 1 - celery/worker/components.py | 24 ++++++++---------------- 4 files changed, 26 insertions(+), 37 deletions(-) diff --git a/celery/tests/worker/test_components.py b/celery/tests/worker/test_components.py index b39865db4..752a6d073 100644 --- a/celery/tests/worker/test_components.py +++ b/celery/tests/worker/test_components.py @@ -4,24 +4,12 @@ # here to complete coverage. Should move everyting to this module at some # point [-ask] -from celery.worker.components import ( - Queues, - Pool, -) +from celery.platforms import IS_WINDOWS +from celery.worker.components import Pool from celery.tests.case import AppCase, Mock -class test_Queues(AppCase): - - def test_create_when_eventloop(self): - w = Mock() - w.use_eventloop = w.pool_putlocks = w.pool_cls.uses_semaphore = True - q = Queues(w) - q.create(w) - self.assertIs(w.process_task, w._process_task_sem) - - class test_Pool(AppCase): def test_close_terminate(self): @@ -36,3 +24,13 @@ def test_close_terminate(self): w.pool = None comp.close(w) comp.terminate(w) + + def test_create_when_eventloop(self): + if IS_WINDOWS: + raise SkipTest('Win32') + w = Mock() + w.use_eventloop = w.pool_putlocks = w.pool_cls.uses_semaphore = True + comp = Pool(w) + pool = w.pool = Mock() + comp.create(w) + self.assertIs(w.process_task, w._process_task_sem) diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index f42f2b1b1..794d10791 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -1133,12 +1133,6 @@ def test_start__terminate(self): for step in worker.steps: self.assertTrue(step.terminate.call_count) - def test_Queues_pool_no_sem(self): - w = Mock() - w.pool_cls.uses_semaphore = False - components.Queues(w).create(w) - self.assertIs(w.process_task, w._process_task) - def test_Hub_crate(self): w = Mock() x = components.Hub(w) @@ -1153,6 +1147,12 @@ def test_Pool_crate_threaded(self): pool = components.Pool(w) pool.create(w) + def test_Pool_pool_no_sem(self): + w = Mock() + w.pool_cls.uses_semaphore = False + components.Pool(w).create(w) + self.assertIs(w.process_task, w._process_task) + def test_Pool_create(self): from kombu.async.semaphore import LaxBoundedSemaphore w = Mock() diff --git a/celery/worker/__init__.py b/celery/worker/__init__.py index 24dc777fe..416262cf1 100644 --- a/celery/worker/__init__.py +++ b/celery/worker/__init__.py @@ -81,7 +81,6 @@ class Blueprint(bootsteps.Blueprint): name = 'Worker' default_steps = { 'celery.worker.components:Hub', - 'celery.worker.components:Queues', 'celery.worker.components:Pool', 'celery.worker.components:Beat', 'celery.worker.components:Timer', diff --git a/celery/worker/components.py b/celery/worker/components.py index 4b5ae0371..d3f219da1 100644 --- a/celery/worker/components.py +++ b/celery/worker/components.py @@ -19,9 +19,11 @@ from celery._state import _set_task_join_will_block from celery.exceptions import ImproperlyConfigured from celery.five import string_t +from celery.platforms import IS_WINDOWS from celery.utils.log import worker_logger as logger -__all__ = ['Timer', 'Hub', 'Queues', 'Pool', 'Beat', 'StateDB', 'Consumer'] + +__all__ = ['Timer', 'Hub', 'Pool', 'Beat', 'StateDB', 'Consumer'] ERR_B_GREEN = """\ -B option doesn't work with eventlet/gevent pools: \ @@ -96,19 +98,6 @@ def _patch_thread_primitives(self, w): pool.Lock = DummyLock -class Queues(bootsteps.Step): - """This bootstep initializes the internal queues - used by the worker.""" - label = 'Queues (intra)' - requires = (Hub,) - - def create(self, w): - w.process_task = w._process_task - if w.use_eventloop: - if w.pool_putlocks and w.pool_cls.uses_semaphore: - w.process_task = w._process_task_sem - - class Pool(bootsteps.StartStopStep): """Bootstep managing the worker pool. @@ -123,7 +112,7 @@ class Pool(bootsteps.StartStopStep): * min_concurrency """ - requires = (Queues,) + requires = (Hub,) def __init__(self, w, autoscale=None, autoreload=None, no_execv=False, optimization=None, **kwargs): @@ -151,14 +140,17 @@ def terminate(self, w): def create(self, w, semaphore=None, max_restarts=None): if w.app.conf.CELERYD_POOL in ('eventlet', 'gevent'): warnings.warn(UserWarning(W_POOL_SETTING)) - threaded = not w.use_eventloop + threaded = not w.use_eventloop or IS_WINDOWS procs = w.min_concurrency forking_enable = w.no_execv if w.force_execv else True + w.process_task = w._process_task if not threaded: semaphore = w.semaphore = LaxBoundedSemaphore(procs) w._quick_acquire = w.semaphore.acquire w._quick_release = w.semaphore.release max_restarts = 100 + if w.pool_putlocks and w.pool_cls.uses_semaphore: + w.process_task = w._process_task_sem allow_restart = self.autoreload_enabled or w.pool_restarts pool = w.pool = self.instantiate( w.pool_cls, w.min_concurrency, From d342f220b7e5a85b2fd1c308dc80570bd4c999c6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 2 Oct 2015 12:03:42 -0700 Subject: [PATCH 0673/1103] Disabled backend now overrides get_many --- celery/backends/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/backends/base.py b/celery/backends/base.py index c4dffaaa6..a8975be25 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -619,3 +619,4 @@ def _is_disabled(self, *args, **kwargs): 'No result backend configured. ' 'Please see the documentation for more information.') wait_for = get_status = get_result = get_traceback = _is_disabled + get_many = _is_disabled From 151442a162be5f78cbdabadbc463cc9e9d9520f0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 2 Oct 2015 18:01:06 -0700 Subject: [PATCH 0674/1103] proc.dead does not exist for some reason --- celery/concurrency/asynpool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index c4829c950..76a5c8da4 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -589,7 +589,7 @@ def _remove_from_index(obj, proc, index, remove_fun, callback=None): def on_process_down(proc): """Called when a worker process exits.""" - if proc.dead: + if getattr(proc, 'dead', None): return process_flush_queues(proc) _remove_from_index( From 045b52f1450d6d5cc500e0057a4b498250dc5692 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 2 Oct 2015 18:07:47 -0700 Subject: [PATCH 0675/1103] flakes --- celery/tests/worker/test_components.py | 4 ++-- celery/utils/abstract.py | 4 ++-- docs/conf.py | 2 +- examples/django/proj/celery.py | 2 -- examples/django/proj/wsgi.py | 2 +- extra/release/bump_version.py | 6 ++++-- funtests/benchmarks/bench_worker.py | 6 +++--- funtests/stress/stress/app.py | 1 + funtests/stress/stress/suite.py | 3 +-- funtests/stress/stress/templates.py | 2 +- funtests/suite/__init__.py | 1 + funtests/suite/test_basic.py | 6 ++---- funtests/suite/test_leak.py | 3 --- setup.py | 12 +++++++++--- 14 files changed, 28 insertions(+), 26 deletions(-) diff --git a/celery/tests/worker/test_components.py b/celery/tests/worker/test_components.py index 752a6d073..c11d48d8e 100644 --- a/celery/tests/worker/test_components.py +++ b/celery/tests/worker/test_components.py @@ -7,7 +7,7 @@ from celery.platforms import IS_WINDOWS from celery.worker.components import Pool -from celery.tests.case import AppCase, Mock +from celery.tests.case import AppCase, Mock, SkipTest class test_Pool(AppCase): @@ -31,6 +31,6 @@ def test_create_when_eventloop(self): w = Mock() w.use_eventloop = w.pool_putlocks = w.pool_cls.uses_semaphore = True comp = Pool(w) - pool = w.pool = Mock() + w.pool = Mock() comp.create(w) self.assertIs(w.process_task, w._process_task_sem) diff --git a/celery/utils/abstract.py b/celery/utils/abstract.py index 601468942..669f347fb 100644 --- a/celery/utils/abstract.py +++ b/celery/utils/abstract.py @@ -8,13 +8,13 @@ """ from __future__ import absolute_import -__all__ = ['CallableTask', 'CallableSignature'] - from abc import ABCMeta, abstractmethod, abstractproperty from collections import Callable from celery.five import with_metaclass +__all__ = ['CallableTask', 'CallableSignature'] + def _hasattr(C, attr): return any(attr in B.__dict__ for B in C.__mro__) diff --git a/docs/conf.py b/docs/conf.py index c23728e83..694af4ee6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,7 +10,7 @@ # absolute, like shown here. sys.path.insert(0, os.path.join(this, os.pardir)) sys.path.append(os.path.join(this, '_ext')) -import celery +import celery # noqa # General configuration # --------------------- diff --git a/examples/django/proj/celery.py b/examples/django/proj/celery.py index a2eeb7444..dc3ad1415 100644 --- a/examples/django/proj/celery.py +++ b/examples/django/proj/celery.py @@ -4,8 +4,6 @@ from celery import Celery -from django.apps import apps as django_apps - # set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings') diff --git a/examples/django/proj/wsgi.py b/examples/django/proj/wsgi.py index 446fcc9d9..6a65b3ff8 100644 --- a/examples/django/proj/wsgi.py +++ b/examples/django/proj/wsgi.py @@ -20,7 +20,7 @@ # This application object is used by any WSGI server configured to use this # file. This includes Django's development server, if the WSGI_APPLICATION # setting points here. -from django.core.wsgi import get_wsgi_application +from django.core.wsgi import get_wsgi_application # noqa application = get_wsgi_application() # Apply WSGI middleware here. diff --git a/extra/release/bump_version.py b/extra/release/bump_version.py index 8e507255a..9415b7046 100755 --- a/extra/release/bump_version.py +++ b/extra/release/bump_version.py @@ -12,11 +12,13 @@ from contextlib import contextmanager from tempfile import NamedTemporaryFile -rq = lambda s: s.strip("\"'") - str_t = str if sys.version_info[0] >= 3 else basestring +def rq(s): + return s.strip("\"'") + + def cmd(*args): return subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0] diff --git a/funtests/benchmarks/bench_worker.py b/funtests/benchmarks/bench_worker.py index 77e743408..07e6e256b 100644 --- a/funtests/benchmarks/bench_worker.py +++ b/funtests/benchmarks/bench_worker.py @@ -8,9 +8,9 @@ USE_FAST_LOCALS='yes', ) -from celery import Celery -from celery.five import range -from kombu.five import monotonic +from celery import Celery # noqa +from celery.five import range # noqa +from kombu.five import monotonic # noqa DEFAULT_ITS = 40000 diff --git a/funtests/stress/stress/app.py b/funtests/stress/stress/app.py index c26481f65..df028d39d 100644 --- a/funtests/stress/stress/app.py +++ b/funtests/stress/stress/app.py @@ -150,6 +150,7 @@ def marker(s, sep='-'): except Exception as exc: print("Retrying marker.delay(). It failed to start: %s" % exc) + @app.on_after_configure.connect def setup_periodic_tasks(sender, **kwargs): sender.add_periodic_task(10, add.s(2, 2), expires=10) diff --git a/funtests/stress/stress/suite.py b/funtests/stress/stress/suite.py index 6e5e6a64a..2556ff16d 100755 --- a/funtests/stress/stress/suite.py +++ b/funtests/stress/stress/suite.py @@ -271,8 +271,7 @@ def manyshort(self): def always_timeout(self): self.join( group(sleeping.s(1).set(time_limit=0.1) - for _ in range(100) - )(), + for _ in range(100))(), timeout=10, propagate=True, ) diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index 14c87f123..f46b12de5 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -50,7 +50,7 @@ def template_names(): @template() class default(object): - BROKER_HEARTBEAT=30 + BROKER_HEARTBEAT = 30 CELERY_ACCEPT_CONTENT = ['json'] CELERY_DEFAULT_QUEUE = CSTRESS_QUEUE CELERY_TASK_SERIALIZER = 'json' diff --git a/funtests/suite/__init__.py b/funtests/suite/__init__.py index aed92042d..847100058 100644 --- a/funtests/suite/__init__.py +++ b/funtests/suite/__init__.py @@ -1,6 +1,7 @@ import os import sys +sys.path.insert(0, os.getcwd()) sys.path.insert(0, os.path.join(os.getcwd(), os.pardir)) config = os.environ.setdefault('CELERY_FUNTEST_CONFIG_MODULE', diff --git a/funtests/suite/test_basic.py b/funtests/suite/test_basic.py index cb0471381..5213baf74 100644 --- a/funtests/suite/test_basic.py +++ b/funtests/suite/test_basic.py @@ -1,10 +1,8 @@ +from __future__ import absolute_import + import operator -import os -import sys # funtest config -sys.path.insert(0, os.getcwd()) -sys.path.insert(0, os.path.join(os.getcwd(), os.pardir)) import suite # noqa from celery.five import range diff --git a/funtests/suite/test_leak.py b/funtests/suite/test_leak.py index 98ea07a54..7a3dcc067 100644 --- a/funtests/suite/test_leak.py +++ b/funtests/suite/test_leak.py @@ -6,9 +6,6 @@ import shlex import subprocess -sys.path.insert(0, os.getcwd()) -sys.path.insert(0, os.path.join(os.getcwd(), os.pardir)) - from celery import current_app from celery.five import range from celery.tests.case import SkipTest, unittest diff --git a/setup.py b/setup.py index 9a86098ca..5ca35eb16 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ from setuptools import setup, find_packages import os +import re import sys import codecs @@ -75,11 +76,13 @@ # -*- Distribution Meta -*- -import re re_meta = re.compile(r'__(\w+?)__\s*=\s*(.*)') re_vers = re.compile(r'VERSION\s*=.*?\((.*?)\)') re_doc = re.compile(r'^"""(.+?)"""') -rq = lambda s: s.strip("\"'") + + +def rq(s): + return s.strip("\"'") def add_default(m): @@ -164,7 +167,10 @@ def reqs(*f): # -*- Extras -*- -extras = lambda *p: reqs('extras', *p) + +def extras(*p): + return reqs('extras', *p) + # Celery specific features = { 'auth', 'cassandra', 'memcache', 'couchbase', 'threads', From 580f06be22a5f311e8f550b46619c385a5b0213c Mon Sep 17 00:00:00 2001 From: Michael Permana Date: Tue, 6 Oct 2015 01:28:49 -0400 Subject: [PATCH 0676/1103] Fix issue #1628 --- celery/app/defaults.py | 1 + celery/app/task.py | 7 +++++++ celery/worker/request.py | 5 ++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index d217032b6..f5edd7fc0 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -132,6 +132,7 @@ def __repr__(self): 'REDIS_DB': Option(type='int', **_REDIS_OLD), 'REDIS_PASSWORD': Option(type='string', **_REDIS_OLD), 'REDIS_MAX_CONNECTIONS': Option(type='int'), + 'REJECT_ON_WORKER_LOST': Option(type='bool'), 'RESULT_BACKEND': Option(type='string'), 'RESULT_DB_SHORT_LIVED_SESSIONS': Option(False, type='bool'), 'RESULT_DB_TABLENAMES': Option(type='dict'), diff --git a/celery/app/task.py b/celery/app/task.py index 5e21e5b1e..707366c72 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -220,6 +220,12 @@ class Task(object): #: :setting:`CELERY_ACKS_LATE` setting. acks_late = None + #: When CELERY_ACKS_LATE is set to True, the default behavior to + #: handle worker crash is to acknowledge the message. Setting + #: this to true allows the message to be rejected and requeued so + #: it will be executed again by another worker. + reject_on_worker_lost = None + #: Tuple of expected exceptions. #: #: These are errors that are expected in normal operation @@ -248,6 +254,7 @@ class Task(object): ('rate_limit', 'CELERY_DEFAULT_RATE_LIMIT'), ('track_started', 'CELERY_TRACK_STARTED'), ('acks_late', 'CELERY_ACKS_LATE'), + ('reject_on_worker_lost', 'CELERY_REJECT_ON_WORKER_LOST'), ('ignore_result', 'CELERY_IGNORE_RESULT'), ('store_errors_even_if_ignored', 'CELERY_STORE_ERRORS_EVEN_IF_IGNORED'), diff --git a/celery/worker/request.py b/celery/worker/request.py index fded7597c..8bf3ffd4f 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -352,7 +352,10 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): ) # (acks_late) acknowledge after result stored. if self.task.acks_late: - self.acknowledge() + if self.task.reject_on_worker_lost and isinstance(exc, WorkerLostError): + self.reject(True) + else: + self.acknowledge() if send_failed_event: self.send_event( From 577e3da6fd6c083f0f4f8f15bbe2b24c36287905 Mon Sep 17 00:00:00 2001 From: Michael Permana Date: Thu, 8 Oct 2015 13:29:52 -0400 Subject: [PATCH 0677/1103] Always reject if acks_late and worker lost --- celery/app/defaults.py | 1 - celery/app/task.py | 7 ------- celery/worker/request.py | 2 +- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index f5edd7fc0..d217032b6 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -132,7 +132,6 @@ def __repr__(self): 'REDIS_DB': Option(type='int', **_REDIS_OLD), 'REDIS_PASSWORD': Option(type='string', **_REDIS_OLD), 'REDIS_MAX_CONNECTIONS': Option(type='int'), - 'REJECT_ON_WORKER_LOST': Option(type='bool'), 'RESULT_BACKEND': Option(type='string'), 'RESULT_DB_SHORT_LIVED_SESSIONS': Option(False, type='bool'), 'RESULT_DB_TABLENAMES': Option(type='dict'), diff --git a/celery/app/task.py b/celery/app/task.py index 707366c72..5e21e5b1e 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -220,12 +220,6 @@ class Task(object): #: :setting:`CELERY_ACKS_LATE` setting. acks_late = None - #: When CELERY_ACKS_LATE is set to True, the default behavior to - #: handle worker crash is to acknowledge the message. Setting - #: this to true allows the message to be rejected and requeued so - #: it will be executed again by another worker. - reject_on_worker_lost = None - #: Tuple of expected exceptions. #: #: These are errors that are expected in normal operation @@ -254,7 +248,6 @@ class Task(object): ('rate_limit', 'CELERY_DEFAULT_RATE_LIMIT'), ('track_started', 'CELERY_TRACK_STARTED'), ('acks_late', 'CELERY_ACKS_LATE'), - ('reject_on_worker_lost', 'CELERY_REJECT_ON_WORKER_LOST'), ('ignore_result', 'CELERY_IGNORE_RESULT'), ('store_errors_even_if_ignored', 'CELERY_STORE_ERRORS_EVEN_IF_IGNORED'), diff --git a/celery/worker/request.py b/celery/worker/request.py index 8bf3ffd4f..209580c76 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -352,7 +352,7 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): ) # (acks_late) acknowledge after result stored. if self.task.acks_late: - if self.task.reject_on_worker_lost and isinstance(exc, WorkerLostError): + if isinstance(exc, WorkerLostError): self.reject(True) else: self.acknowledge() From 01f921adee7a2f95291fd2238bb653c49a42bb40 Mon Sep 17 00:00:00 2001 From: Michael Permana Date: Thu, 8 Oct 2015 03:23:38 -0700 Subject: [PATCH 0678/1103] Only reject and requeue on non redelivered message to avoid infinite crash Conflicts: celery/worker/request.py --- celery/app/defaults.py | 1 + celery/app/task.py | 7 +++++++ celery/worker/request.py | 8 +++++--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index d217032b6..f5edd7fc0 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -132,6 +132,7 @@ def __repr__(self): 'REDIS_DB': Option(type='int', **_REDIS_OLD), 'REDIS_PASSWORD': Option(type='string', **_REDIS_OLD), 'REDIS_MAX_CONNECTIONS': Option(type='int'), + 'REJECT_ON_WORKER_LOST': Option(type='bool'), 'RESULT_BACKEND': Option(type='string'), 'RESULT_DB_SHORT_LIVED_SESSIONS': Option(False, type='bool'), 'RESULT_DB_TABLENAMES': Option(type='dict'), diff --git a/celery/app/task.py b/celery/app/task.py index 5e21e5b1e..707366c72 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -220,6 +220,12 @@ class Task(object): #: :setting:`CELERY_ACKS_LATE` setting. acks_late = None + #: When CELERY_ACKS_LATE is set to True, the default behavior to + #: handle worker crash is to acknowledge the message. Setting + #: this to true allows the message to be rejected and requeued so + #: it will be executed again by another worker. + reject_on_worker_lost = None + #: Tuple of expected exceptions. #: #: These are errors that are expected in normal operation @@ -248,6 +254,7 @@ class Task(object): ('rate_limit', 'CELERY_DEFAULT_RATE_LIMIT'), ('track_started', 'CELERY_TRACK_STARTED'), ('acks_late', 'CELERY_ACKS_LATE'), + ('reject_on_worker_lost', 'CELERY_REJECT_ON_WORKER_LOST'), ('ignore_result', 'CELERY_IGNORE_RESULT'), ('store_errors_even_if_ignored', 'CELERY_STORE_ERRORS_EVEN_IF_IGNORED'), diff --git a/celery/worker/request.py b/celery/worker/request.py index 209580c76..153866ed0 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -326,7 +326,6 @@ def on_retry(self, exc_info): def on_failure(self, exc_info, send_failed_event=True, return_ok=False): """Handler called if the task raised an exception.""" task_ready(self) - if isinstance(exc_info.exception, MemoryError): raise MemoryError('Process got: %s' % (exc_info.exception,)) elif isinstance(exc_info.exception, Reject): @@ -352,8 +351,11 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): ) # (acks_late) acknowledge after result stored. if self.task.acks_late: - if isinstance(exc, WorkerLostError): - self.reject(True) + reject_and_requeue = (self.task.reject_on_worker_lost and + isinstance(exc, WorkerLostError) and + not self.delivery_info.get('redelivered', False)) + if reject_and_requeue: + self.reject(requeue=True) else: self.acknowledge() From c7bf57098a725e3bfda1c20d92a967d115f380e6 Mon Sep 17 00:00:00 2001 From: Justin Patrin Date: Thu, 6 Aug 2015 12:01:15 -0700 Subject: [PATCH 0679/1103] Use all redis pipelines as context managers to ensure that they are always cleaned up properly, especially in the case of exceptions --- celery/backends/redis.py | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index fb1eaba6d..3fc1cfab6 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -160,13 +160,13 @@ def set(self, key, value, **retry_policy): return self.ensure(self._set, (key, value), **retry_policy) def _set(self, key, value): - pipe = self.client.pipeline() - if self.expires: - pipe.setex(key, value, self.expires) - else: - pipe.set(key, value) - pipe.publish(key, value) - pipe.execute() + with self.client.pipeline() as pipe: + if self.expires: + pipe.setex(key, value, self.expires) + else: + pipe.set(key, value) + pipe.publish(key, value) + pipe.execute() def delete(self, key): self.client.delete(key) @@ -207,13 +207,14 @@ def _new_chord_return(self, task, state, result, propagate=None): jkey = self.get_key_for_group(gid, '.j') tkey = self.get_key_for_group(gid, '.t') result = self.encode_result(result, state) - _, readycount, totaldiff, _, _ = client.pipeline() \ - .rpush(jkey, self.encode([1, tid, state, result])) \ - .llen(jkey) \ - .get(tkey) \ - .expire(jkey, 86400) \ - .expire(tkey, 86400) \ - .execute() + with client.pipeline() as pipe: + _, readycount, totaldiff, _, _ = pipe \ + .rpush(jkey, self.encode([1, tid, state, result])) \ + .llen(jkey) \ + .get(tkey) \ + .expire(jkey, 86400) \ + .expire(tkey, 86400) \ + .execute() totaldiff = int(totaldiff or 0) @@ -222,11 +223,12 @@ def _new_chord_return(self, task, state, result, propagate=None): total = callback['chord_size'] + totaldiff if readycount == total: decode, unpack = self.decode, self._unpack_chord_result - resl, _, _ = client.pipeline() \ - .lrange(jkey, 0, total) \ - .delete(jkey) \ - .delete(tkey) \ - .execute() + with client.pipeline() as pipe: + resl, _, _ = pipe \ + .lrange(jkey, 0, total) \ + .delete(jkey) \ + .delete(tkey) \ + .execute() try: callback.delay([unpack(tup, decode) for tup in resl]) except Exception as exc: From 899769d401e3c55a8491c8b408ca51c48e0c6f1e Mon Sep 17 00:00:00 2001 From: Justin Patrin Date: Thu, 6 Aug 2015 12:08:33 -0700 Subject: [PATCH 0680/1103] Add myself to CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 3c15e7246..0bc480f30 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -190,4 +190,5 @@ Alexander Lebedev, 2015/04/25 Frantisek Holop, 2015/05/21 Feanil Patel, 2015/05/21 Jocelyn Delalande, 2015/06/03 +Justin Patrin, 2015/08/06 Juan Rossi, 2015/08/10 From 043f7935654798e949897c6bd0c70ebd5a6dfe90 Mon Sep 17 00:00:00 2001 From: Justin Patrin Date: Thu, 6 Aug 2015 12:12:54 -0700 Subject: [PATCH 0681/1103] Add contextmanager methods to testing Pipeline --- celery/tests/backends/test_redis.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/celery/tests/backends/test_redis.py b/celery/tests/backends/test_redis.py index b2ebcd2a3..fd30a4727 100644 --- a/celery/tests/backends/test_redis.py +++ b/celery/tests/backends/test_redis.py @@ -37,6 +37,12 @@ def add_step(*args, **kwargs): return self return add_step + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + pass + def execute(self): return [step(*a, **kw) for step, a, kw in self.steps] From 1974371d0ad6f8fde062b8468a2fee59e6b71d49 Mon Sep 17 00:00:00 2001 From: Justin Patrin Date: Thu, 6 Aug 2015 14:49:40 -0700 Subject: [PATCH 0682/1103] Support redis timeout paramaters in the URL. They need to be float to work. --- celery/backends/redis.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 3fc1cfab6..3b74bf563 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -130,6 +130,10 @@ def _params_from_url(self, url, defaults): db = db.strip('/') if isinstance(db, string_t) else db connparams['db'] = int(db) + for key in ['socket_timeout', 'socket_connect_timeout']: + if key in query: + query[key] = float(query[key]) + # Query parameters override other parameters connparams.update(query) return connparams From 7f30d902e321c4a12ea5185fd83b2e22853474f0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 12 Oct 2015 12:49:44 -0700 Subject: [PATCH 0683/1103] Try new Travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index f9cb0a0e4..fffe22e6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,3 +23,4 @@ notifications: - "chat.freenode.net#celery" on_success: change on_failure: change +sudo: false From 1b67e644e8be343cae7b131f141f3fa13b295a4c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 12 Oct 2015 13:39:15 -0700 Subject: [PATCH 0684/1103] Fixed outdated django example --- docs/whatsnew-3.1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/whatsnew-3.1.rst b/docs/whatsnew-3.1.rst index da481f743..3dc416017 100644 --- a/docs/whatsnew-3.1.rst +++ b/docs/whatsnew-3.1.rst @@ -319,7 +319,7 @@ but if you would like to experiment with it you should know that: .. code-block:: python from django.conf import settings - app.autodiscover_tasks(settings.INSTALLED_APPS) + app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) - You no longer use ``manage.py`` From 44355dc07794c583aa0524becb58c084e27867e2 Mon Sep 17 00:00:00 2001 From: Michael Permana Date: Tue, 13 Oct 2015 15:30:43 -0400 Subject: [PATCH 0685/1103] Only reject and retry when we really know the redelivered is False --- celery/worker/request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/worker/request.py b/celery/worker/request.py index 153866ed0..f809c10b5 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -353,7 +353,7 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): if self.task.acks_late: reject_and_requeue = (self.task.reject_on_worker_lost and isinstance(exc, WorkerLostError) and - not self.delivery_info.get('redelivered', False)) + self.delivery_info.get('redelivered', False) is False) if reject_and_requeue: self.reject(requeue=True) else: From 22bde6c739967db1cf570d5877860b1d3ce46716 Mon Sep 17 00:00:00 2001 From: Michael Permana Date: Tue, 13 Oct 2015 16:48:46 -0400 Subject: [PATCH 0686/1103] Add test case for on_failure with WorkerLostError, acks_late and reject_on_worker_lost --- celery/tests/worker/test_request.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index b642199ce..f52856254 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -325,6 +325,20 @@ def test_on_failure_Reject_rejects_with_requeue(self): req_logger, req.connection_errors, True, ) + def test_on_failure_WrokerLostError_rejects_with_requeue(self): + einfo = None + try: + raise WorkerLostError() + except: + einfo = ExceptionInfo(internal=True) + req = self.get_request(self.add.s(2, 2)) + req.task.acks_late = True + req.task.reject_on_worker_lost = True + req.delivery_info['redelivered'] = False + req.on_failure(einfo) + req.on_reject.assert_called_with(req_logger, + req.connection_errors, True) + def test_tzlocal_is_cached(self): req = self.get_request(self.add.s(2, 2)) req._tzlocal = 'foo' From 8c8ee7c317021a76e9ee7a6e6aaaec8a3581c91e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 13 Oct 2015 15:05:16 -0700 Subject: [PATCH 0687/1103] New cassandra backend, small bugs, cosmetics, flakes (Issue #2782) --- celery/backends/__init__.py | 2 +- celery/backends/cassandra.py | 9 +- celery/backends/new_cassandra.py | 118 ++++++++++++-------- celery/tests/backends/test_new_cassandra.py | 28 ++--- 4 files changed, 93 insertions(+), 64 deletions(-) diff --git a/celery/backends/__init__.py b/celery/backends/__init__.py index afff815c2..e214a9129 100644 --- a/celery/backends/__init__.py +++ b/celery/backends/__init__.py @@ -30,7 +30,7 @@ 'db': 'celery.backends.database:DatabaseBackend', 'database': 'celery.backends.database:DatabaseBackend', 'cassandra': 'celery.backends.cassandra:CassandraBackend', - 'new_cassandra': 'celery.backends.new_cassandra:NewCassandraBackend', + 'new_cassandra': 'celery.backends.new_cassandra:CassandraBackend', 'couchbase': 'celery.backends.couchbase:CouchBaseBackend', 'couchdb': 'celery.backends.couchdb:CouchDBBackend', 'riak': 'celery.backends.riak:RiakBackend', diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index a427688f9..caf3477f1 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -17,11 +17,11 @@ import socket import time -import warnings from celery import states from celery.exceptions import ImproperlyConfigured from celery.five import monotonic +from celery.utils import deprecated from celery.utils.log import get_logger from .base import BaseBackend @@ -50,6 +50,10 @@ class CassandraBackend(BaseBackend): _retry_wait = 3 supports_autoexpire = True + @deprecated(description='The old cassandra backend', + deprecation='3.2', + removal='4.0', + alternative='Use the `new_cassandra` result backend instead') def __init__(self, servers=None, keyspace=None, column_family=None, cassandra_options=None, detailed_mode=False, **kwargs): """Initialize Cassandra backend. @@ -99,9 +103,6 @@ def __init__(self, servers=None, keyspace=None, column_family=None, self._column_family = None - warnings.warn("cassandra backend is deprecated. Use new_cassandra instead.", - DeprecationWarning) - def _retry_on_error(self, fun, *args, **kwargs): ts = monotonic() + self._retry_timeout while 1: diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index 3c530f022..02610c887 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -11,6 +11,7 @@ import sys try: # pragma: no cover import cassandra + import cassandra.cluster except ImportError: # pragma: no cover cassandra = None # noqa @@ -19,22 +20,62 @@ from celery.utils.log import get_logger from .base import BaseBackend -__all__ = ['NewCassandraBackend'] +__all__ = ['CassandraBackend'] logger = get_logger(__name__) +E_NO_CASSANDRA = """ +You need to install the cassandra-driver library to +use the Cassandra backend. See https://github.com/datastax/python-driver +""" + +Q_INSERT_RESULT = """ +INSERT INTO {table} ( + task_id, status, result, date_done, traceback, children) VALUES ( + %s, %s, %s, %s, %s, %s) {expires}; +""" + +Q_SELECT_RESULT = """ +SELECT status, result, date_done, traceback, children +FROM {table} +WHERE task_id=%s +LIMIT 1 +""" + +Q_CREATE_RESULT_TABLE = """ +CREATE TABLE {table} ( + task_id text, + status text, + result blob, + date_done timestamp, + traceback blob, + children blob, + PRIMARY KEY ((task_id), date_done) +) WITH CLUSTERING ORDER BY (date_done DESC); +""" + +Q_EXPIRES = """ + USING TTL {0} +""" -class NewCassandraBackend(BaseBackend): - """New Cassandra backend utilizing DataStax driver +if sys.version_info[0] == 3: + def buf_t(x): + return bytes(x, 'utf8') +else: + buf_t = buffer # noqa - .. attribute:: servers - List of Cassandra servers with format: ``hostname`` +class CassandraBackend(BaseBackend): + """Cassandra backend utilizing DataStax driver :raises celery.exceptions.ImproperlyConfigured: if module :mod:`cassandra` is not available. """ + + #: List of Cassandra servers with format: ``hostname``. + servers = None + supports_autoexpire = True # autoexpire supported via entry_ttl def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, @@ -45,12 +86,10 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, the :setting:`CASSANDRA_SERVERS` setting is not set. """ - super(NewCassandraBackend, self).__init__(**kwargs) + super(CassandraBackend, self).__init__(**kwargs) if not cassandra: - raise ImproperlyConfigured( - 'You need to install the cassandra library to use the ' - 'Cassandra backend. See https://github.com/datastax/python-driver') + raise ImproperlyConfigured(E_NO_CASSANDRA) conf = self.app.conf self.servers = (servers or @@ -67,18 +106,20 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, expires = (entry_ttl or conf.get('CASSANDRA_ENTRY_TTL', None)) - if expires is not None: - self.cqlexpires = ' USING TTL %s' % (expires, ) - else: - self.cqlexpires = '' + self.cqlexpires = (Q_EXPIRES.format(expires) + if expires is not None else '') read_cons = conf.get('CASSANDRA_READ_CONSISTENCY') or 'LOCAL_QUORUM' write_cons = conf.get('CASSANDRA_WRITE_CONSISTENCY') or 'LOCAL_QUORUM' - self.read_consistency = getattr(cassandra.ConsistencyLevel, - read_cons, cassandra.ConsistencyLevel.LOCAL_QUORUM) - self.write_consistency = getattr(cassandra.ConsistencyLevel, - write_cons, cassandra.ConsistencyLevel.LOCAL_QUORUM) + self.read_consistency = getattr( + cassandra.ConsistencyLevel, read_cons, + cassandra.ConsistencyLevel.LOCAL_QUORUM, + ) + self.write_consistency = getattr( + cassandra.ConsistencyLevel, write_cons, + cassandra.ConsistencyLevel.LOCAL_QUORUM, + ) self._connection = None self._session = None @@ -87,15 +128,16 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, def process_cleanup(self): if self._connection is not None: - self._session.shutdown() self._connection = None + if self._session is not None: + self._session.shutdown() self._session = None def _get_connection(self, write=False): - """ - Prepare the connection for action + """Prepare the connection for action :param write: bool - are we a writer? + """ if self._connection is None: self._connection = cassandra.cluster.Cluster(self.servers, @@ -105,15 +147,14 @@ def _get_connection(self, write=False): # We are forced to do concatenation below, as formatting would # blow up on superficial %s that will be processed by Cassandra self._write_stmt = cassandra.query.SimpleStatement( - 'INSERT INTO '+self.table+''' (task_id, status, result,''' - ''' date_done, traceback, children) VALUES''' - ' (%s, %s, %s, %s, %s, %s) '+self.cqlexpires+';') + Q_INSERT_RESULT.format( + table=self.table, expires=self.cqlexpires), + ) self._write_stmt.consistency_level = self.write_consistency self._read_stmt = cassandra.query.SimpleStatement( - '''SELECT status, result, date_done, traceback, children - FROM '''+self.table+''' - WHERE task_id=%s LIMIT 1''') + Q_SELECT_RESULT.format(table=self.table), + ) self._read_stmt.consistency_level = self.read_consistency if write: @@ -126,16 +167,8 @@ def _get_connection(self, write=False): # have probably created this table in advance, in which case # this query will be a no-op (instant fail with AlreadyExists) self._make_stmt = cassandra.query.SimpleStatement( - '''CREATE TABLE '''+self.table+''' ( - task_id text, - status text, - result blob, - date_done timestamp, - traceback blob, - children blob, - PRIMARY KEY ((task_id), date_done) - ) - WITH CLUSTERING ORDER BY (date_done DESC);''') + Q_CREATE_RESULT_TABLE.format(table=self.table), + ) self._make_stmt.consistency_level = self.write_consistency try: self._session.execute(self._make_stmt) @@ -147,18 +180,13 @@ def _store_result(self, task_id, result, status, """Store return value and status of an executed task.""" self._get_connection(write=True) - if sys.version_info >= (3,): - buf = lambda x: bytes(x, 'utf8') - else: - buf = buffer - self._session.execute(self._write_stmt, ( task_id, status, - buf(self.encode(result)), + buf_t(self.encode(result)), self.app.now(), - buf(self.encode(traceback)), - buf(self.encode(self.current_task_children(request))) + buf_t(self.encode(traceback)), + buf_t(self.encode(self.current_task_children(request))) )) def _get_task_meta_for(self, task_id): @@ -185,4 +213,4 @@ def __reduce__(self, args=(), kwargs={}): dict(servers=self.servers, keyspace=self.keyspace, table=self.table)) - return super(NewCassandraBackend, self).__reduce__(args, kwargs) + return super(CassandraBackend, self).__reduce__(args, kwargs) diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 17c0ace85..3701b7f91 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + from pickle import loads, dumps from datetime import datetime @@ -8,11 +9,12 @@ AppCase, Mock, mock_module, depends_on_current_app ) + class Object(object): pass -class test_NewCassandraBackend(AppCase): +class test_CassandraBackend(AppCase): def setup(self): self.app.conf.update( @@ -22,16 +24,14 @@ def setup(self): ) def test_init_no_cassandra(self): - """ - Tests behaviour when no python-driver is installed. - new_cassandra should raise ImproperlyConfigured - """ + """should raise ImproperlyConfigured when no python-driver + installed.""" with mock_module('cassandra'): from celery.backends import new_cassandra as mod prev, mod.cassandra = mod.cassandra, None try: with self.assertRaises(ImproperlyConfigured): - mod.NewCassandraBackend(app=self.app) + mod.CassandraBackend(app=self.app) finally: mod.cassandra = prev @@ -45,28 +45,28 @@ def test_init_with_and_without_LOCAL_QUROM(self): self.app.conf.CASSANDRA_READ_CONSISTENCY = 'LOCAL_FOO' self.app.conf.CASSANDRA_WRITE_CONSISTENCY = 'LOCAL_FOO' - mod.NewCassandraBackend(app=self.app) + mod.CassandraBackend(app=self.app) cons.LOCAL_FOO = 'bar' - mod.NewCassandraBackend(app=self.app) + mod.CassandraBackend(app=self.app) # no servers raises ImproperlyConfigured with self.assertRaises(ImproperlyConfigured): self.app.conf.CASSANDRA_SERVERS = None - mod.NewCassandraBackend( + mod.CassandraBackend( app=self.app, keyspace='b', column_family='c', ) @depends_on_current_app def test_reduce(self): with mock_module('cassandra'): - from celery.backends.new_cassandra import NewCassandraBackend - self.assertTrue(loads(dumps(NewCassandraBackend(app=self.app)))) + from celery.backends.new_cassandra import CassandraBackend + self.assertTrue(loads(dumps(CassandraBackend(app=self.app)))) def test_get_task_meta_for(self): with mock_module('cassandra'): from celery.backends import new_cassandra as mod mod.cassandra = Mock() - x = mod.NewCassandraBackend(app=self.app) + x = mod.CassandraBackend(app=self.app) x._connection = True session = x._session = Mock() execute = session.execute = Mock() @@ -86,7 +86,7 @@ def test_store_result(self): from celery.backends import new_cassandra as mod mod.cassandra = Mock() - x = mod.NewCassandraBackend(app=self.app) + x = mod.CassandraBackend(app=self.app) x._connection = True session = x._session = Mock() session.execute = Mock() @@ -95,7 +95,7 @@ def test_store_result(self): def test_process_cleanup(self): with mock_module('cassandra'): from celery.backends import new_cassandra as mod - x = mod.NewCassandraBackend(app=self.app) + x = mod.CassandraBackend(app=self.app) x.process_cleanup() self.assertIsNone(x._connection) From f0bf13c053fa830989fda960ca4fd118244186e5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 13 Oct 2015 17:23:00 -0700 Subject: [PATCH 0688/1103] Fixes broken test --- celery/tests/backends/test_new_cassandra.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 3701b7f91..bc0188f18 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -9,6 +9,8 @@ AppCase, Mock, mock_module, depends_on_current_app ) +CASSANDRA_MODULES = ['cassandra', 'cassandra.cluster'] + class Object(object): pass @@ -26,7 +28,7 @@ def setup(self): def test_init_no_cassandra(self): """should raise ImproperlyConfigured when no python-driver installed.""" - with mock_module('cassandra'): + with mock_module(*CASSANDRA_MODULES): from celery.backends import new_cassandra as mod prev, mod.cassandra = mod.cassandra, None try: @@ -36,7 +38,7 @@ def test_init_no_cassandra(self): mod.cassandra = prev def test_init_with_and_without_LOCAL_QUROM(self): - with mock_module('cassandra'): + with mock_module(*CASSANDRA_MODULES): from celery.backends import new_cassandra as mod mod.cassandra = Mock() cons = mod.cassandra.ConsistencyLevel = Object() @@ -58,12 +60,12 @@ def test_init_with_and_without_LOCAL_QUROM(self): @depends_on_current_app def test_reduce(self): - with mock_module('cassandra'): + with mock_module(*CASSANDRA_MODULES): from celery.backends.new_cassandra import CassandraBackend self.assertTrue(loads(dumps(CassandraBackend(app=self.app)))) def test_get_task_meta_for(self): - with mock_module('cassandra'): + with mock_module(*CASSANDRA_MODULES): from celery.backends import new_cassandra as mod mod.cassandra = Mock() x = mod.CassandraBackend(app=self.app) @@ -82,7 +84,7 @@ def test_get_task_meta_for(self): self.assertEqual(meta['status'], states.PENDING) def test_store_result(self): - with mock_module('cassandra'): + with mock_module(*CASSANDRA_MODULES): from celery.backends import new_cassandra as mod mod.cassandra = Mock() @@ -93,7 +95,7 @@ def test_store_result(self): x._store_result('task_id', 'result', states.SUCCESS) def test_process_cleanup(self): - with mock_module('cassandra'): + with mock_module(*CASSANDRA_MODULES): from celery.backends import new_cassandra as mod x = mod.CassandraBackend(app=self.app) x.process_cleanup() From a4bed4dd625e5ca20b7682eac4081c556978ddde Mon Sep 17 00:00:00 2001 From: Paul Pearce Date: Wed, 14 Oct 2015 10:41:47 -0700 Subject: [PATCH 0689/1103] Fixed Control.disable_events() documentation bug --- celery/app/control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/control.py b/celery/app/control.py index 284537493..10baf59e9 100644 --- a/celery/app/control.py +++ b/celery/app/control.py @@ -263,7 +263,7 @@ def enable_events(self, destination=None, **kwargs): return self.broadcast('enable_events', {}, destination, **kwargs) def disable_events(self, destination=None, **kwargs): - """Tell all (or specific) workers to enable events.""" + """Tell all (or specific) workers to disable events.""" return self.broadcast('disable_events', {}, destination, **kwargs) def pool_grow(self, n=1, destination=None, **kwargs): From a06b94ea82b12670746c3c60f8796bc0188f4402 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 14 Oct 2015 13:01:09 -0700 Subject: [PATCH 0690/1103] Adds missing Cassandra settings to celery.app.defaults --- celery/app/defaults.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index f5edd7fc0..4f1558aaf 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -93,6 +93,8 @@ def __repr__(self): 'KEYSPACE': Option(type='string'), 'READ_CONSISTENCY': Option(type='string'), 'SERVERS': Option(type='list'), + 'PORT': Option(type="string"), + 'ENTRY_TTL': Option(type="float"), 'WRITE_CONSISTENCY': Option(type='string'), }, 'CELERY': { From c0fc4217fe26365ee31ee86261d84c67bbbaa32e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 14 Oct 2015 13:01:49 -0700 Subject: [PATCH 0691/1103] Document the reject_on_worker_lost setting properly. #2840 --- celery/app/task.py | 14 ++++++++++---- docs/configuration.rst | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index 707366c72..4b422c90d 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -220,10 +220,16 @@ class Task(object): #: :setting:`CELERY_ACKS_LATE` setting. acks_late = None - #: When CELERY_ACKS_LATE is set to True, the default behavior to - #: handle worker crash is to acknowledge the message. Setting - #: this to true allows the message to be rejected and requeued so - #: it will be executed again by another worker. + #: Even if :attr:`acks_late` is enabled, the worker will + #: acknowledge tasks when the worker process executing them abrubtly + #: exits or is signalled (e.g. :sig:`KILL`/:sig:`INT`, etc). + #: + #: Setting this to true allows the message to be requeued instead, + #: so that the task will execute again by the same worker, or another + #: worker. + #: + #: Warning: Enabling this can cause message loops; make sure you know + #: what you're doing. reject_on_worker_lost = None #: Tuple of expected exceptions. diff --git a/docs/configuration.rst b/docs/configuration.rst index bf65dd1d1..90cba2a4f 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1585,6 +1585,24 @@ has been executed, not *just before*, which is the default behavior. FAQ: :ref:`faq-acks_late-vs-retry`. +.. setting:: CELERY_REJECT_ON_WORKER_LOST + +CELERY_REJECT_ON_WORKER_LOST +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Even if :attr:`acks_late` is enabled, the worker will +acknowledge tasks when the worker process executing them abrubtly +exits or is signalled (e.g. :sig:`KILL`/:sig:`INT`, etc). + +Setting this to true allows the message to be requeued instead, +so that the task will execute again by the same worker, or another +worker. + +.. warning:: + + Enabling this can cause message loops; make sure you know + what you're doing. + .. _conf-worker: Worker From 96fc21e40bfa4d903b7fd519ca2f48d4b777bd01 Mon Sep 17 00:00:00 2001 From: Paul Pearce Date: Thu, 15 Oct 2015 12:50:48 -0700 Subject: [PATCH 0692/1103] Document the local-only behavior of Task.retry() max_retries --- celery/app/task.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index 4b422c90d..76c4d1f2f 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -571,10 +571,12 @@ def retry(self, args=None, kwargs=None, exc=None, throw=True, :keyword countdown: Time in seconds to delay the retry for. :keyword eta: Explicit time and date to run the retry at (must be a :class:`~datetime.datetime` instance). - :keyword max_retries: If set, overrides the default retry limit. - A value of :const:`None`, means "use the default", so if you want - infinite retries you would have to set the :attr:`max_retries` - attribute of the task to :const:`None` first. + :keyword max_retries: If set, overrides the default retry limit for + this execution. Changes to this parameter do not propagate to + subsequent task retry attempts. A value of :const:`None`, means + "use the default", so if you want infinite retries you would + have to set the :attr:`max_retries` attribute of the task to + :const:`None` first. :keyword time_limit: If set, overrides the default time limit. :keyword soft_time_limit: If set, overrides the default soft time limit. From c53928b21a0b024dcc6df98d25004c7ccf8705c6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 16 Oct 2015 11:46:48 -0700 Subject: [PATCH 0693/1103] Celery 3.2 is now Celery 4.0 --- Changelog | 12 ++++---- README.rst | 2 +- celery/__init__.py | 4 +-- celery/app/base.py | 4 +-- celery/apps/worker.py | 4 +-- celery/backends/cassandra.py | 4 +-- celery/backends/redis.py | 8 +++--- celery/events/__init__.py | 2 +- celery/events/state.py | 32 ++++++++++----------- celery/result.py | 4 +-- docs/configuration.rst | 2 +- docs/history/changelog-3.1.rst | 8 +++--- docs/includes/introduction.txt | 2 +- docs/index.rst | 2 +- docs/internals/deprecation.rst | 6 ++-- docs/internals/protocol.rst | 2 +- docs/userguide/tasks.rst | 4 +-- docs/whatsnew-3.1.rst | 8 +++--- docs/{whatsnew-3.2.rst => whatsnew-4.0.rst} | 4 +-- setup.py | 2 +- 20 files changed, 58 insertions(+), 58 deletions(-) rename docs/{whatsnew-3.2.rst => whatsnew-4.0.rst} (99%) diff --git a/Changelog b/Changelog index 11eb699e6..201d85cd3 100644 --- a/Changelog +++ b/Changelog @@ -4,15 +4,15 @@ Change history ================ -This document contains change notes for bugfix releases in the 3.2.x series -(Cipater), please see :ref:`whatsnew-3.2` for an overview of what's -new in Celery 3.2. +This document contains change notes for bugfix releases in the 4.0.x series +(Cipater), please see :ref:`whatsnew-4.0` for an overview of what's +new in Celery 4.0. -.. _version-3.2.0: +.. _version-4.0.0: -3.2.0 +4.0.0 ======= :release-date: TBA :release-by: -See :ref:`whatsnew-3.2`. +See :ref:`whatsnew-4.0`. diff --git a/README.rst b/README.rst index 3391e16be..f42044da3 100644 --- a/README.rst +++ b/README.rst @@ -6,7 +6,7 @@ |build-status| |coverage-status| -:Version: 3.2.0a1 (Cipater) +:Version: 4.0.0a1 (Cipater) :Web: http://celeryproject.org/ :Download: http://pypi.python.org/pypi/celery/ :Source: http://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index 65ef1446c..d94678441 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,8 +17,8 @@ 'version_info_t', ('major', 'minor', 'micro', 'releaselevel', 'serial'), ) -SERIES = 'DEV' -VERSION = version_info_t(3, 2, 0, 'a2', '') +SERIES = '0today8' +VERSION = version_info_t(4, 0, 0, 'a1', '') __version__ = '{0.major}.{0.minor}.{0.micro}{0.releaselevel}'.format(VERSION) __author__ = 'Ask Solem' __contact__ = 'ask@celeryproject.org' diff --git a/celery/app/base.py b/celery/app/base.py index 32f5ffcd0..34cfbd4e1 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -195,7 +195,7 @@ def __init__(self, main=None, loader=None, backend=None, # Signals if self.on_configure is None: - # used to be a method pre 3.2 + # used to be a method pre 4.0 self.on_configure = Signal() self.on_after_configure = Signal() self.on_after_finalize = Signal() @@ -521,7 +521,7 @@ def _load_config(self): if isinstance(self.on_configure, Signal): self.on_configure.send(sender=self) else: - # used to be a method pre 3.2 + # used to be a method pre 4.0 self.on_configure() if self._config_source: self.loader.config_from_object(self._config_source) diff --git a/celery/apps/worker.py b/celery/apps/worker.py index 27b419d78..cfb302795 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -45,7 +45,7 @@ is_pypy = hasattr(sys, 'pypy_version_info') W_PICKLE_DEPRECATED = """ -Starting from version 3.2 Celery will refuse to accept pickle by default. +Starting from version 4.0 Celery will refuse to accept pickle by default. The pickle serializer is a security concern as it may give attackers the ability to execute any command. It's important to secure @@ -55,7 +55,7 @@ If you depend on pickle then you should set a setting to disable this warning and to be sure that everything will continue working -when you upgrade to Celery 3.2:: +when you upgrade to Celery 4.0:: CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml'] diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index caf3477f1..07c5880eb 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -51,8 +51,8 @@ class CassandraBackend(BaseBackend): supports_autoexpire = True @deprecated(description='The old cassandra backend', - deprecation='3.2', - removal='4.0', + deprecation='4.0', + removal='5.0', alternative='Use the `new_cassandra` result backend instead') def __init__(self, servers=None, keyspace=None, column_family=None, cassandra_options=None, detailed_mode=False, **kwargs): diff --git a/celery/backends/redis.py b/celery/backends/redis.py index fb1eaba6d..6a0f13c65 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -264,18 +264,18 @@ def __reduce__(self, args=(), kwargs={}): (self.url,), {'expires': self.expires}, ) - @deprecated_property(3.2, 3.3) + @deprecated_property(4.0, 5.0) def host(self): return self.connparams['host'] - @deprecated_property(3.2, 3.3) + @deprecated_property(4.0, 5.0) def port(self): return self.connparams['port'] - @deprecated_property(3.2, 3.3) + @deprecated_property(4.0, 5.0) def db(self): return self.connparams['db'] - @deprecated_property(3.2, 3.3) + @deprecated_property(4.0, 5.0) def password(self): return self.connparams['password'] diff --git a/celery/events/__init__.py b/celery/events/__init__.py index d21df35a8..1fcf36ee7 100644 --- a/celery/events/__init__.py +++ b/celery/events/__init__.py @@ -394,7 +394,7 @@ def event_from_message(self, body, localize=True, return type, body def _receive(self, body, message, list=list, isinstance=isinstance): - if isinstance(body, list): # 3.2: List of events + if isinstance(body, list): # celery 4.0: List of events process, from_message = self.process, self.event_from_message [process(*from_message(event)) for event in body] else: diff --git a/celery/events/state.py b/celery/events/state.py index 74284a6d1..549f8dfcf 100644 --- a/celery/events/state.py +++ b/celery/events/state.py @@ -200,25 +200,25 @@ def alive(self, nowfun=time): def id(self): return '{0.hostname}.{0.pid}'.format(self) - @deprecated(3.2, 3.3) + @deprecated(4.0, 5.0) def update_heartbeat(self, received, timestamp): self.event(None, timestamp, received) - @deprecated(3.2, 3.3) + @deprecated(4.0, 5.0) def on_online(self, timestamp=None, local_received=None, **fields): self.event('online', timestamp, local_received, fields) - @deprecated(3.2, 3.3) + @deprecated(4.0, 5.0) def on_offline(self, timestamp=None, local_received=None, **fields): self.event('offline', timestamp, local_received, fields) - @deprecated(3.2, 3.3) + @deprecated(4.0, 5.0) def on_heartbeat(self, timestamp=None, local_received=None, **fields): self.event('heartbeat', timestamp, local_received, fields) @class_property def _defaults(cls): - """Deprecated, to be removed in 3.3""" + """Deprecated, to be removed in 5.0""" source = cls() return {k: getattr(source, k) for k in cls._fields} @@ -336,44 +336,44 @@ def origin(self): def ready(self): return self.state in states.READY_STATES - @deprecated(3.2, 3.3) + @deprecated(4.0, 5.0) def on_sent(self, timestamp=None, **fields): self.event('sent', timestamp, fields) - @deprecated(3.2, 3.3) + @deprecated(4.0, 5.0) def on_received(self, timestamp=None, **fields): self.event('received', timestamp, fields) - @deprecated(3.2, 3.3) + @deprecated(4.0, 5.0) def on_started(self, timestamp=None, **fields): self.event('started', timestamp, fields) - @deprecated(3.2, 3.3) + @deprecated(4.0, 5.0) def on_failed(self, timestamp=None, **fields): self.event('failed', timestamp, fields) - @deprecated(3.2, 3.3) + @deprecated(4.0, 5.0) def on_retried(self, timestamp=None, **fields): self.event('retried', timestamp, fields) - @deprecated(3.2, 3.3) + @deprecated(4.0, 5.0) def on_succeeded(self, timestamp=None, **fields): self.event('succeeded', timestamp, fields) - @deprecated(3.2, 3.3) + @deprecated(4.0, 5.0) def on_revoked(self, timestamp=None, **fields): self.event('revoked', timestamp, fields) - @deprecated(3.2, 3.3) + @deprecated(4.0, 5.0) def on_unknown_event(self, shortype, timestamp=None, **fields): self.event(shortype, timestamp, fields) - @deprecated(3.2, 3.3) + @deprecated(4.0, 5.0) def update(self, state, timestamp, fields, _state=states.state, RETRY=states.RETRY): return self.event(state, timestamp, None, fields) - @deprecated(3.2, 3.3) + @deprecated(4.0, 5.0) def merge(self, state, timestamp, fields): keep = self.merge_rules.get(state) if keep is not None: @@ -383,7 +383,7 @@ def merge(self, state, timestamp, fields): @class_property def _defaults(cls): - """Deprecated, to be removed in 3.3.""" + """Deprecated, to be removed in 5.0.""" source = cls() return {k: getattr(source, k) for k in source._fields} diff --git a/celery/result.py b/celery/result.py index 12c01d121..b12de6857 100644 --- a/celery/result.py +++ b/celery/result.py @@ -34,7 +34,7 @@ See http://docs.celeryq.org/en/latest/userguide/tasks.html\ #task-synchronous-subtasks -In Celery 3.2 this will result in an exception being +In Celery 4.0 this will result in an exception being raised instead of just being a warning. """ @@ -542,7 +542,7 @@ def __getitem__(self, index): """`res[i] -> res.results[i]`""" return self.results[index] - @deprecated('3.2', '3.3') + @deprecated('4.0', '5.0') def iterate(self, timeout=None, propagate=True, interval=0.5): """Deprecated method, use :meth:`get` with a callback argument.""" elapsed = 0.0 diff --git a/docs/configuration.rst b/docs/configuration.rst index 90cba2a4f..e2d139816 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1813,7 +1813,7 @@ The default is 2 seconds. EMAIL_CHARSET ~~~~~~~~~~~~~ -.. versionadded:: 3.2.0 +.. versionadded:: 4.0 Charset for outgoing emails. Default is "us-ascii". diff --git a/docs/history/changelog-3.1.rst b/docs/history/changelog-3.1.rst index 860580250..1240e3a99 100644 --- a/docs/history/changelog-3.1.rst +++ b/docs/history/changelog-3.1.rst @@ -467,7 +467,7 @@ News See :ref:`redis-caveats`. - This will be the default in Celery 3.2. + This will be the default in Celery 4.0. - **Results**: The :class:`@AsyncResult` object now keeps a local cache of the final state of the task. @@ -476,7 +476,7 @@ News and you can do so by setting :setting:`CELERY_MAX_CACHED_RESULTS` to :const:`-1`. The lifetime of the cache will then be bound to the lifetime of the result object, which will be the default behavior - in Celery 3.2. + in Celery 4.0. - **Events**: The "Substantial drift" warning message is now logged once per node name only (Issue #1802). @@ -682,7 +682,7 @@ News - **Results:** ``ResultSet.iterate`` is now pending deprecation. - The method will be deprecated in version 3.2 and removed in version 3.3. + The method will be removed in version 4.0 and removed in version 5.0. Use ``result.get(callback=)`` (or ``result.iter_native()`` where available) instead. @@ -832,7 +832,7 @@ Synchronous subtasks Tasks waiting for the result of a subtask will now emit a :exc:`RuntimeWarning` warning when using the prefork pool, -and in 3.2 this will result in an exception being raised. +and in 4.0 this will result in an exception being raised. It's not legal for tasks to block by waiting for subtasks as this is likely to lead to resource starvation and eventually diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index da5fda4a1..0aff1ea0b 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 3.2.0a1 (Cipater) +:Version: 4.0.0a1 (0today8) :Web: http://celeryproject.org/ :Download: http://pypi.python.org/pypi/celery/ :Source: http://github.com/celery/celery/ diff --git a/docs/index.rst b/docs/index.rst index 7d2c32381..bb0418df7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -49,7 +49,7 @@ Contents tutorials/index faq changelog - whatsnew-3.2 + whatsnew-4.0 whatsnew-3.1 whatsnew-3.0 whatsnew-2.5 diff --git a/docs/internals/deprecation.rst b/docs/internals/deprecation.rst index ef68be949..746e7ae24 100644 --- a/docs/internals/deprecation.rst +++ b/docs/internals/deprecation.rst @@ -7,14 +7,14 @@ .. contents:: :local: -.. _deprecations-v3.2: +.. _deprecations-v4.0: -Removals for version 3.2 +Removals for version 4.0 ======================== - Module ``celery.task.trace`` has been renamed to ``celery.app.trace`` as the ``celery.task`` package is being phased out. The compat module - will be removed in version 3.2 so please change any import from:: + will be removed in version 4.0 so please change any import from:: from celery.task.trace import … diff --git a/docs/internals/protocol.rst b/docs/internals/protocol.rst index 9e6ffd7f8..7cc334982 100644 --- a/docs/internals/protocol.rst +++ b/docs/internals/protocol.rst @@ -310,7 +310,7 @@ Event Messages Event messages are always JSON serialized and can contain arbitrary message body fields. -Since version 3.2. the body can consist of either a single mapping (one event), +Since version 4.0. the body can consist of either a single mapping (one event), or a list of mappings (multiple events). There are also standard fields that must always be present in an event diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 6a5ae378c..9fe417af4 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -224,7 +224,7 @@ on the automatic naming: Changing the automatic naming behavior -------------------------------------- -.. versionadded:: 3.2 +.. versionadded:: 4.0 There are some cases when the default automatic naming is not suitable. Consider you have many tasks within many different modules:: @@ -503,7 +503,7 @@ override this default. Autoretrying ------------ -.. versionadded:: 3.2 +.. versionadded:: 4.0 Sometimes you may want to retry a task on particular exception. To do so, you should wrap a task body with `try-except` statement, for example: diff --git a/docs/whatsnew-3.1.rst b/docs/whatsnew-3.1.rst index 3dc416017..5a77ef926 100644 --- a/docs/whatsnew-3.1.rst +++ b/docs/whatsnew-3.1.rst @@ -73,7 +73,7 @@ these transports or donate resources to improve them, but as the situation is now I don't think the quality is up to date with the rest of the code-base so I cannot recommend them for production use. -The next version of Celery 3.2 will focus on performance and removing +The next version of Celery 4.0 will focus on performance and removing rarely used parts of the library. Work has also started on a new message protocol, supporting multiple languages and more. The initial draft can be found :ref:`here `. @@ -101,13 +101,13 @@ requiring the ``2to3`` porting tool. .. note:: - This is also the last version to support Python 2.6! From Celery 3.2 and + This is also the last version to support Python 2.6! From Celery 4.0 and onwards Python 2.7 or later will be required. Last version to enable Pickle by default ---------------------------------------- -Starting from Celery 3.2 the default serializer will be json. +Starting from Celery 4.0 the default serializer will be json. If you depend on pickle being accepted you should be prepared for this change by explicitly allowing your worker @@ -138,7 +138,7 @@ Everyone should move to the new :program:`celery` umbrella command, so we are incrementally deprecating the old command names. In this version we've removed all commands that are not used -in init scripts. The rest will be removed in 3.2. +in init scripts. The rest will be removed in 4.0. +-------------------+--------------+-------------------------------------+ | Program | New Status | Replacement | diff --git a/docs/whatsnew-3.2.rst b/docs/whatsnew-4.0.rst similarity index 99% rename from docs/whatsnew-3.2.rst rename to docs/whatsnew-4.0.rst index df39c186f..aed087003 100644 --- a/docs/whatsnew-3.2.rst +++ b/docs/whatsnew-4.0.rst @@ -1,7 +1,7 @@ -.. _whatsnew-3.2: +.. _whatsnew-4.0: =========================================== - What's new in Celery 3.2 (TBA) + What's new in Celery 4.0 (TBA) =========================================== :Author: Ask Solem (ask at celeryproject.org) diff --git a/setup.py b/setup.py index b08e44654..e678ee7bb 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ CELERY_COMPAT_PROGRAMS = int(os.environ.get('CELERY_COMPAT_PROGRAMS', 1)) if sys.version_info < (2, 7): - raise Exception('Celery 3.2 requires Python 2.7 or higher.') + raise Exception('Celery 4.0 requires Python 2.7 or higher.') # -*- Upgrading from older versions -*- From e71652d384b1b5df2a4e6145df9f0efb456bc71c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 15 Oct 2015 15:43:51 -0700 Subject: [PATCH 0694/1103] Message protocol v2 now includes repr of args/kwargs. Closes #2847 --- celery/app/amqp.py | 15 ++- celery/tests/utils/test_saferepr.py | 167 +++++++++++++++++++++++++++ celery/tests/utils/test_text.py | 9 +- celery/utils/saferepr.py | 170 ++++++++++++++++++++++++++++ celery/utils/text.py | 2 +- celery/worker/request.py | 11 +- celery/worker/strategy.py | 9 +- docs/internals/protocol.rst | 8 +- 8 files changed, 376 insertions(+), 15 deletions(-) create mode 100644 celery/tests/utils/test_saferepr.py create mode 100644 celery/utils/saferepr.py diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 640442b8c..e1aa3dcc8 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -18,11 +18,11 @@ from kombu.common import Broadcast from kombu.pools import ProducerPool from kombu.utils import cached_property -from kombu.utils.encoding import safe_repr from kombu.utils.functional import maybe_list from celery import signals from celery.five import items, string_t +from celery.utils.saferepr import saferepr from celery.utils.text import indent as textindent from celery.utils.timeutils import to_utc @@ -293,6 +293,9 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, eta = eta and eta.isoformat() expires = expires and expires.isoformat() + argsrepr = saferepr(args) + kwargsrepr = saferepr(kwargs) + return task_message( headers={ 'lang': 'py', @@ -305,6 +308,8 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, 'timelimit': [time_limit, soft_time_limit], 'root_id': root_id, 'parent_id': parent_id, + 'argsrepr': argsrepr, + 'kwargsrepr': kwargsrepr, }, properties={ 'correlation_id': task_id, @@ -323,8 +328,8 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, 'root': root_id, 'parent': parent_id, 'name': name, - 'args': safe_repr(args), - 'kwargs': safe_repr(kwargs), + 'args': argsrepr, + 'kwargs': kwargsrepr, 'retries': retries, 'eta': eta, 'expires': expires, @@ -385,8 +390,8 @@ def as_task_v1(self, task_id, name, args=None, kwargs=None, sent_event={ 'uuid': task_id, 'name': name, - 'args': safe_repr(args), - 'kwargs': safe_repr(kwargs), + 'args': saferepr(args), + 'kwargs': saferepr(kwargs), 'retries': retries, 'eta': eta, 'expires': expires, diff --git a/celery/tests/utils/test_saferepr.py b/celery/tests/utils/test_saferepr.py new file mode 100644 index 000000000..4b04143e3 --- /dev/null +++ b/celery/tests/utils/test_saferepr.py @@ -0,0 +1,167 @@ +from __future__ import absolute_import, unicode_literals + +import re + +from decimal import Decimal +from pprint import pprint + +from celery.five import items, long_t, values + +from celery.utils.saferepr import saferepr + +from celery.tests.case import Case + +EXPECTED_1 = """\ +{'rest': {'baz': 'The quick brown fox jumps over the lazy dog.', \ +'foo': 'The quick brown fox jumps...', ...}}\ +""" + +D_NUMBERS = { + b'integer': 1, + b'float': 1.3, + b'decimal': Decimal("1.3"), + b'long': long_t(1.3), + b'complex': complex(13.3), +} +D_INT_KEYS = {v: k for k, v in items(D_NUMBERS)} + +QUICK_BROWN_FOX = 'The quick brown fox jumps over the lazy dog.' +B_QUICK_BROWN_FOX = b'The quick brown fox jumps over the lazy dog.' + +D_TEXT = { + b'foo': QUICK_BROWN_FOX, + b'bar': B_QUICK_BROWN_FOX, + b'baz': B_QUICK_BROWN_FOX, + b'xuzzy': B_QUICK_BROWN_FOX, +} + +L_NUMBERS = list(values(D_NUMBERS)) + +D_TEXT_LARGE = { + b'bazxuzzyfoobarlongverylonglong': QUICK_BROWN_FOX * 30, +} + +D_ALL = { + b'numbers': D_NUMBERS, + b'intkeys': D_INT_KEYS, + b'text': D_TEXT, + b'largetext': D_TEXT_LARGE, +} + +D_D_TEXT = {b'rest': D_TEXT} + +RE_OLD_SET_REPR = re.compile(r'(?:frozen)?set\d?\(\[(.+?)\]\)') +RE_OLD_SET_REPR_REPLACE = r'{\1}' + + +def from_old_repr(s): + return RE_OLD_SET_REPR.sub( + RE_OLD_SET_REPR_REPLACE, s).replace("u'", "'") + + +class list2(list): + pass + + +class list3(list): + + def __repr__(self): + return list.__repr__(self) + + +class tuple2(tuple): + pass + + +class tuple3(tuple): + + def __repr__(self): + return tuple.__repr__(self) + + +class set2(set): + pass + + +class set3(set): + + def __repr__(self): + return set.__repr__(self) + + +class frozenset2(frozenset): + pass + + +class frozenset3(frozenset): + + def __repr__(self): + return frozenset.__repr__(self) + + +class dict2(dict): + pass + + +class dict3(dict): + + def __repr__(self): + return dict.__repr__(self) + + +class Unorderable: + + def __repr__(self): + return str(id(self)) + + +class test_saferepr(Case): + + def test_safe_types(self): + for value in values(D_NUMBERS): + self.assertEqual(saferepr(value), repr(value)) + + def test_numbers_dict(self): + self.assertEqual(saferepr(D_NUMBERS), repr(D_NUMBERS)) + + def test_numbers_list(self): + self.assertEqual(saferepr(L_NUMBERS), repr(L_NUMBERS)) + + def test_numbers_keys(self): + self.assertEqual(saferepr(D_INT_KEYS), repr(D_INT_KEYS)) + + def test_text(self): + self.assertEqual(saferepr(D_TEXT), repr(D_TEXT).replace("u'", "'")) + + def test_text_maxlen(self): + self.assertEqual(saferepr(D_D_TEXT, 100), EXPECTED_1) + + def test_same_as_repr(self): + # Simple objects, small containers and classes that overwrite __repr__ + # For those the result should be the same as repr(). + # Ahem. The docs don't say anything about that -- this appears to + # be testing an implementation quirk. Starting in Python 2.5, it's + # not true for dicts: pprint always sorts dicts by key now; before, + # it sorted a dict display if and only if the display required + # multiple lines. For that reason, dicts with more than one element + # aren't tested here. + types = ( + 0, 0, 0+0j, 0.0, "", b"", + (), tuple2(), tuple3(), + [], list2(), list3(), + set(), set2(), set3(), + frozenset(), frozenset2(), frozenset3(), + {}, dict2(), dict3(), + self.assertTrue, pprint, + -6, -6, -6-6j, -1.5, "x", b"x", (3,), [3], {3: 6}, + (1, 2), [3, 4], {5: 6}, + tuple2((1, 2)), tuple3((1, 2)), tuple3(range(100)), + [3, 4], list2([3, 4]), list3([3, 4]), list3(range(100)), + set({7}), set2({7}), set3({7}), + frozenset({8}), frozenset2({8}), frozenset3({8}), + dict2({5: 6}), dict3({5: 6}), + range(10, -11, -1) + ) + for simple in types: + native = from_old_repr(repr(simple)) + self.assertEqual(saferepr(simple), native) diff --git a/celery/tests/utils/test_text.py b/celery/tests/utils/test_text.py index 383bdb6ee..1b0ca2805 100644 --- a/celery/tests/utils/test_text.py +++ b/celery/tests/utils/test_text.py @@ -1,13 +1,14 @@ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals from celery.utils.text import ( - indent, - ensure_2lines, abbr, - truncate, abbrtask, + ensure_2lines, + indent, pretty, + truncate, ) + from celery.tests.case import AppCase, Case RANDTEXT = """\ diff --git a/celery/utils/saferepr.py b/celery/utils/saferepr.py new file mode 100644 index 000000000..b49c79a00 --- /dev/null +++ b/celery/utils/saferepr.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- +""" + celery.utils.saferepr + ~~~~~~~~~~~~~~~~~~~~~ + + Streaming, truncating, non-recursive version of :func:`repr`. + + Differences from regular :func:`repr`: + + - Sets are represented the Python 3 way: ``{1, 2}`` vs ``set([1, 2])``. + - Unicode strings does not have the ``u'`` prefix, even on Python 2. + + Very slow with no limits, super quick with limits. + +""" +from collections import Iterable, Mapping, deque, namedtuple + +from itertools import chain +from numbers import Number +from pprint import _recursion + +from celery.five import items, text_t + +from .text import truncate + +__all__ = ['saferepr'] + +_literal = namedtuple('_literal', ('value', 'truncate', 'direction')) +_key = namedtuple('_key', ('value',)) +_quoted = namedtuple('_quoted', ('value',)) +_dirty = namedtuple('_dirty', ('objid',)) + +chars_t = (bytes, text_t) +literal_t = (_literal, _key) +safe_t = (Number,) +set_t = (frozenset, set) + +LIT_DICT_START = _literal('{', False, +1) +LIT_DICT_KVSEP = _literal(': ', True, 0) +LIT_DICT_END = _literal('}', False, -1) +LIT_LIST_START = _literal('[', False, +1) +LIT_LIST_END = _literal(']', False, -1) +LIT_LIST_SEP = _literal(', ', True, 0) +LIT_SET_START = _literal('{', False, +1) +LIT_SET_END = _literal('}', False, -1) +LIT_TUPLE_START = _literal('(', False, +1) +LIT_TUPLE_END = _literal(')', False, -1) +LIT_TUPLE_END_SV = _literal(',)', False, -1) + + +def saferepr(o, maxlen=None, maxlevels=3, seen=None): + return ''.join(_saferepr( + o, maxlen=maxlen, maxlevels=maxlevels, seen=seen + )) + + +def _chaindict(mapping, + LIT_DICT_KVSEP=LIT_DICT_KVSEP, + LIT_LIST_SEP=LIT_LIST_SEP): + size = len(mapping) + for i, (k, v) in enumerate(items(mapping)): + yield _key(k) + yield LIT_DICT_KVSEP + yield v + if i < (size - 1): + yield LIT_LIST_SEP + + +def _chainlist(it, LIT_LIST_SEP=LIT_LIST_SEP): + size = len(it) + for i, v in enumerate(it): + yield v + if i < (size - 1): + yield LIT_LIST_SEP + + +def _repr_empty_set(s): + return '%s([])' % (type(s).__name__,) + + +def _saferepr(o, maxlen=None, maxlevels=3, seen=None): + stack = deque([iter([o])]) + for token, it in reprstream(stack, seen=seen, maxlevels=maxlevels): + if maxlen is not None and maxlen <= 0: + yield ', ...' + # move rest back to stack, so that we can include + # dangling parens. + stack.append(it) + break + if isinstance(token, _literal): + val = str(token.value) + elif isinstance(token, _key): + val = repr(token.value).replace("u'", "'") + elif isinstance(token, _quoted): + val = "'%s'" % (truncate(token.value, maxlen),) + else: + val = truncate(token, maxlen) + yield val + if maxlen is not None: + maxlen -= len(val) + for rest1 in stack: + # maxlen exceeded, process any dangling parens. + for rest2 in rest1: + if isinstance(rest2, _literal) and not rest2.truncate: + yield rest2.value + + +def reprstream(stack, seen=None, maxlevels=3, level=0, isinstance=isinstance): + seen = seen or set() + append = stack.append + popleft = stack.popleft + is_in_seen = seen.__contains__ + discard_from_seen = seen.discard + add_to_seen = seen.add + + while stack: + lit_start = lit_end = None + it = popleft() + for val in it: + orig = val + if isinstance(val, _dirty): + discard_from_seen(val.objid) + continue + elif isinstance(val, _literal): + level += val.direction + yield val, it + elif isinstance(val, _key): + yield val, it + elif isinstance(val, safe_t): + yield repr(val), it + elif isinstance(val, chars_t): + yield _quoted(val), it + else: + if isinstance(val, set_t): + if not val: + yield _repr_empty_set(val), it + continue + lit_start, lit_end, val = ( + LIT_SET_START, LIT_SET_END, _chainlist(val)) + elif isinstance(val, tuple): + lit_start, lit_end, val = ( + LIT_TUPLE_START, + LIT_TUPLE_END_SV if len(val) == 1 else LIT_TUPLE_END, + _chainlist(val)) + elif isinstance(val, Mapping): + lit_start, lit_end, val = ( + LIT_DICT_START, LIT_DICT_END, _chaindict(val)) + elif isinstance(val, Iterable): + lit_start, lit_end, val = ( + LIT_LIST_START, LIT_LIST_END, _chainlist(val)) + else: + # other type of object + yield repr(val), it + continue + + if maxlevels and level >= maxlevels: + yield "%s...%s" % (lit_start.value, lit_end.value), it + continue + + objid = id(orig) + if is_in_seen(objid): + yield _recursion(orig), it + continue + add_to_seen(objid) + + # Recurse into the new list/tuple/dict/etc by tacking + # the rest of our iterable onto the new it: this way + # it works similar to a linked list. + append(chain([lit_start], val, [_dirty(objid), lit_end], it)) + break diff --git a/celery/utils/text.py b/celery/utils/text.py index ffd2d72fa..d416b030e 100644 --- a/celery/utils/text.py +++ b/celery/utils/text.py @@ -64,7 +64,7 @@ def indent(t, indent=0, sep='\n'): def truncate(text, maxlen=128, suffix='...'): """Truncates text to a maximum number of characters.""" - if len(text) >= maxlen: + if maxlen and len(text) >= maxlen: return text[:maxlen].rsplit(' ', 1)[0] + suffix return text diff --git a/celery/worker/request.py b/celery/worker/request.py index f809c10b5..bab64b54d 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -79,7 +79,7 @@ class Request(object): 'app', 'type', 'name', 'id', 'on_ack', 'body', 'hostname', 'eventer', 'connection_errors', 'task', 'eta', 'expires', 'request_dict', 'on_reject', 'utc', - 'content_type', 'content_encoding', + 'content_type', 'content_encoding', 'argsrepr', 'kwargsrepr', '__weakref__', '__dict__', ) @@ -111,6 +111,8 @@ def __init__(self, message, on_ack=noop, self.name = headers['shadow'] if 'timelimit' in headers: self.time_limits = headers['timelimit'] + self.argsrepr = headers.get('argsrepr', '') + self.kwargsrepr = headers.get('kwargsrepr', '') self.on_ack = on_ack self.on_reject = on_reject self.hostname = hostname or socket.gethostname() @@ -384,6 +386,8 @@ def reject(self, requeue=False): def info(self, safe=False): return {'id': self.id, 'name': self.name, + 'args': self.argsrepr, + 'kwargs': self.kwargsrepr, 'type': self.type, 'body': self.body, 'hostname': self.hostname, @@ -404,7 +408,10 @@ def humaninfo(self): return '{0.name}[{0.id}]'.format(self) def __repr__(self): - return '<{0}: {1}>'.format(type(self).__name__, self.humaninfo()) + return '<{0}: {1} {2} {3}>'.format( + type(self).__name__, self.humaninfo(), + self.argsrepr, self.kwargsrepr, + ) @property def tzlocal(self): diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index ac8f2ad50..b135ace1a 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -15,6 +15,7 @@ from celery.exceptions import InvalidTaskError from celery.utils.log import get_logger +from celery.utils.saferepr import saferepr from celery.utils.timeutils import timezone from .request import Request, create_request_cls @@ -40,7 +41,11 @@ def proto1_to_proto2(message, body): raise InvalidTaskError( 'Task keyword arguments must be a mapping', ) - body['headers'] = message.headers + body.update( + argsrepr=saferepr(args), + kwargsrepr=saferepr(kwargs), + headers=message.headers, + ) try: body['group'] = body['taskset'] except KeyError: @@ -95,7 +100,7 @@ def task_message_handler(message, body, ack, reject, callbacks, send_event( 'task-received', uuid=req.id, name=req.name, - args='', kwargs='', + args=req.argsrepr, kwargs=req.kwargsrepr, retries=req.request_dict.get('retries', 0), eta=req.eta and req.eta.isoformat(), expires=req.expires and req.expires.isoformat(), diff --git a/docs/internals/protocol.rst b/docs/internals/protocol.rst index 7cc334982..623d9b184 100644 --- a/docs/internals/protocol.rst +++ b/docs/internals/protocol.rst @@ -46,6 +46,8 @@ Definition 'expires'; iso8601 expires, 'retries': int retries, 'timelimit': (soft, hard), + 'argsrepr': str repr(args), + 'kwargsrepr': str repr(kwargs), } body = ( @@ -69,11 +71,15 @@ This example sends a task message using version 2 of the protocol: # chain: add(add(add(2, 2), 4), 8) == 2 + 2 + 4 + 8 task_id = uuid() + args = (2, 2) + kwargs = {} basic_publish( - message=json.dumps(([2, 2], {}, None), + message=json.dumps((args, kwargs, None), application_headers={ 'lang': 'py', 'task': 'proj.tasks.add', + 'argsrepr': repr(args), + 'kwargsrepr': repr(kwargs), } properties={ 'correlation_id': task_id, From 27dc6d021651230727d9b1fd9f419d826554f769 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 16 Oct 2015 14:51:51 -0700 Subject: [PATCH 0695/1103] flakes --- celery/tests/worker/test_request.py | 4 ++-- celery/worker/request.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index f52856254..ee2b881f8 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -336,8 +336,8 @@ def test_on_failure_WrokerLostError_rejects_with_requeue(self): req.task.reject_on_worker_lost = True req.delivery_info['redelivered'] = False req.on_failure(einfo) - req.on_reject.assert_called_with(req_logger, - req.connection_errors, True) + req.on_reject.assert_called_with( + req_logger, req.connection_errors, True) def test_tzlocal_is_cached(self): req = self.get_request(self.add.s(2, 2)) diff --git a/celery/worker/request.py b/celery/worker/request.py index bab64b54d..bfdfb7d3f 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -353,7 +353,8 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): ) # (acks_late) acknowledge after result stored. if self.task.acks_late: - reject_and_requeue = (self.task.reject_on_worker_lost and + reject_and_requeue = ( + self.task.reject_on_worker_lost and isinstance(exc, WorkerLostError) and self.delivery_info.get('redelivered', False) is False) if reject_and_requeue: From 907f2c7a110807e008b6b71e9ef093ebcaa50809 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 16 Oct 2015 16:55:00 -0700 Subject: [PATCH 0696/1103] Fixing tests --- celery/tests/utils/test_saferepr.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/celery/tests/utils/test_saferepr.py b/celery/tests/utils/test_saferepr.py index 4b04143e3..0ed91970d 100644 --- a/celery/tests/utils/test_saferepr.py +++ b/celery/tests/utils/test_saferepr.py @@ -11,11 +11,6 @@ from celery.tests.case import Case -EXPECTED_1 = """\ -{'rest': {'baz': 'The quick brown fox jumps over the lazy dog.', \ -'foo': 'The quick brown fox jumps...', ...}}\ -""" - D_NUMBERS = { b'integer': 1, b'float': 1.3, @@ -134,7 +129,8 @@ def test_text(self): self.assertEqual(saferepr(D_TEXT), repr(D_TEXT).replace("u'", "'")) def test_text_maxlen(self): - self.assertEqual(saferepr(D_D_TEXT, 100), EXPECTED_1) + self.assertEqual(saferepr(D_D_TEXT, 100), + from_old_repr(repr(D_D_TEXT)[:99] + "...', ...}}")) def test_same_as_repr(self): # Simple objects, small containers and classes that overwrite __repr__ From 1d4cbbcc921aa34975bde4b503b8df9c2f1816e0 Mon Sep 17 00:00:00 2001 From: Gerald Manipon Date: Mon, 19 Oct 2015 20:23:08 +0000 Subject: [PATCH 0697/1103] Add support for RabbitMQ priority queues Add configuration QUEUE_MAX_PRIORITY. Add coverage test. --- celery/app/amqp.py | 21 ++++++++++++++++++--- celery/app/defaults.py | 1 + celery/tests/app/test_amqp.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index e1aa3dcc8..343b4b72e 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -49,6 +49,7 @@ class Queues(dict): the occurrence of unknown queues in `wanted` will raise :exc:`KeyError`. :keyword ha_policy: Default HA policy for queues with none set. + :keyword max_priority: Default x-max-priority for queues with none set. """ @@ -57,13 +58,15 @@ class Queues(dict): _consume_from = None def __init__(self, queues=None, default_exchange=None, - create_missing=True, ha_policy=None, autoexchange=None): + create_missing=True, ha_policy=None, autoexchange=None, + max_priority=None): dict.__init__(self) self.aliases = WeakValueDictionary() self.default_exchange = default_exchange self.create_missing = create_missing self.ha_policy = ha_policy self.autoexchange = Exchange if autoexchange is None else autoexchange + self.max_priority = max_priority if isinstance(queues, (tuple, list)): queues = {q.name: q for q in queues} for name, q in items(queues or {}): @@ -109,6 +112,10 @@ def add(self, queue, **kwargs): if queue.queue_arguments is None: queue.queue_arguments = {} self._set_ha_policy(queue.queue_arguments) + if self.max_priority is not None: + if queue.queue_arguments is None: + queue.queue_arguments = {} + self._set_max_priority(queue.queue_arguments) self[queue.name] = queue return queue @@ -119,6 +126,8 @@ def add_compat(self, name, **options): options['routing_key'] = name if self.ha_policy is not None: self._set_ha_policy(options.setdefault('queue_arguments', {})) + if self.max_priority is not None: + self._set_max_priority(options.setdefault('queue_arguments', {})) q = self[name] = Queue.from_dict(name, **options) return q @@ -129,6 +138,10 @@ def _set_ha_policy(self, args): 'x-ha-policy-params': list(policy)}) args['x-ha-policy'] = policy + def _set_max_priority(self, args): + if 'x-max-priority' not in args and self.max_priority is not None: + return args.update({'x-max-priority': self.max_priority}) + def format(self, indent=0, indent_first=True): """Format routing table into string for log dumps.""" active = self.consume_from @@ -227,7 +240,7 @@ def send_task_message(self): return self._create_task_sender() def Queues(self, queues, create_missing=None, ha_policy=None, - autoexchange=None): + autoexchange=None, max_priority=None): """Create new :class:`Queues` instance, using queue defaults from the current configuration.""" conf = self.app.conf @@ -235,6 +248,8 @@ def Queues(self, queues, create_missing=None, ha_policy=None, create_missing = conf.CELERY_CREATE_MISSING_QUEUES if ha_policy is None: ha_policy = conf.CELERY_QUEUE_HA_POLICY + if max_priority is None: + max_priority = conf.CELERY_QUEUE_MAX_PRIORITY if not queues and conf.CELERY_DEFAULT_QUEUE: queues = (Queue(conf.CELERY_DEFAULT_QUEUE, exchange=self.default_exchange, @@ -243,7 +258,7 @@ def Queues(self, queues, create_missing=None, ha_policy=None, else autoexchange) return self.queues_cls( queues, self.default_exchange, create_missing, - ha_policy, autoexchange, + ha_policy, autoexchange, max_priority, ) def Router(self, queues=None, create_missing=None): diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 4f1558aaf..2b8753919 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -165,6 +165,7 @@ def __repr__(self): 'REDIRECT_STDOUTS_LEVEL': Option('WARNING'), 'QUEUES': Option(type='dict'), 'QUEUE_HA_POLICY': Option(None, type='string'), + 'QUEUE_MAX_PRIORITY': Option(None, type='int'), 'SECURITY_KEY': Option(type='string'), 'SECURITY_CERTIFICATE': Option(type='string'), 'SECURITY_CERT_STORE': Option(type='string'), diff --git a/celery/tests/app/test_amqp.py b/celery/tests/app/test_amqp.py index e4e8873a2..254c594cd 100644 --- a/celery/tests/app/test_amqp.py +++ b/celery/tests/app/test_amqp.py @@ -134,3 +134,38 @@ def test_alias(self): q = Queues() q.add(Queue('foo', alias='barfoo')) self.assertIs(q['barfoo'], q['foo']) + + def test_with_max_priority(self): + qs1 = Queues(max_priority=10) + qs1.add('foo') + self.assertEqual(qs1['foo'].queue_arguments, {'x-max-priority': 10}) + + q1 = Queue('xyx', queue_arguments={'x-max-priority': 3}) + qs1.add(q1) + self.assertEqual(qs1['xyx'].queue_arguments, { + 'x-max-priority': 3, + }) + + qs2 = Queues(ha_policy='all', max_priority=5) + qs2.add('bar') + self.assertEqual(qs2['bar'].queue_arguments, { + 'x-ha-policy': 'all', + 'x-max-priority': 5 + }) + + q2 = Queue('xyx2', queue_arguments={'x-max-priority': 2}) + qs2.add(q2) + self.assertEqual(qs2['xyx2'].queue_arguments, { + 'x-ha-policy': 'all', + 'x-max-priority': 2, + }) + + qs3 = Queues(max_priority=None) + qs3.add('foo2') + self.assertEqual(qs3['foo2'].queue_arguments, None) + + q3 = Queue('xyx3', queue_arguments={'x-max-priority': 7}) + qs3.add(q3) + self.assertEqual(qs3['xyx3'].queue_arguments, { + 'x-max-priority': 7, + }) From ceef8b9b32b880b9231cb32121ef4fc7e434bb2a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 Oct 2015 14:01:33 -0700 Subject: [PATCH 0698/1103] Fixes bugs in saferepr on Python3 --- celery/tests/utils/test_saferepr.py | 46 ++++++++++++++++--------- celery/utils/saferepr.py | 52 ++++++++++++++++++++++++----- celery/utils/text.py | 14 +++++--- 3 files changed, 85 insertions(+), 27 deletions(-) diff --git a/celery/tests/utils/test_saferepr.py b/celery/tests/utils/test_saferepr.py index 0ed91970d..7204d880d 100644 --- a/celery/tests/utils/test_saferepr.py +++ b/celery/tests/utils/test_saferepr.py @@ -5,7 +5,7 @@ from decimal import Decimal from pprint import pprint -from celery.five import items, long_t, values +from celery.five import items, long_t, text_t, values from celery.utils.saferepr import saferepr @@ -45,13 +45,28 @@ D_D_TEXT = {b'rest': D_TEXT} -RE_OLD_SET_REPR = re.compile(r'(?:frozen)?set\d?\(\[(.+?)\]\)') +RE_OLD_SET_REPR = re.compile(r'(?= maxlen: - return text[:maxlen].rsplit(' ', 1)[0] + suffix - return text + if maxlen and len(s) >= maxlen: + return s[:maxlen].rsplit(' ', 1)[0] + suffix + return s + + +def truncate_bytes(s, maxlen=128, suffix=b'...'): + if maxlen and len(s) >= maxlen: + return s[:maxlen].rsplit(b' ', 1)[0] + suffix + return s def pluralize(n, text, suffix='s'): From 0e0ef00e68482ee99158ee1e1110e3b3312d49db Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 Oct 2015 14:18:05 -0700 Subject: [PATCH 0699/1103] Python2.7isms --- celery/result.py | 2 +- celery/tests/backends/test_base.py | 4 ++-- celery/tests/case.py | 2 +- celery/tests/utils/test_saferepr.py | 2 +- docs/_ext/literals_to_xrefs.py | 4 ++-- extra/release/attribution.py | 8 ++++---- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/celery/result.py b/celery/result.py index b12de6857..4e3770164 100644 --- a/celery/result.py +++ b/celery/result.py @@ -671,7 +671,7 @@ def iter_native(self, timeout=None, interval=0.5, no_ack=True, if not results: return iter([]) return self.backend.get_many( - set(r.id for r in results), + {r.id for r in results}, timeout=timeout, interval=interval, no_ack=no_ack, on_message=on_message, ) diff --git a/celery/tests/backends/test_base.py b/celery/tests/backends/test_base.py index f1cde8984..c98d138b8 100644 --- a/celery/tests/backends/test_base.py +++ b/celery/tests/backends/test_base.py @@ -139,7 +139,7 @@ def set(self, key, value): def mget(self, keys): if self.mget_returns_dict: - return dict((key, self.get(key)) for key in keys) + return {key: self.get(key) for key in keys} else: return [self.get(k) for k in keys] @@ -273,7 +273,7 @@ def test_strip_prefix(self): def test_get_many(self): for is_dict in True, False: self.b.mget_returns_dict = is_dict - ids = dict((uuid(), i) for i in range(10)) + ids = {uuid(): i for i in range(10)} for id, i in items(ids): self.b.mark_as_done(id, i) it = self.b.get_many(list(ids)) diff --git a/celery/tests/case.py b/celery/tests/case.py index aedd3f4fc..89e95ad9a 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -721,7 +721,7 @@ def sys_platform(value): @contextmanager def reset_modules(*modules): - prev = dict((k, sys.modules.pop(k)) for k in modules if k in sys.modules) + prev = {k: sys.modules.pop(k) for k in modules if k in sys.modules} try: yield finally: diff --git a/celery/tests/utils/test_saferepr.py b/celery/tests/utils/test_saferepr.py index 7204d880d..95d437840 100644 --- a/celery/tests/utils/test_saferepr.py +++ b/celery/tests/utils/test_saferepr.py @@ -169,7 +169,7 @@ def test_same_as_repr(self): (1, 2), [3, 4], {5: 6}, tuple2((1, 2)), tuple3((1, 2)), tuple3(range(100)), [3, 4], list2([3, 4]), list3([3, 4]), list3(range(100)), - set({7}), set2({7}), set3({7}), + {7}, set2({7}), set3({7}), frozenset({8}), frozenset2({8}), frozenset3({8}), dict2({5: 6}), dict3({5: 6}), range(10, -11, -1) diff --git a/docs/_ext/literals_to_xrefs.py b/docs/_ext/literals_to_xrefs.py index 38dad0b74..debd8953b 100644 --- a/docs/_ext/literals_to_xrefs.py +++ b/docs/_ext/literals_to_xrefs.py @@ -146,8 +146,8 @@ def colorize(text='', opts=(), **kwargs): """ color_names = ('black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white') - foreground = dict([(color_names[x], '3%s' % x) for x in range(8)]) - background = dict([(color_names[x], '4%s' % x) for x in range(8)]) + foreground = {color_names[x]: '3%s' % x for x in range(8)} + background = {color_names[x]: '4%s' % x for x in range(8)} RESET = '0' opt_dict = {'bold': '1', diff --git a/extra/release/attribution.py b/extra/release/attribution.py index d48a46603..dcc70033b 100755 --- a/extra/release/attribution.py +++ b/extra/release/attribution.py @@ -23,11 +23,11 @@ def find_missing_authors(seen): with open("AUTHORS") as authors: known = [author(line) for line in authors.readlines()] - seen_authors = set(filter(proper_name, (t[0] for t in seen))) - known_authors = set(t[0] for t in known) + seen_authors = {t[0] for t in seen if proper_name(t[0])} + known_authors = {t[0] for t in known} # maybe later?: - # seen_emails = set(t[1] for t in seen) - # known_emails = set(t[1] for t in known) + # seen_emails = {t[1] for t in seen} + # known_emails = {t[1] for t in known} pprint(seen_authors - known_authors) From 0e6792ea2bbccfc22ed18149a817af919cefcf1f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 Oct 2015 15:36:40 -0700 Subject: [PATCH 0700/1103] Fixes tests --- celery/tests/utils/test_saferepr.py | 2 +- celery/utils/saferepr.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/tests/utils/test_saferepr.py b/celery/tests/utils/test_saferepr.py index 95d437840..a7e8348ef 100644 --- a/celery/tests/utils/test_saferepr.py +++ b/celery/tests/utils/test_saferepr.py @@ -15,7 +15,7 @@ b'integer': 1, b'float': 1.3, b'decimal': Decimal("1.3"), - b'long': long_t(1.3), + b'long': long_t(4), b'complex': complex(13.3), } D_INT_KEYS = {v: k for k, v in items(D_NUMBERS)} diff --git a/celery/utils/saferepr.py b/celery/utils/saferepr.py index 88e8ff157..57e6cb0b4 100644 --- a/celery/utils/saferepr.py +++ b/celery/utils/saferepr.py @@ -107,7 +107,7 @@ def _saferepr(o, maxlen=None, maxlevels=3, seen=None): if isinstance(token, _literal): val = token.value elif isinstance(token, _key): - val = repr(token.value).replace("u'", "'") + val = saferepr(token.value, maxlen, maxlevels) elif isinstance(token, _quoted): val = token.value if IS_PY3 and isinstance(val, bytes): From 7c763a0c51ce60517201b53e0e0d88fd38b01bcd Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 Oct 2015 15:44:53 -0700 Subject: [PATCH 0701/1103] Fixed make apicheck --- docs/internals/reference/celery.backends.couchdb.rst | 11 +++++++++++ docs/internals/reference/celery.backends.riak.rst | 11 +++++++++++ docs/internals/reference/celery.utils.abstract.rst | 11 +++++++++++ docs/internals/reference/celery.utils.saferepr.rst | 11 +++++++++++ docs/internals/reference/index.rst | 6 +++++- docs/reference/celery.bin.logtool.rst | 11 +++++++++++ docs/reference/index.rst | 1 + 7 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 docs/internals/reference/celery.backends.couchdb.rst create mode 100644 docs/internals/reference/celery.backends.riak.rst create mode 100644 docs/internals/reference/celery.utils.abstract.rst create mode 100644 docs/internals/reference/celery.utils.saferepr.rst create mode 100644 docs/reference/celery.bin.logtool.rst diff --git a/docs/internals/reference/celery.backends.couchdb.rst b/docs/internals/reference/celery.backends.couchdb.rst new file mode 100644 index 000000000..bd836abc4 --- /dev/null +++ b/docs/internals/reference/celery.backends.couchdb.rst @@ -0,0 +1,11 @@ +=========================================== + celery.backends.couchdb +=========================================== + +.. contents:: + :local: +.. currentmodule:: celery.backends.couchdb + +.. automodule:: celery.backends.couchdb + :members: + :undoc-members: diff --git a/docs/internals/reference/celery.backends.riak.rst b/docs/internals/reference/celery.backends.riak.rst new file mode 100644 index 000000000..edbdb1c2d --- /dev/null +++ b/docs/internals/reference/celery.backends.riak.rst @@ -0,0 +1,11 @@ +=========================================== + celery.backends.riak +=========================================== + +.. contents:: + :local: +.. currentmodule:: celery.backends.riak + +.. automodule:: celery.backends.riak + :members: + :undoc-members: diff --git a/docs/internals/reference/celery.utils.abstract.rst b/docs/internals/reference/celery.utils.abstract.rst new file mode 100644 index 000000000..70ec49749 --- /dev/null +++ b/docs/internals/reference/celery.utils.abstract.rst @@ -0,0 +1,11 @@ +=========================================== + celery.utils.abstract +=========================================== + +.. contents:: + :local: +.. currentmodule:: celery.utils.abstract + +.. automodule:: celery.utils.abstract + :members: + :undoc-members: diff --git a/docs/internals/reference/celery.utils.saferepr.rst b/docs/internals/reference/celery.utils.saferepr.rst new file mode 100644 index 000000000..e01790857 --- /dev/null +++ b/docs/internals/reference/celery.utils.saferepr.rst @@ -0,0 +1,11 @@ +=========================================== + celery.utils.saferepr +=========================================== + +.. contents:: + :local: +.. currentmodule:: celery.utils.saferepr + +.. automodule:: celery.utils.saferepr + :members: + :undoc-members: diff --git a/docs/internals/reference/index.rst b/docs/internals/reference/index.rst index 31b606139..16897b9d0 100644 --- a/docs/internals/reference/index.rst +++ b/docs/internals/reference/index.rst @@ -26,10 +26,12 @@ celery.backends.base celery.backends.rpc celery.backends.database - celery.backends.cache celery.backends.amqp + celery.backends.cache + celery.backends.couchdb celery.backends.mongodb celery.backends.redis + celery.backends.riak celery.backends.cassandra celery.backends.couchbase celery.app.trace @@ -46,12 +48,14 @@ celery.backends.database.models celery.backends.database.session celery.utils + celery.utils.abstract celery.utils.functional celery.utils.objects celery.utils.term celery.utils.timeutils celery.utils.iso8601 celery.utils.compat + celery.utils.saferepr celery.utils.serialization celery.utils.sysinfo celery.utils.threads diff --git a/docs/reference/celery.bin.logtool.rst b/docs/reference/celery.bin.logtool.rst new file mode 100644 index 000000000..3242835ce --- /dev/null +++ b/docs/reference/celery.bin.logtool.rst @@ -0,0 +1,11 @@ +===================================================== + celery.bin.logtool +===================================================== + +.. contents:: + :local: +.. currentmodule:: celery.bin.logtool + +.. automodule:: celery.bin.logtool + :members: + :undoc-members: diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 118f220c4..2f104e89c 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -54,6 +54,7 @@ celery.bin.worker celery.bin.beat celery.bin.events + celery.bin.logtool celery.bin.amqp celery.bin.multi celery.bin.graph From dcc464351457c82ad2ff7de529cc36a5c463d5b6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 19 Oct 2015 15:45:47 -0700 Subject: [PATCH 0702/1103] Version 3.2 no longer exists --- docs/.templates/page.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/.templates/page.html b/docs/.templates/page.html index 7562de304..89292a458 100644 --- a/docs/.templates/page.html +++ b/docs/.templates/page.html @@ -2,14 +2,14 @@ {% block body %}

- {% if version == "3.2" or version == "4.0" %} + {% if version == "4.0" %}

This document is for Celery's development version, which can be significantly different from previous releases. Get old docs here: 3.1.

- {% else %} + {% else %}

This document describes the current stable version of Celery ({{ version }}). For development docs, go here. From 0b901dd32cb17f7ccba05fbb3697922b39c9e538 Mon Sep 17 00:00:00 2001 From: Gerald Manipon Date: Tue, 20 Oct 2015 08:57:14 -0700 Subject: [PATCH 0703/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 08eb2ccf2..68664e8f2 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -191,4 +191,5 @@ Frantisek Holop, 2015/05/21 Feanil Patel, 2015/05/21 Jocelyn Delalande, 2015/06/03 Juan Rossi, 2015/08/10 -Piotr Maślanka, 2015/08/24 \ No newline at end of file +Piotr Maślanka, 2015/08/24 +Gerald Manipon, 2015/10/19 From 45c9825492d4195e6f77cfaa2c37fc9b3a158262 Mon Sep 17 00:00:00 2001 From: Michael Permana Date: Tue, 20 Oct 2015 10:28:32 -0700 Subject: [PATCH 0704/1103] use reject+requeue=False when redelivered is not known, so that a dead letter queue can be used --- celery/worker/request.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/celery/worker/request.py b/celery/worker/request.py index bfdfb7d3f..4014d2cc7 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -353,12 +353,11 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): ) # (acks_late) acknowledge after result stored. if self.task.acks_late: - reject_and_requeue = ( - self.task.reject_on_worker_lost and - isinstance(exc, WorkerLostError) and - self.delivery_info.get('redelivered', False) is False) - if reject_and_requeue: - self.reject(requeue=True) + requeue = self.delivery_info.get('redelivered', None) is False + reject = (self.task.reject_on_worker_lost and + isinstance(exc, WorkerLostError)) + if reject: + self.reject(requeue=requeue) else: self.acknowledge() From 458bbb09acbdd10c5e3fc27b42d857935cecb33d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 Oct 2015 13:04:32 -0700 Subject: [PATCH 0705/1103] Cosmetics for #2751 --- celery/backends/redis.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 9d473c69e..18db9a113 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -130,10 +130,6 @@ def _params_from_url(self, url, defaults): db = db.strip('/') if isinstance(db, string_t) else db connparams['db'] = int(db) - for key in ['socket_timeout', 'socket_connect_timeout']: - if key in query: - query[key] = float(query[key]) - # Query parameters override other parameters connparams.update(query) return connparams @@ -253,6 +249,16 @@ def _new_chord_return(self, task, state, result, propagate=None): callback.id, exc=ChordError('Join error: {0!r}'.format(exc)), ) + def _create_client(self, socket_timeout=None, socket_connect_timeout=None, + **params): + return self.redis.Redis( + connection_pool=self.ConnectionPool( + socket_timeout=socket_timeout and float(socket_timeout), + socket_connect_timeout=socket_connect_timeout and float( + socket_connect_timeout), + **params), + ) + @property def ConnectionPool(self): if self._ConnectionPool is None: @@ -261,9 +267,7 @@ def ConnectionPool(self): @cached_property def client(self): - return self.redis.Redis( - connection_pool=self.ConnectionPool(**self.connparams), - ) + return self._create_client(**self.connparams) def __reduce__(self, args=(), kwargs={}): return super(RedisBackend, self).__reduce__( From 6c80ba7b48a23f3743450101f30eea65b942d167 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 Oct 2015 13:38:53 -0700 Subject: [PATCH 0706/1103] ContextMock should return self --- celery/tests/case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/tests/case.py b/celery/tests/case.py index 89e95ad9a..6446fd98c 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -149,7 +149,7 @@ class _ContextMock(Mock): in the class, not just the instance.""" def __enter__(self): - pass + return self def __exit__(self, *exc_info): pass From 5a2aab7a1498cbdb09a0344bed7f75812a32412a Mon Sep 17 00:00:00 2001 From: gmanipon Date: Tue, 20 Oct 2015 03:49:57 +0000 Subject: [PATCH 0707/1103] Set priority from message properties if not in delivery_info Add contributor. --- celery/worker/request.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/worker/request.py b/celery/worker/request.py index bfdfb7d3f..55672df58 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -152,7 +152,8 @@ def __init__(self, message, on_ack=noop, 'delivery_info': { 'exchange': delivery_info.get('exchange'), 'routing_key': delivery_info.get('routing_key'), - 'priority': delivery_info.get('priority'), + 'priority': delivery_info.get('priority', + properties.get('priority')), 'redelivered': delivery_info.get('redelivered'), } From a7f806c90f697af3974a28dfc4953382d068cb99 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 Oct 2015 14:01:24 -0700 Subject: [PATCH 0708/1103] kombu 4.0 will no longer send priority message field in delivery_info --- celery/worker/request.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/celery/worker/request.py b/celery/worker/request.py index 55672df58..0187fbc7a 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -152,8 +152,7 @@ def __init__(self, message, on_ack=noop, 'delivery_info': { 'exchange': delivery_info.get('exchange'), 'routing_key': delivery_info.get('routing_key'), - 'priority': delivery_info.get('priority', - properties.get('priority')), + 'priority': properties.get('priority'), 'redelivered': delivery_info.get('redelivered'), } From 55678442dec5792a8ff6e988858f1ec4a26a0885 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 20 Oct 2015 16:12:27 -0700 Subject: [PATCH 0709/1103] PyPy: Try to handle KeyError when setting key. Closes #2862 --- celery/beat.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/celery/beat.py b/celery/beat.py index 045b65a72..9dbd4386f 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -401,22 +401,31 @@ def _remove_db(self): with platforms.ignore_errno(errno.ENOENT): os.remove(self.schedule_filename + suffix) + def _open_schedule(self): + return self.persistence.open(self.schedule_filename, writeback=True) + + def _destroy_open_corrupted_schedule(self, exc): + error('Removing corrupted schedule file %r: %r', + self.schedule_filename, exc, exc_info=True) + self._remove_db() + return self._open_schedule() + def setup_schedule(self): try: - self._store = self.persistence.open(self.schedule_filename, - writeback=True) + self._store = self._open_schedule() except Exception as exc: - error('Removing corrupted schedule file %r: %r', - self.schedule_filename, exc, exc_info=True) - self._remove_db() - self._store = self.persistence.open(self.schedule_filename, - writeback=True) - else: + self._store = self._destroy_open_corrupted_schedule(exc) + + for _ in (1, 2): try: self._store['entries'] except KeyError: # new schedule db - self._store['entries'] = {} + try: + self._store['entries'] = {} + except KeyError as exc: + self._store = self._destroy_open_corrupted_schedule(exc) + continue else: if '__version__' not in self._store: warning('DB Reset: Account for new __version__ field') @@ -427,6 +436,7 @@ def setup_schedule(self): elif 'utc_enabled' not in self._store: warning('DB Reset: Account for new utc_enabled field') self._store.clear() # remove schedule at 3.0.9 upgrade + break tz = self.app.conf.CELERY_TIMEZONE stored_tz = self._store.get('tz') From da74c90f7fa4a9b3a9a25a84032a477abdda2082 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 22 Oct 2015 02:46:42 +0300 Subject: [PATCH 0710/1103] Remove duplicate line "sudo: false" in .travis.yml sudo: false has already been added in 4fd22bb88aeef1385ce9d057f46cedfac07b569a. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 325607767..700106f3e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,4 +24,3 @@ notifications: - "chat.freenode.net#celery" on_success: change on_failure: change -sudo: false From ebfe73a646e5fba81b43a72582843069feba1b36 Mon Sep 17 00:00:00 2001 From: Michael Permana Date: Thu, 22 Oct 2015 11:27:56 -0700 Subject: [PATCH 0711/1103] Test case for acks_late with reject and requeue, when there is no redelivered information --- celery/tests/worker/test_request.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index ee2b881f8..9703f6cab 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -325,7 +325,7 @@ def test_on_failure_Reject_rejects_with_requeue(self): req_logger, req.connection_errors, True, ) - def test_on_failure_WrokerLostError_rejects_with_requeue(self): + def test_on_failure_WorkerLostError_rejects_with_requeue(self): einfo = None try: raise WorkerLostError() @@ -339,6 +339,20 @@ def test_on_failure_WrokerLostError_rejects_with_requeue(self): req.on_reject.assert_called_with( req_logger, req.connection_errors, True) + def test_on_failure_WorkerLostError_redelivered_None(self): + einfo = None + try: + raise WorkerLostError() + except: + einfo = ExceptionInfo(internal=True) + req = self.get_request(self.add.s(2, 2)) + req.task.acks_late = True + req.task.reject_on_worker_lost = True + req.delivery_info['redelivered'] = None + req.on_failure(einfo) + req.on_reject.assert_called_with( + req_logger, req.connection_errors, False) + def test_tzlocal_is_cached(self): req = self.get_request(self.add.s(2, 2)) req._tzlocal = 'foo' From 0e4ac4ca80be2db688030b04c0b4655930efa1d3 Mon Sep 17 00:00:00 2001 From: Juan Rossi Date: Thu, 22 Oct 2015 18:55:33 -0300 Subject: [PATCH 0712/1103] Fixed Security docs typo --- docs/userguide/security.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/security.rst b/docs/userguide/security.rst index 4ccdb9d8c..f000294bb 100644 --- a/docs/userguide/security.rst +++ b/docs/userguide/security.rst @@ -200,7 +200,7 @@ Logs are usually the first place to look for evidence of security breaches, but they are useless if they can be tampered with. A good solution is to set up centralized logging with a dedicated logging -server. Acess to it should be restricted. +server. Access to it should be restricted. In addition to having all of the logs in a single place, if configured correctly, it can make it harder for intruders to tamper with your logs. From cd48cd34ae764c6a7e22ead1d51e2e154dd0e194 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 Oct 2015 16:14:06 -0700 Subject: [PATCH 0713/1103] Report chord errors when task process terminated --- celery/app/trace.py | 5 ----- celery/backends/base.py | 30 ++++++++++++++++------------- celery/backends/redis.py | 3 +-- celery/tests/backends/test_base.py | 16 ++++++++++----- celery/tests/backends/test_cache.py | 4 ++-- celery/tests/backends/test_redis.py | 2 +- celery/tests/tasks/test_trace.py | 15 +++++++++++++-- celery/worker/request.py | 23 ++++++++++++++++++---- 8 files changed, 64 insertions(+), 34 deletions(-) diff --git a/celery/app/trace.py b/celery/app/trace.py index 5b588b881..6137fcd7b 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -291,7 +291,6 @@ def build_tracer(name, task, loader=None, hostname=None, store_errors=True, pop_request = request_stack.pop push_task = _task_stack.push pop_task = _task_stack.pop - on_chord_part_return = backend.on_chord_part_return _does_info = logger.isEnabledFor(logging.INFO) prerun_receivers = signals.task_prerun.receivers @@ -368,8 +367,6 @@ def trace_task(uuid, args, kwargs, request=None): ) except Exception as exc: I, R, state, retval = on_error(task_request, exc, uuid) - if task_request.chord: - on_chord_part_return(task, state, exc) except BaseException as exc: raise else: @@ -404,8 +401,6 @@ def trace_task(uuid, args, kwargs, request=None): except EncodeError as exc: I, R, state, retval = on_error(task_request, exc, uuid) else: - if task_request.chord: - on_chord_part_return(task, state, retval) if task_on_success: task_on_success(retval, uuid, args, kwargs) if success_receivers: diff --git a/celery/backends/base.py b/celery/backends/base.py index a8975be25..bb4b5bb59 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -112,15 +112,19 @@ def mark_as_started(self, task_id, **meta): """Mark a task as started""" return self.store_result(task_id, meta, status=states.STARTED) - def mark_as_done(self, task_id, result, request=None): + def mark_as_done(self, task_id, result, request=None, state=states.SUCCESS): """Mark task as successfully executed.""" - return self.store_result(task_id, result, - status=states.SUCCESS, request=request) + self.store_result(task_id, result, status=state, request=request) + if request and request.chord: + self.on_chord_part_return(request, state) - def mark_as_failure(self, task_id, exc, traceback=None, request=None): + def mark_as_failure(self, task_id, exc, + traceback=None, request=None, state=states.FAILURE): """Mark task as executed with failure. Stores the exception.""" - return self.store_result(task_id, exc, status=states.FAILURE, - traceback=traceback, request=request) + self.store_result(task_id, exc, status=state, + traceback=traceback, request=request) + if request and request.chord: + self.on_chord_part_return(request, state, exc) def chord_error_from_stack(self, callback, exc=None): from celery import group @@ -346,7 +350,7 @@ def on_task_call(self, producer, task_id): def add_to_chord(self, chord_id, result): raise NotImplementedError('Backend does not support add_to_chord') - def on_chord_part_return(self, task, state, result, propagate=False): + def on_chord_part_return(self, request, state, result, propagate=False): pass def fallback_chord_unlock(self, group_id, body, result=None, @@ -540,20 +544,20 @@ def _apply_chord_incr(self, header, partial_args, group_id, body, return header(*partial_args, task_id=group_id, **fixed_options or {}) - def on_chord_part_return(self, task, state, result, propagate=None): + def on_chord_part_return(self, request, state, result, propagate=None): if not self.implements_incr: return app = self.app if propagate is None: propagate = app.conf.CELERY_CHORD_PROPAGATES - gid = task.request.group + gid = request.group if not gid: return key = self.get_key_for_chord(gid) try: - deps = GroupResult.restore(gid, backend=task.backend) + deps = GroupResult.restore(gid, backend=self) except Exception as exc: - callback = maybe_signature(task.request.chord, app=app) + callback = maybe_signature(request.chord, app=app) logger.error('Chord %r raised: %r', gid, exc, exc_info=1) return self.chord_error_from_stack( callback, @@ -563,7 +567,7 @@ def on_chord_part_return(self, task, state, result, propagate=None): try: raise ValueError(gid) except ValueError as exc: - callback = maybe_signature(task.request.chord, app=app) + callback = maybe_signature(request.chord, app=app) logger.error('Chord callback %r raised: %r', gid, exc, exc_info=1) return self.chord_error_from_stack( @@ -576,7 +580,7 @@ def on_chord_part_return(self, task, state, result, propagate=None): logger.warning('Chord counter incremented too many times for %r', gid) elif val == size: - callback = maybe_signature(task.request.chord, app=app) + callback = maybe_signature(request.chord, app=app) j = deps.join_native if deps.supports_native_join else deps.join try: with allow_join_result(): diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 18db9a113..8afc33aaf 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -196,9 +196,8 @@ def _new_chord_apply(self, header, partial_args, group_id, body, options['task_id'] = group_id return header(*partial_args, **options or {}) - def _new_chord_return(self, task, state, result, propagate=None): + def _new_chord_return(self, request, state, result, propagate=None): app = self.app - request = task.request tid, gid = request.id, request.group if not gid or not tid: return diff --git a/celery/tests/backends/test_base.py b/celery/tests/backends/test_base.py index c98d138b8..0728ae890 100644 --- a/celery/tests/backends/test_base.py +++ b/celery/tests/backends/test_base.py @@ -298,7 +298,9 @@ def test_chord_part_return_no_gid(self): self.b.get_key_for_chord.side_effect = AssertionError( 'should not get here', ) - self.assertIsNone(self.b.on_chord_part_return(task, state, result)) + self.assertIsNone( + self.b.on_chord_part_return(task.request, state, result), + ) @contextmanager def _chord_part_context(self, b): @@ -326,14 +328,18 @@ def callback(result): def test_chord_part_return_propagate_set(self): with self._chord_part_context(self.b) as (task, deps, _): - self.b.on_chord_part_return(task, 'SUCCESS', 10, propagate=True) + self.b.on_chord_part_return( + task.request, 'SUCCESS', 10, propagate=True, + ) self.assertFalse(self.b.expire.called) deps.delete.assert_called_with() deps.join_native.assert_called_with(propagate=True, timeout=3.0) def test_chord_part_return_propagate_default(self): with self._chord_part_context(self.b) as (task, deps, _): - self.b.on_chord_part_return(task, 'SUCCESS', 10, propagate=None) + self.b.on_chord_part_return( + task.request, 'SUCCESS', 10, propagate=None, + ) self.assertFalse(self.b.expire.called) deps.delete.assert_called_with() deps.join_native.assert_called_with( @@ -345,7 +351,7 @@ def test_chord_part_return_join_raises_internal(self): with self._chord_part_context(self.b) as (task, deps, callback): deps._failed_join_report = lambda: iter([]) deps.join_native.side_effect = KeyError('foo') - self.b.on_chord_part_return(task, 'SUCCESS', 10) + self.b.on_chord_part_return(task.request, 'SUCCESS', 10) self.assertTrue(self.b.fail_from_current_stack.called) args = self.b.fail_from_current_stack.call_args exc = args[1]['exc'] @@ -359,7 +365,7 @@ def test_chord_part_return_join_raises_task(self): self.app.AsyncResult('culprit'), ]) deps.join_native.side_effect = KeyError('foo') - b.on_chord_part_return(task, 'SUCCESS', 10) + b.on_chord_part_return(task.request, 'SUCCESS', 10) self.assertTrue(b.fail_from_current_stack.called) args = b.fail_from_current_stack.call_args exc = args[1]['exc'] diff --git a/celery/tests/backends/test_cache.py b/celery/tests/backends/test_cache.py index f741b852e..4121df84d 100644 --- a/celery/tests/backends/test_cache.py +++ b/celery/tests/backends/test_cache.py @@ -87,10 +87,10 @@ def test_on_chord_part_return(self, restore): tb.apply_chord(group(app=self.app), (), gid, {}, result=res) self.assertFalse(deps.join_native.called) - tb.on_chord_part_return(task, 'SUCCESS', 10) + tb.on_chord_part_return(task.request, 'SUCCESS', 10) self.assertFalse(deps.join_native.called) - tb.on_chord_part_return(task, 'SUCCESS', 10) + tb.on_chord_part_return(task.request, 'SUCCESS', 10) deps.join_native.assert_called_with(propagate=True, timeout=3.0) deps.delete.assert_called_with() diff --git a/celery/tests/backends/test_redis.py b/celery/tests/backends/test_redis.py index fd30a4727..ac54bb75f 100644 --- a/celery/tests/backends/test_redis.py +++ b/celery/tests/backends/test_redis.py @@ -259,7 +259,7 @@ def create_task(): tasks = [create_task() for i in range(10)] for i in range(10): - b.on_chord_part_return(tasks[i], states.SUCCESS, i) + b.on_chord_part_return(tasks[i].request, states.SUCCESS, i) self.assertTrue(b.client.rpush.call_count) b.client.rpush.reset_mock() self.assertTrue(b.client.lrange.call_count) diff --git a/celery/tests/tasks/test_trace.py b/celery/tests/tasks/test_trace.py index 0714acc2e..7e1fe33b4 100644 --- a/celery/tests/tasks/test_trace.py +++ b/celery/tests/tasks/test_trace.py @@ -103,8 +103,19 @@ def add(x, y): return x + y add.backend = Mock() - self.trace(add, (2, 2), {}, request={'chord': uuid()}) - add.backend.on_chord_part_return.assert_called_with(add, 'SUCCESS', 4) + class TestRequest(object): + + def __init__(self, request): + self.request = request + + def __eq__(self, other): + return self.request['chord'] == other['chord'] + + request = {'chord': uuid()} + self.trace(add, (2, 2), {}, request=request) + add.backend.on_chord_part_return.assert_called_with( + TestRequest(request), 'SUCCESS', 4, + ) def test_when_backend_cleanup_raises(self): diff --git a/celery/worker/request.py b/celery/worker/request.py index 0187fbc7a..a340a5617 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -211,7 +211,7 @@ def execute(self, loglevel=None, logfile=None): self.acknowledge() request = self.request_dict - args, kwargs, embed = self.message.payload + args, kwargs, embed = self._payload request.update({'loglevel': loglevel, 'logfile': logfile, 'hostname': self.hostname, 'is_eager': False, 'args': args, 'kwargs': kwargs}, **embed or {}) @@ -348,9 +348,7 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): 'terminated', True, string(exc), False) send_failed_event = False # already sent revoked event elif isinstance(exc, WorkerLostError) or not return_ok: - self.task.backend.mark_as_failure( - self.id, exc, request=self, - ) + self.task.backend.mark_as_failure(self.id, exc, request=self) # (acks_late) acknowledge after result stored. if self.task.acks_late: reject_and_requeue = ( @@ -453,6 +451,23 @@ def correlation_id(self): # used similarly to reply_to return self.request_dict['correlation_id'] + @cached_property + def _payload(self): + return self.message.payload + + @cached_property + def chord(self): + # used by backend.on_chord_part_return when failures reported + # by parent process + _, _, embed = self._payload + return embed['chord'] + + @cached_property + def group(self): + # used by backend.on_chord_part_return when failures reported + # by parent process + return self.request_dict['group'] + def create_request_cls(base, task, pool, hostname, eventer, ref=ref, revoked_tasks=revoked_tasks, From 3fff58c174d33f85873b92f194484ae3ca214141 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 Oct 2015 16:47:58 -0700 Subject: [PATCH 0714/1103] Last commit forgot changes --- celery/app/trace.py | 13 ++++----- celery/backends/base.py | 41 ++++++++++++++++++----------- celery/tests/tasks/test_trace.py | 17 +++++------- celery/tests/worker/test_request.py | 2 +- celery/worker/request.py | 28 +++++++++++--------- 5 files changed, 53 insertions(+), 48 deletions(-) diff --git a/celery/app/trace.py b/celery/app/trace.py index 6137fcd7b..393aeb461 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -187,10 +187,9 @@ def handle_failure(self, task, req, store_errors=True): einfo = ExceptionInfo() einfo.exception = get_pickleable_exception(einfo.exception) einfo.type = get_pickleable_etype(einfo.type) - if store_errors: - task.backend.mark_as_failure( - req.id, exc, einfo.traceback, request=req, - ) + task.backend.mark_as_failure( + req.id, exc, einfo.traceback, req, store_errors, + ) task.on_failure(exc, req.id, req.args, req.kwargs, einfo) signals.task_failure.send(sender=task, task_id=req.id, exception=exc, args=req.args, @@ -282,6 +281,7 @@ def build_tracer(name, task, loader=None, hostname=None, store_errors=True, task_after_return = task.after_return store_result = backend.store_result + mark_as_done = backend.mark_as_done backend_cleanup = backend.process_cleanup pid = os.getpid() @@ -394,10 +394,7 @@ def trace_task(uuid, args, kwargs, request=None): group(sigs).apply_async((retval,)) else: signature(callbacks[0], app=app).delay(retval) - if publish_result: - store_result( - uuid, retval, SUCCESS, request=task_request, - ) + mark_as_done(uuid, retval, task_request, publish_result) except EncodeError as exc: I, R, state, retval = on_error(task_request, exc, uuid) else: diff --git a/celery/backends/base.py b/celery/backends/base.py index bb4b5bb59..4b7ae24d4 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -112,20 +112,40 @@ def mark_as_started(self, task_id, **meta): """Mark a task as started""" return self.store_result(task_id, meta, status=states.STARTED) - def mark_as_done(self, task_id, result, request=None, state=states.SUCCESS): + def mark_as_done(self, task_id, result, + request=None, store_result=True, state=states.SUCCESS): """Mark task as successfully executed.""" - self.store_result(task_id, result, status=state, request=request) + if store_result: + self.store_result(task_id, result, status=state, request=request) if request and request.chord: self.on_chord_part_return(request, state) def mark_as_failure(self, task_id, exc, - traceback=None, request=None, state=states.FAILURE): + traceback=None, request=None, store_result=True, + state=states.FAILURE): """Mark task as executed with failure. Stores the exception.""" - self.store_result(task_id, exc, status=state, - traceback=traceback, request=request) + if store_result: + self.store_result(task_id, exc, status=state, + traceback=traceback, request=request) if request and request.chord: self.on_chord_part_return(request, state, exc) + def mark_as_revoked(self, task_id, reason='', + request=None, store_result=True, state=states.REVOKED): + exc = TaskRevokedError(reason) + if store_result: + self.store_result(task_id, exc, + status=state, traceback=None, request=request) + if request and request.chord: + self.on_chord_part_return(request, state, exc) + + def mark_as_retry(self, task_id, exc, traceback=None, + request=None, store_result=True, state=states.RETRY): + """Mark task as being retries. Stores the current + exception (if any).""" + return self.store_result(task_id, exc, status=state, + traceback=traceback, request=request) + def chord_error_from_stack(self, callback, exc=None): from celery import group app = self.app @@ -151,17 +171,6 @@ def fail_from_current_stack(self, task_id, exc=None): finally: del(tb) - def mark_as_retry(self, task_id, exc, traceback=None, request=None): - """Mark task as being retries. Stores the current - exception (if any).""" - return self.store_result(task_id, exc, status=states.RETRY, - traceback=traceback, request=request) - - def mark_as_revoked(self, task_id, reason='', request=None): - return self.store_result(task_id, TaskRevokedError(reason), - status=states.REVOKED, traceback=None, - request=request) - def prepare_exception(self, exc, serializer=None): """Prepare exception for serialization.""" serializer = self.serializer if serializer is None else serializer diff --git a/celery/tests/tasks/test_trace.py b/celery/tests/tasks/test_trace.py index 7e1fe33b4..037acc4d6 100644 --- a/celery/tests/tasks/test_trace.py +++ b/celery/tests/tasks/test_trace.py @@ -103,19 +103,14 @@ def add(x, y): return x + y add.backend = Mock() - class TestRequest(object): - - def __init__(self, request): - self.request = request - - def __eq__(self, other): - return self.request['chord'] == other['chord'] - request = {'chord': uuid()} self.trace(add, (2, 2), {}, request=request) - add.backend.on_chord_part_return.assert_called_with( - TestRequest(request), 'SUCCESS', 4, - ) + self.assertTrue(add.backend.mark_as_done.called) + args, kwargs = add.backend.mark_as_done.call_args + self.assertEqual(args[0], 'id-1') + self.assertEqual(args[1], 4) + self.assertEqual(args[2].chord, request['chord']) + self.assertFalse(args[3]) def test_when_backend_cleanup_raises(self): diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index ee2b881f8..25505bafd 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -146,7 +146,7 @@ def test_process_cleanup_fails(self, _logger): tid = uuid() ret = jail(self.app, tid, self.mytask.name, [2], {}) self.assertEqual(ret, 4) - self.assertTrue(self.mytask.backend.store_result.called) + self.assertTrue(self.mytask.backend.mark_as_done.called) self.assertIn('Process cleanup failed', _logger.error.call_args[0][0]) def test_process_cleanup_BaseException(self): diff --git a/celery/worker/request.py b/celery/worker/request.py index a340a5617..fdeec9924 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -27,6 +27,7 @@ ) from celery.five import string from celery.platforms import signals as _signals +from celery.utils import cached_property from celery.utils.functional import noop from celery.utils.log import get_logger from celery.utils.timeutils import maybe_iso8601, timezone, maybe_make_aware @@ -245,8 +246,9 @@ def _announce_revoked(self, reason, terminated, signum, expired): task_ready(self) self.send_event('task-revoked', terminated=terminated, signum=signum, expired=expired) - if self.store_errors: - self.task.backend.mark_as_revoked(self.id, reason, request=self) + self.task.backend.mark_as_revoked( + self.id, reason, request=self, store_result=self.store_errors, + ) self.acknowledge() self._already_revoked = True send_revoked(self.task, request=self, @@ -296,8 +298,9 @@ def on_timeout(self, soft, timeout): timeout, self.name, self.id) exc = TimeLimitExceeded(timeout) - if self.store_errors: - self.task.backend.mark_as_failure(self.id, exc, request=self) + self.task.backend.mark_as_failure( + self.id, exc, request=self, store_result=self.store_errors, + ) if self.task.acks_late: self.acknowledge() @@ -342,13 +345,14 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): # These are special cases where the process would not have had # time to write the result. - if self.store_errors: - if isinstance(exc, Terminated): - self._announce_revoked( - 'terminated', True, string(exc), False) - send_failed_event = False # already sent revoked event - elif isinstance(exc, WorkerLostError) or not return_ok: - self.task.backend.mark_as_failure(self.id, exc, request=self) + if isinstance(exc, Terminated): + self._announce_revoked( + 'terminated', True, string(exc), False) + send_failed_event = False # already sent revoked event + elif isinstance(exc, WorkerLostError) or not return_ok: + self.task.backend.mark_as_failure( + self.id, exc, request=self, store_result=self.store_errors, + ) # (acks_late) acknowledge after result stored. if self.task.acks_late: reject_and_requeue = ( @@ -460,7 +464,7 @@ def chord(self): # used by backend.on_chord_part_return when failures reported # by parent process _, _, embed = self._payload - return embed['chord'] + return embed.get('chord') @cached_property def group(self): From 516bc98f22417f34a1ade4fbf394d2c12be11ca7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 Oct 2015 17:03:46 -0700 Subject: [PATCH 0715/1103] Remove extra/centos init scripts: Use generic-init.d Cloes #1895 --- extra/centos/celerybeat | 239 --------------------------- extra/centos/celerybeat.sysconfig | 15 -- extra/centos/celeryd | 266 ------------------------------ extra/centos/celeryd.sysconfig | 27 --- extra/centos/test_celerybeat.sh | 6 - extra/centos/test_celeryd.sh | 6 - extra/centos/test_service.sh | 43 ----- 7 files changed, 602 deletions(-) delete mode 100644 extra/centos/celerybeat delete mode 100644 extra/centos/celerybeat.sysconfig delete mode 100644 extra/centos/celeryd delete mode 100644 extra/centos/celeryd.sysconfig delete mode 100755 extra/centos/test_celerybeat.sh delete mode 100755 extra/centos/test_celeryd.sh delete mode 100755 extra/centos/test_service.sh diff --git a/extra/centos/celerybeat b/extra/centos/celerybeat deleted file mode 100644 index b51ab0762..000000000 --- a/extra/centos/celerybeat +++ /dev/null @@ -1,239 +0,0 @@ -#!/bin/sh -# ============================================ -# celerybeat - Starts the Celery periodic task scheduler. -# ============================================ -# -# :Usage: /etc/init.d/celerybeat {start|stop|restart|status} -# :Configuration file: /etc/sysconfig/celerybeat -# -# See http://docs.celeryproject.org/en/latest/tutorials/daemonizing.html - -### BEGIN INIT INFO -# Provides: celerybeat -# Required-Start: $network $local_fs $remote_fs -# Required-Stop: $network $local_fs $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: celery task worker daemon -### END INIT INFO -# -# -# To implement separate init scripts, do NOT copy this script. Instead, -# symlink it. I.e., if my new application, "little-worker" needs an init, I -# should just use: -# -# ln -s /etc/init.d/celerybeat /etc/init.d/little-worker -# -# You can then configure this by manipulating /etc/sysconfig/little-worker. -# -# Setting `prog` here allows you to symlink this init script, making it easy -# to run multiple processes on the system. - -# If we're invoked via SysV-style runlevel scripts we need to follow the -# link from rcX.d before working out the script name. -if [[ `dirname $0` == /etc/rc*.d ]]; then - target="$(readlink $0)" -else - target=$0 -fi - -prog="$(basename $target)" - -# Source the centos service helper functions -source /etc/init.d/functions -# NOTE: "set -e" does not work with the above functions, -# which use non-zero return codes as non-error return conditions - -# some commands work asyncronously, so we'll wait this many seconds -SLEEP_SECONDS=5 - -DEFAULT_PID_FILE="/var/run/celery/$prog.pid" -DEFAULT_LOG_FILE="/var/log/celery/$prog.log" -DEFAULT_LOG_LEVEL="INFO" -DEFAULT_NODES="celery" - -CELERY_DEFAULTS=${CELERY_DEFAULTS:-"/etc/sysconfig/$prog"} - -test -f "$CELERY_DEFAULTS" && . "$CELERY_DEFAULTS" - -# Set CELERY_CREATE_DIRS to always create log/pid dirs. -CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} -CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS -CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS -if [ -z "$CELERYBEAT_PID_FILE" ]; then - CELERYBEAT_PID_FILE="$DEFAULT_PID_FILE" - CELERY_CREATE_RUNDIR=1 -fi -if [ -z "$CELERYBEAT_LOG_FILE" ]; then - CELERYBEAT_LOG_FILE="$DEFAULT_LOG_FILE" - CELERY_CREATE_LOGDIR=1 -fi - -CELERYBEAT_LOG_LEVEL=${CELERYBEAT_LOG_LEVEL:-${CELERYBEAT_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} -CELERYBEAT=${CELERYBEAT:-"${CELERY_BIN} beat"} -CELERYBEAT=${CELERYBEAT:-$DEFAULT_CELERYBEAT} -CELERYBEAT_NODES=${CELERYBEAT_NODES:-$DEFAULT_NODES} - -# This is used to change how Celery loads in the configs. It does not need to -# be set to be run. -export CELERY_LOADER - -if [ -n "$2" ]; then - CELERYBEAT_OPTS="$CELERYBEAT_OPTS $2" -fi - -CELERYBEAT_OPTS=${CELERYBEAT_OPTS:-"--app=$CELERY_APP"} -CELERYBEAT_LOG_DIR=`dirname $CELERYBEAT_LOG_FILE` -CELERYBEAT_PID_DIR=`dirname $CELERYBEAT_PID_FILE` - -# Extra start-stop-daemon options, like user/group. -if [ -n "$CELERYBEAT_USER" ]; then - DAEMON_OPTS="$DAEMON_OPTS --uid=$CELERYBEAT_USER" -fi -if [ -n "$CELERYBEAT_GROUP" ]; then - DAEMON_OPTS="$DAEMON_OPTS --gid=$CELERYBEAT_GROUP" -fi - -if [ -n "$CELERYBEAT_CHDIR" ]; then - DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYBEAT_CHDIR" -fi - -check_dev_null() { - if [ ! -c /dev/null ]; then - echo "/dev/null is not a character device!" - exit 75 # EX_TEMPFAIL - fi -} - - -maybe_die() { - if [ $? -ne 0 ]; then - echo "Exiting: $* (errno $?)" - exit 77 # EX_NOPERM - fi -} - -create_default_dir() { - if [ ! -d "$1" ]; then - echo "- Creating default directory: '$1'" - mkdir -p "$1" - maybe_die "Couldn't create directory $1" - echo "- Changing permissions of '$1' to 02755" - chmod 02755 "$1" - maybe_die "Couldn't change permissions for $1" - if [ -n "$CELERYBEAT_USER" ]; then - echo "- Changing owner of '$1' to '$CELERYBEAT_USER'" - chown "$CELERYBEAT_USER" "$1" - maybe_die "Couldn't change owner of $1" - fi - if [ -n "$CELERYBEAT_GROUP" ]; then - echo "- Changing group of '$1' to '$CELERYBEAT_GROUP'" - chgrp "$CELERYBEAT_GROUP" "$1" - maybe_die "Couldn't change group of $1" - fi - fi -} - - -check_paths() { - if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then - create_default_dir "$CELERYBEAT_LOG_DIR" - fi - if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then - create_default_dir "$CELERYBEAT_PID_DIR" - fi -} - -create_paths() { - create_default_dir "$CELERYBEAT_LOG_DIR" - create_default_dir "$CELERYBEAT_PID_DIR" -} - -export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" - -stop() { - [[ ! -f "$CELERYBEAT_PID_FILE" ]] && echo "$prog is stopped" && return 0 - - local one_failed= - echo -n $"Stopping $prog: " - - # killproc comes from 'functions' and brings three nice features: - # 1. sending TERM, sleeping, then sleeping more if needed, then sending KILL - # 2. handling 'success' and 'failure' output - # 3. removes stale pid files, if any remain - killproc -p "$CELERYBEAT_PID_FILE" -d "$SLEEP_SECONDS" $prog || one_failed=true - echo - - [[ "$one_failed" ]] && return 1 || return 0 -} - -start() { - echo -n $"Starting $prog: " - - # If Celery is already running, bail out - if [[ -f "$CELERYBEAT_PID_FILE" ]]; then - echo -n "$prog is already running. Use 'restart'." - failure - echo - return 1 - fi - - $CELERYBEAT $CELERYBEAT_OPTS $DAEMON_OPTS --detach \ - --pidfile="$CELERYBEAT_PID_FILE" \ - --logfile="$CELERYBEAT_LOG_FILE" \ - --loglevel="$CELERYBEAT_LOG_LEVEL" - - if [[ "$?" == "0" ]]; then - # Sleep a few seconds to give Celery a chance to initialize itself. - # This is useful to prevent scripts following this one from trying to - # use Celery (or its pid files) too early. - sleep $SLEEP_SECONDS - if [[ -f "$CELERYBEAT_PID_FILE" ]]; then - success - echo - return 0 - else # celerybeat succeeded but no pid files found - failure - fi - else # celerybeat did not succeed - failure - fi - echo - return 1 -} - -check_status() { - status -p "$CELERYBEAT_PID_FILE" $"$prog" || return 1 - return 0 -} - -case "$1" in - start) - check_dev_null - check_paths - start - ;; - - stop) - check_dev_null - check_paths - stop - ;; - - status) - check_status - ;; - - restart) - check_dev_null - check_paths - stop && start - ;; - - *) - echo "Usage: /etc/init.d/$prog {start|stop|restart|status}" - exit 3 - ;; -esac - -exit $? diff --git a/extra/centos/celerybeat.sysconfig b/extra/centos/celerybeat.sysconfig deleted file mode 100644 index 50015151e..000000000 --- a/extra/centos/celerybeat.sysconfig +++ /dev/null @@ -1,15 +0,0 @@ -# In CentOS, contents should be placed in the file /etc/sysconfig/celeryd -# Available options: http://celery.readthedocs.org/en/latest/tutorials/daemonizing.html#init-script-celerybeat - -# Where the Django project is. -#CELERYBEAT_CHDIR="/path/to/my_application" - -# Absolute or relative path to the celery program -#CELERY_BIN="/usr/local/bin/celery" - -# App instance to use (value for --app argument). -#CELERY_APP="my_application.path.to.worker" - -# Beat run as an unprivileged user -#CELERYBEAT_USER="brandings" -#CELERYBEAT_GROUP="brandings" diff --git a/extra/centos/celeryd b/extra/centos/celeryd deleted file mode 100644 index 1292cc84c..000000000 --- a/extra/centos/celeryd +++ /dev/null @@ -1,266 +0,0 @@ -#!/bin/sh -# ============================================ -# celeryd - Starts the Celery worker daemon. -# ============================================ -# -# :Usage: /etc/init.d/celeryd {start|stop|restart|status} -# :Configuration file: /etc/sysconfig/celeryd -# -# See http://docs.celeryproject.org/en/latest/tutorials/daemonizing.html - -### BEGIN INIT INFO -# Provides: celeryd -# Required-Start: $network $local_fs $remote_fs -# Required-Stop: $network $local_fs $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: celery task worker daemon -### END INIT INFO -# -# -# To implement separate init scripts, do NOT copy this script. Instead, -# symlink it. I.e., if my new application, "little-worker" needs an init, I -# should just use: -# -# ln -s /etc/init.d/celeryd /etc/init.d/little-worker -# -# You can then configure this by manipulating /etc/sysconfig/little-worker. -# -# Setting `prog` here allows you to symlink this init script, making it easy -# to run multiple processes on the system. - -# If we're invoked via SysV-style runlevel scripts we need to follow the -# link from rcX.d before working out the script name. -if [[ `dirname $0` == /etc/rc*.d ]]; then - target="$(readlink $0)" -else - target=$0 -fi - -prog="$(basename $target)" - -# Source the centos service helper functions -source /etc/init.d/functions -# NOTE: "set -e" does not work with the above functions, -# which use non-zero return codes as non-error return conditions - -# some commands work asyncronously, so we'll wait this many seconds -SLEEP_SECONDS=5 - -DEFAULT_PID_FILE="/var/run/celery/$prog-%n.pid" -DEFAULT_LOG_FILE="/var/log/celery/$prog-%n%I.log" -DEFAULT_LOG_LEVEL="INFO" -DEFAULT_NODES="celery" -DEFAULT_CELERYD="-m celery.bin.celeryd_detach" - -CELERY_DEFAULTS=${CELERY_DEFAULTS:-"/etc/sysconfig/$prog"} - -test -f "$CELERY_DEFAULTS" && . "$CELERY_DEFAULTS" - -# Set CELERY_CREATE_DIRS to always create log/pid dirs. -CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} -CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS -CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS -if [ -z "$CELERYD_PID_FILE" ]; then - CELERYD_PID_FILE="$DEFAULT_PID_FILE" - CELERY_CREATE_RUNDIR=1 -fi -if [ -z "$CELERYD_LOG_FILE" ]; then - CELERYD_LOG_FILE="$DEFAULT_LOG_FILE" - CELERY_CREATE_LOGDIR=1 -fi - -CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} -CELERYD_MULTI=${CELERYD_MULTI:-"${CELERY_BIN} multi"} -CELERYD=${CELERYD:-$DEFAULT_CELERYD} -CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES} - -# This is used to change how Celery loads in the configs. It does not need to -# be set to be run. -export CELERY_LOADER - -if [ -n "$2" ]; then - CELERYD_OPTS="$CELERYD_OPTS $2" -fi - -CELERYD_LOG_DIR=`dirname $CELERYD_LOG_FILE` -CELERYD_PID_DIR=`dirname $CELERYD_PID_FILE` -CELERYD_OPTS=${CELERYD_OPTS:-"--app=$CELERY_APP"} - -# Extra start-stop-daemon options, like user/group. -if [ -n "$CELERYD_USER" ]; then - DAEMON_OPTS="$DAEMON_OPTS --uid=$CELERYD_USER" -fi -if [ -n "$CELERYD_GROUP" ]; then - DAEMON_OPTS="$DAEMON_OPTS --gid=$CELERYD_GROUP" -fi - -if [ -n "$CELERYD_CHDIR" ]; then - DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYD_CHDIR" -fi - -check_dev_null() { - if [ ! -c /dev/null ]; then - echo "/dev/null is not a character device!" - exit 75 # EX_TEMPFAIL - fi -} - - -maybe_die() { - if [ $? -ne 0 ]; then - echo "Exiting: $* (errno $?)" - exit 77 # EX_NOPERM - fi -} - -create_default_dir() { - if [ ! -d "$1" ]; then - echo "- Creating default directory: '$1'" - mkdir -p "$1" - maybe_die "Couldn't create directory $1" - echo "- Changing permissions of '$1' to 02755" - chmod 02755 "$1" - maybe_die "Couldn't change permissions for $1" - if [ -n "$CELERYD_USER" ]; then - echo "- Changing owner of '$1' to '$CELERYD_USER'" - chown "$CELERYD_USER" "$1" - maybe_die "Couldn't change owner of $1" - fi - if [ -n "$CELERYD_GROUP" ]; then - echo "- Changing group of '$1' to '$CELERYD_GROUP'" - chgrp "$CELERYD_GROUP" "$1" - maybe_die "Couldn't change group of $1" - fi - fi -} - - -check_paths() { - if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then - create_default_dir "$CELERYD_LOG_DIR" - fi - if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then - create_default_dir "$CELERYD_PID_DIR" - fi -} - -create_paths() { - create_default_dir "$CELERYD_LOG_DIR" - create_default_dir "$CELERYD_PID_DIR" -} - -export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" - - -_get_pid_files() { - [[ ! -d "$CELERYD_PID_DIR" ]] && return - echo $(ls -1 "$CELERYD_PID_DIR"/$prog-*.pid 2> /dev/null) -} - -stop() { - local pid_files=$(_get_pid_files) - [[ -z "$pid_files" ]] && echo "$prog is stopped" && return 0 - - local one_failed= - for pid_file in $pid_files; do - local pid=$(cat "$pid_file") - echo -n $"Stopping $prog (pid $pid): " - - # killproc comes from 'functions' and brings three nice features: - # 1. sending TERM, sleeping, then sleeping more if needed, then sending KILL - # 2. handling 'success' and 'failure' output - # 3. removes stale pid files, if any remain - killproc -p "$pid_file" -d "$SLEEP_SECONDS" $prog || one_failed=true - echo - done - - [[ "$one_failed" ]] && return 1 || return 0 -} - -start() { - echo -n $"Starting $prog: " - - # If Celery is already running, bail out - local pid_files=$(_get_pid_files) - if [[ "$pid_files" ]]; then - echo -n $"$prog is already running. Use 'restart'." - failure - echo - return 1 - fi - - $CELERYD_MULTI start $CELERYD_NODES $DAEMON_OPTS \ - --pidfile="$CELERYD_PID_FILE" \ - --logfile="$CELERYD_LOG_FILE" \ - --loglevel="$CELERYD_LOG_LEVEL" \ - --cmd="$CELERYD" \ - --quiet \ - $CELERYD_OPTS - - if [[ "$?" == "0" ]]; then - # Sleep a few seconds to give Celery a chance to initialize itself. - # This is useful to prevent scripts following this one from trying to - # use Celery (or its pid files) too early. - sleep $SLEEP_SECONDS - pid_files=$(_get_pid_files) - if [[ "$pid_files" ]]; then - for pid_file in $pid_files; do - local node=$(basename "$pid_file" .pid) - local pid=$(cat "$pid_file") - echo - echo -n " $node (pid $pid):" - success - done - echo - return 0 - else # celeryd_multi succeeded but no pid files found - failure - fi - else # celeryd_multi did not succeed - failure - fi - echo - return 1 -} - -check_status() { - local pid_files=$(_get_pid_files) - [[ -z "$pid_files" ]] && echo "$prog is stopped" && return 1 - for pid_file in $pid_files; do - local node=$(basename "$pid_file" .pid) - status -p "$pid_file" $"$prog (node $node)" || return 1 # if one node is down celeryd is down - done - return 0 -} - -case "$1" in - start) - check_dev_null - check_paths - start - ;; - - stop) - check_dev_null - check_paths - stop - ;; - - status) - check_status - ;; - - restart) - check_dev_null - check_paths - stop && start - ;; - - *) - echo "Usage: /etc/init.d/$prog {start|stop|restart|status}" - exit 3 - ;; -esac - -exit $? diff --git a/extra/centos/celeryd.sysconfig b/extra/centos/celeryd.sysconfig deleted file mode 100644 index c243b8b57..000000000 --- a/extra/centos/celeryd.sysconfig +++ /dev/null @@ -1,27 +0,0 @@ -# In CentOS, contents should be placed in the file /etc/sysconfig/celeryd -# Available options: http://celery.readthedocs.org/en/latest/tutorials/daemonizing.html#available-options - -# Names of nodes to start (space-separated) -#CELERYD_NODES="my_application-node_1" - -# Where to chdir at start. This could be the root of a virtualenv. -#CELERYD_CHDIR="/path/to/my_application" - -# Absolute or relative path to the celery program -#CELERY_BIN="/usr/local/bin/celery" - -# App instance to use (value for --app argument). -#CELERY_APP="my_application" - -# Create log/pid dirs, if they don't already exist -#CELERY_CREATE_DIRS=1 - -# - %n will be replaced with the first part of the nodename. -# - %I will be replaced with the current child process index -# and is important when using the prefork pool to avoid race conditions. -#CELERYD_LOG_FILE="/path/to/my_application/log/%n%I.log" -#CELERYD_PID_FILE="/var/run/celery/%n.pid" - -# Workers run as an unprivileged user -#CELERYD_USER=celery -#CELERYD_GROUP=celery diff --git a/extra/centos/test_celerybeat.sh b/extra/centos/test_celerybeat.sh deleted file mode 100755 index d60829d2d..000000000 --- a/extra/centos/test_celerybeat.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -# If you make changes to the celerybeat init script, -# you can use this test script to verify you didn't break the universe - -./test_service.sh celerybeat diff --git a/extra/centos/test_celeryd.sh b/extra/centos/test_celeryd.sh deleted file mode 100755 index 89429e924..000000000 --- a/extra/centos/test_celeryd.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -# If you make changes to the celeryd init script, -# you can use this test script to verify you didn't break the universe - -./test_service.sh celeryd diff --git a/extra/centos/test_service.sh b/extra/centos/test_service.sh deleted file mode 100755 index d5a33ba38..000000000 --- a/extra/centos/test_service.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh - -if [ -z "$1" ]; then - echo 'service name is not specified' - exit -1 -fi - -SERVICE="$1" -SERVICE_CMD="sudo /sbin/service $SERVICE" - -run_test() { - local msg="$1" - local cmd="$2" - local expected_retval="${3:-0}" - local n=${#msg} - - echo - echo `printf "%$((${n}+4))s" | tr " " "#"` - echo "# $msg #" - echo `printf "%$((${n}+4))s" | tr " " "#"` - - $cmd - local retval=$? - if [[ "$retval" == "$expected_retval" ]]; then - echo "[PASSED]" - else - echo "[FAILED]" - echo "Exit status: $retval, but expected: $expected_retval" - exit $retval - fi -} - -run_test "stop should succeed" "$SERVICE_CMD stop" 0 -run_test "status on a stopped service should return 1" "$SERVICE_CMD status" 1 -run_test "stopping a stopped celery should not fail" "$SERVICE_CMD stop" 0 -run_test "start should succeed" "$SERVICE_CMD start" 0 -run_test "status on a running service should return 0" "$SERVICE_CMD status" 0 -run_test "starting a running service should fail" "$SERVICE_CMD start" 1 -run_test "restarting a running service should succeed" "$SERVICE_CMD restart" 0 -run_test "status on a restarted service should return 0" "$SERVICE_CMD status" 0 -run_test "stop should succeed" "$SERVICE_CMD stop" 0 - -echo "All tests passed!" From 1d331d76f0ce8e2d4c71924269dd79adf12708c5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 Oct 2015 17:31:02 -0700 Subject: [PATCH 0716/1103] Fix Sphinx issues with new_cassandra --- docs/configuration.rst | 4 ++-- docs/internals/reference/index.rst | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index f53975d25..15f952713 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -537,9 +537,8 @@ Example configuration .. _conf-new_cassandra-result-backend: - new_cassandra backend settings --------------------------- +------------------------------ .. note:: @@ -625,6 +624,7 @@ Example configuration CASSANDRA_WRITE_CONSISTENCY = 'ONE' CASSANDRA_ENTRY_TTL = 86400 +.. _conf-cassandra-result-backend: Cassandra backend settings -------------------------- diff --git a/docs/internals/reference/index.rst b/docs/internals/reference/index.rst index 16897b9d0..52611b186 100644 --- a/docs/internals/reference/index.rst +++ b/docs/internals/reference/index.rst @@ -32,6 +32,7 @@ celery.backends.mongodb celery.backends.redis celery.backends.riak + celery.backends.new_cassandra celery.backends.cassandra celery.backends.couchbase celery.app.trace From 091dbe8a9b45f7385ef4172b707ee59739110833 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 Oct 2015 17:31:21 -0700 Subject: [PATCH 0717/1103] Moves docstrings from celery.rst into celery/app/base.py. Closes #2018 --- celery/app/base.py | 288 ++++++++++++++++++++++++++++- docs/reference/celery.rst | 374 +++++--------------------------------- 2 files changed, 329 insertions(+), 333 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 34cfbd4e1..7fd8c2a37 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -108,12 +108,50 @@ def _ensure_after_fork(): class Celery(object): + """Celery application. + + :param main: Name of the main module if running as `__main__`. + This is used as a prefix for task names. + :keyword broker: URL of the default broker used. + :keyword loader: The loader class, or the name of the loader class to use. + Default is :class:`celery.loaders.app.AppLoader`. + :keyword backend: The result store backend class, or the name of the + backend class to use. Default is the value of the + :setting:`CELERY_RESULT_BACKEND` setting. + :keyword amqp: AMQP object or class name. + :keyword events: Events object or class name. + :keyword log: Log object or class name. + :keyword control: Control object or class name. + :keyword set_as_current: Make this the global current app. + :keyword tasks: A task registry or the name of a registry class. + :keyword include: List of modules every worker should import. + :keyword fixups: List of fixup plug-ins (see e.g. + :mod:`celery.fixups.django`). + :keyword autofinalize: If set to False a :exc:`RuntimeError` + will be raised if the task registry or tasks are used before + the app is finalized. + + """ #: This is deprecated, use :meth:`reduce_keys` instead Pickler = AppPickler SYSTEM = platforms.SYSTEM IS_OSX, IS_WINDOWS = platforms.IS_OSX, platforms.IS_WINDOWS + #: Name of the `__main__` module. Required for standalone scripts. + #: + #: If set this will be used instead of `__main__` when automatically + #: generating task names. + main = None + + #: Custom options for command-line programs. + #: See :ref:`extending-commandoptions` + user_options = None + + #: Custom bootsteps to extend and modify the worker. + #: See :ref:`extending-bootsteps`. + steps = None + amqp_cls = 'celery.app.amqp:AMQP' backend_cls = None events_cls = 'celery.events:Events' @@ -204,9 +242,11 @@ def __init__(self, main=None, loader=None, backend=None, _register_app(self) def set_current(self): + """Makes this the current app for this thread.""" _set_current_app(self) def set_default(self): + """Makes this the default app for all threads.""" set_default_app(self) def __enter__(self): @@ -216,6 +256,16 @@ def __exit__(self, *exc_info): self.close() def close(self): + """Close any open pool connections and do any other steps necessary + to clean up after the application. + + Only necessary for dynamically created apps for which you can + use the with statement instead:: + + with Celery(set_as_current=False) as app: + with app.connection() as conn: + pass + """ self._maybe_close_pool() def on_init(self): @@ -223,17 +273,55 @@ def on_init(self): pass def start(self, argv=None): + """Run :program:`celery` using `argv`. + + Uses :data:`sys.argv` if `argv` is not specified. + + """ return instantiate( 'celery.bin.celery:CeleryCommand', app=self).execute_from_commandline(argv) def worker_main(self, argv=None): + """Run :program:`celery worker` using `argv`. + + Uses :data:`sys.argv` if `argv` is not specified. + + """ return instantiate( 'celery.bin.worker:worker', app=self).execute_from_commandline(argv) def task(self, *args, **opts): - """Creates new task class from any callable.""" + """Decorator to create a task class out of any callable. + + Examples: + + .. code-block:: python + + @app.task + def refresh_feed(url): + return … + + with setting extra options: + + .. code-block:: python + + @app.task(exchange="feeds") + def refresh_feed(url): + return … + + .. admonition:: App Binding + + For custom apps the task decorator will return a proxy + object, so that the act of creating the task is not performed + until the task is used or the task registry is accessed. + + If you are depending on binding to be deferred, then you must + not access any attributes on the returned object until the + application is fully set up (finalized). + + """ if _EXECV and opts.get('lazy', True): # When using execv the task in the original module will point to a # different app, so doing things like 'add.request' will point to @@ -316,6 +404,8 @@ def gen_task_name(self, name, module): return gen_task_name(self, name, module) def finalize(self, auto=False): + """Finalizes the app by loading built-in tasks, + and evaluating pending task decorators.""" with self._finalize_mutex: if not self.finalized: if auto and not self.autofinalize: @@ -333,6 +423,22 @@ def finalize(self, auto=False): self.on_after_finalize.send(sender=self) def add_defaults(self, fun): + """Add default configuration from dict ``d``. + + If the argument is a callable function then it will be regarded + as a promise, and it won't be loaded until the configuration is + actually needed. + + This method can be compared to:: + + >>> celery.conf.update(d) + + with a difference that 1) no copy will be made and 2) the dict will + not be transferred when the worker spawns child processes, so + it's important that the same configuration happens at import time + when pickle restores the object on the other side. + + """ if not callable(fun): d, fun = fun, lambda: d if self.configured: @@ -340,12 +446,39 @@ def add_defaults(self, fun): self._pending_defaults.append(fun) def config_from_object(self, obj, silent=False, force=False): + """Reads configuration from object, where object is either + an object or the name of a module to import. + + :keyword silent: If true then import errors will be ignored. + + :keyword force: Force reading configuration immediately. + By default the configuration will be read only when required. + + .. code-block:: pycon + + >>> celery.config_from_object("myapp.celeryconfig") + + >>> from myapp import celeryconfig + >>> celery.config_from_object(celeryconfig) + + """ self._config_source = obj if force or self.configured: self._conf = None return self.loader.config_from_object(obj, silent=silent) def config_from_envvar(self, variable_name, silent=False, force=False): + """Read configuration from environment variable. + + The value of the environment variable must be the name + of a module to import. + + .. code-block:: pycon + + >>> os.environ["CELERY_CONFIG_MODULE"] = "myapp.celeryconfig" + >>> celery.config_from_envvar("CELERY_CONFIG_MODULE") + + """ module_name = os.environ.get(variable_name) if not module_name: if silent: @@ -361,12 +494,69 @@ def config_from_cmdline(self, argv, namespace='celery'): def setup_security(self, allowed_serializers=None, key=None, cert=None, store=None, digest='sha1', serializer='json'): + """Setup the message-signing serializer. + + This will affect all application instances (a global operation). + + Disables untrusted serializers and if configured to use the ``auth`` + serializer will register the auth serializer with the provided settings + into the Kombu serializer registry. + + :keyword allowed_serializers: List of serializer names, or content_types + that should be exempt from being disabled. + :keyword key: Name of private key file to use. + Defaults to the :setting:`CELERY_SECURITY_KEY` setting. + :keyword cert: Name of certificate file to use. + Defaults to the :setting:`CELERY_SECURITY_CERTIFICATE` setting. + :keyword store: Directory containing certificates. + Defaults to the :setting:`CELERY_SECURITY_CERT_STORE` setting. + :keyword digest: Digest algorithm used when signing messages. + Default is ``sha1``. + :keyword serializer: Serializer used to encode messages after + they have been signed. See :setting:`CELERY_TASK_SERIALIZER` for + the serializers supported. + Default is ``json``. + + """ from celery.security import setup_security return setup_security(allowed_serializers, key, cert, store, digest, serializer, app=self) def autodiscover_tasks(self, packages=None, related_name='tasks', force=False): + """Try to autodiscover and import modules with a specific name (by + default 'tasks'). + + If the name is empty, this will be delegated to fixups (e.g. Django). + + For example if you have an (imagined) directory tree like this:: + + foo/__init__.py + tasks.py + models.py + + bar/__init__.py + tasks.py + models.py + + baz/__init__.py + models.py + + Then calling ``app.autodiscover_tasks(['foo', bar', 'baz'])`` will + result in the modules ``foo.tasks`` and ``bar.tasks`` being imported. + + :param packages: List of packages to search. + This argument may also be a callable, in which case the + value returned is used (for lazy evaluation). + :keyword related_name: The name of the module to find. Defaults + to "tasks", which means it look for "module.tasks" for every + module in ``packages``. + :keyword force: By default this call is lazy so that the actual + autodiscovery will not happen until an application imports the + default modules. Forcing will cause the autodiscovery to happen + immediately. + + """ if force: return self._autodiscover_tasks(packages, related_name) signals.import_modules.connect(promise( @@ -399,6 +589,15 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, reply_to=None, time_limit=None, soft_time_limit=None, root_id=None, parent_id=None, route_name=None, shadow=None, **options): + """Send task by name. + + :param name: Name of task to call (e.g. `"tasks.add"`). + :keyword result_cls: Specify custom result class. Default is + using :meth:`AsyncResult`. + + Otherwise supports the same arguments as :meth:`@-Task.apply_async`. + + """ amqp = self.amqp task_id = task_id or uuid() producer = producer or publisher # XXX compat @@ -436,6 +635,24 @@ def connection(self, hostname=None, userid=None, password=None, connect_timeout=None, transport=None, transport_options=None, heartbeat=None, login_method=None, failover_strategy=None, **kwargs): + """Establish a connection to the message broker. + + :param url: Either the URL or the hostname of the broker to use. + + :keyword hostname: URL, Hostname/IP-address of the broker. + If an URL is used, then the other argument below will + be taken from the URL instead. + :keyword userid: Username to authenticate as. + :keyword password: Password to authenticate with + :keyword virtual_host: Virtual host to use (domain). + :keyword port: Port to connect to. + :keyword ssl: Defaults to the :setting:`BROKER_USE_SSL` setting. + :keyword transport: defaults to the :setting:`BROKER_TRANSPORT` + setting. + + :returns :class:`kombu.Connection`: + + """ conf = self.conf return self.amqp.Connection( hostname or conf.BROKER_URL, @@ -466,10 +683,23 @@ def _acquire_connection(self, pool=True): return self.connection() def connection_or_acquire(self, connection=None, pool=True, *_, **__): + """For use within a with-statement to get a connection from the pool + if one is not already provided. + + :keyword connection: If not provided, then a connection will be + acquired from the connection pool. + """ return FallbackContext(connection, self._acquire_connection, pool=pool) default_connection = connection_or_acquire # XXX compat def producer_or_acquire(self, producer=None): + """For use within a with-statement to get a producer from the pool + if one is not already provided + + :keyword producer: If not provided, then a producer will be + acquired from the producer pool. + + """ return FallbackContext( producer, self.amqp.producer_pool.acquire, block=True, ) @@ -480,9 +710,12 @@ def prepare_config(self, c): return find_deprecated_settings(c) def now(self): + """Return the current time and date as a + :class:`~datetime.datetime` object.""" return self.loader.now(utc=self.conf.CELERY_ENABLE_UTC) def mail_admins(self, subject, body, fail_silently=False): + """Sends an email to the admins in the :setting:`ADMINS` setting.""" conf = self.conf if conf.ADMINS: to = [admin_email for _, admin_email in conf.ADMINS] @@ -500,6 +733,9 @@ def mail_admins(self, subject, body, fail_silently=False): ) def select_queues(self, queues=None): + """Select a subset of queues, where queues must be a list of queue + names to keep.""" + return self.amqp.queues.select(queues) def either(self, default_key, *values): @@ -508,6 +744,8 @@ def either(self, default_key, *values): return first(None, values) or self.conf.get(default_key) def bugreport(self): + """Return a string with information useful for the Celery core + developers when reporting a bug.""" return bugreport(self) def _get_backend(self): @@ -560,6 +798,11 @@ def _maybe_close_pool(self): amqp._producer_pool = None def signature(self, *args, **kwargs): + """Return a new :class:`~celery.canvas.Signature` bound to this app. + + See :meth:`~celery.signature` + + """ kwargs['app'] = self return self.canvas.signature(*args, **kwargs) @@ -671,18 +914,26 @@ def __reduce_args__(self): @cached_property def Worker(self): + """Worker application. See :class:`~@Worker`.""" return self.subclass_with_self('celery.apps.worker:Worker') @cached_property def WorkController(self, **kwargs): + """Embeddable worker. See :class:`~@WorkController`.""" return self.subclass_with_self('celery.worker:WorkController') @cached_property def Beat(self, **kwargs): + """Celerybeat scheduler application. + + See :class:`~@Beat`. + + """ return self.subclass_with_self('celery.apps.beat:Beat') @cached_property def Task(self): + """Base task class for this app.""" return self.create_task_cls() @cached_property @@ -691,6 +942,11 @@ def annotations(self): @cached_property def AsyncResult(self): + """Create new result instance. + + See :class:`celery.result.AsyncResult`. + + """ return self.subclass_with_self('celery.result:AsyncResult') @cached_property @@ -699,6 +955,11 @@ def ResultSet(self): @cached_property def GroupResult(self): + """Create new group result instance. + + See :class:`celery.result.GroupResult`. + + """ return self.subclass_with_self('celery.result:GroupResult') @cached_property @@ -713,6 +974,11 @@ def TaskSetResult(self): # XXX compat @property def pool(self): + """Broker connection pool: :class:`~@pool`. + + This attribute is not related to the workers concurrency pool. + + """ if self._pool is None: _ensure_after_fork() limit = self.conf.BROKER_POOL_LIMIT @@ -721,6 +987,8 @@ def pool(self): @property def current_task(self): + """The instance of the task that is being executed, or + :const:`None`.""" return _task_stack.top @cached_property @@ -729,14 +997,17 @@ def oid(self): @cached_property def amqp(self): + """AMQP related functionality: :class:`~@amqp`.""" return instantiate(self.amqp_cls, app=self) @cached_property def backend(self): + """Current backend instance.""" return self._get_backend() @property def conf(self): + """Current configuration.""" if self._conf is None: self._load_config() return self._conf @@ -747,18 +1018,22 @@ def conf(self, d): # noqa @cached_property def control(self): + """Remote control: :class:`~@control`.""" return instantiate(self.control_cls, app=self) @cached_property def events(self): + """Consuming and sending events: :class:`~@events`.""" return instantiate(self.events_cls, app=self) @cached_property def loader(self): + """Current loader instance.""" return get_loader_cls(self.loader_cls)(app=self) @cached_property def log(self): + """Logging: :class:`~@log`.""" return instantiate(self.log_cls, app=self) @cached_property @@ -768,11 +1043,22 @@ def canvas(self): @cached_property def tasks(self): + """Task registry. + + Accessing this attribute will also finalize the app. + + """ self.finalize(auto=True) return self._tasks @cached_property def timezone(self): + """Current timezone for this app. + + This is a cached property taking the time zone from the + :setting:`CELERY_TIMEZONE` setting. + + """ from celery.utils.timeutils import timezone conf = self.conf tz = conf.CELERY_TIMEZONE diff --git a/docs/reference/celery.rst b/docs/reference/celery.rst index 449479cfb..d8e8626b6 100644 --- a/docs/reference/celery.rst +++ b/docs/reference/celery.rst @@ -29,374 +29,84 @@ and creating Celery applications. .. versionadded:: 2.5 -.. class:: Celery(main='__main__', broker='amqp://localhost//', …) +.. autoclass:: Celery - :param main: Name of the main module if running as `__main__`. - This is used as a prefix for task names. - :keyword broker: URL of the default broker used. - :keyword loader: The loader class, or the name of the loader class to use. - Default is :class:`celery.loaders.app.AppLoader`. - :keyword backend: The result store backend class, or the name of the - backend class to use. Default is the value of the - :setting:`CELERY_RESULT_BACKEND` setting. - :keyword amqp: AMQP object or class name. - :keyword events: Events object or class name. - :keyword log: Log object or class name. - :keyword control: Control object or class name. - :keyword set_as_current: Make this the global current app. - :keyword tasks: A task registry or the name of a registry class. - :keyword include: List of modules every worker should import. - :keyword fixups: List of fixup plug-ins (see e.g. - :mod:`celery.fixups.django`). - :keyword autofinalize: If set to False a :exc:`RuntimeError` - will be raised if the task registry or tasks are used before - the app is finalized. - .. attribute:: Celery.main + .. autoattribute:: user_options - Name of the `__main__` module. Required for standalone scripts. + .. autoattribute:: steps - If set this will be used instead of `__main__` when automatically - generating task names. + .. autoattribute:: current_task - .. attribute:: Celery.conf + .. autoattribute:: amqp - Current configuration. + .. autoattribute:: backend - .. attribute:: user_options + .. autoattribute:: loader - Custom options for command-line programs. - See :ref:`extending-commandoptions` + .. autoattribute:: control + .. autoattribute:: events + .. autoattribute:: log + .. autoattribute:: tasks + .. autoattribute:: pool + .. autoattribute:: Task + .. autoattribute:: timezone - .. attribute:: steps + .. automethod:: close - Custom bootsteps to extend and modify the worker. - See :ref:`extending-bootsteps`. + .. automethod:: signature - .. attribute:: Celery.current_task + .. automethod:: bugreport - The instance of the task that is being executed, or :const:`None`. + .. automethod:: config_from_object - .. attribute:: Celery.amqp + .. automethod:: config_from_envvar - AMQP related functionality: :class:`~@amqp`. + .. automethod:: autodiscover_tasks - .. attribute:: Celery.backend + .. automethod:: add_defaults - Current backend instance. + .. automethod:: setup_security - .. attribute:: Celery.loader + .. automethod:: start - Current loader instance. + .. automethod:: task - .. attribute:: Celery.control + .. automethod:: send_task - Remote control: :class:`~@control`. + .. autoattribute:: AsyncResult - .. attribute:: Celery.events + .. autoattribute:: GroupResult - Consuming and sending events: :class:`~@events`. + .. automethod:: worker_main - .. attribute:: Celery.log + .. autoattribute:: Worker - Logging: :class:`~@log`. + .. autoattribute:: WorkController - .. attribute:: Celery.tasks + .. autoattribute:: Beat - Task registry. + .. automethod:: connection - Accessing this attribute will also finalize the app. + .. automethod:: connection_or_acquire - .. attribute:: Celery.pool + .. automethod:: producer_or_acquire - Broker connection pool: :class:`~@pool`. - This attribute is not related to the workers concurrency pool. + .. automethod:: mail_admins - .. attribute:: Celery.Task + .. automethod:: select_queues - Base task class for this app. + .. automethod:: now - .. attribute:: Celery.timezone + .. automethod:: set_current - Current timezone for this app. - This is a cached property taking the time zone from the - :setting:`CELERY_TIMEZONE` setting. + .. automethod:: finalize - .. method:: Celery.close + .. autodata:: on_configure - Close any open pool connections and do any other steps necessary - to clean up after the application. + .. autodata:: on_after_configure - Only necessary for dynamically created apps for which you can - use the with statement instead:: - - with Celery(set_as_current=False) as app: - with app.connection() as conn: - pass - - .. method:: Celery.signature - - Return a new :class:`~celery.canvas.Signature` bound to this app. - See :meth:`~celery.signature` - - .. method:: Celery.bugreport - - Return a string with information useful for the Celery core - developers when reporting a bug. - - .. method:: Celery.config_from_object(obj, silent=False, force=False) - - Reads configuration from object, where object is either - an object or the name of a module to import. - - :keyword silent: If true then import errors will be ignored. - - :keyword force: Force reading configuration immediately. - By default the configuration will be read only when required. - - .. code-block:: pycon - - >>> celery.config_from_object("myapp.celeryconfig") - - >>> from myapp import celeryconfig - >>> celery.config_from_object(celeryconfig) - - .. method:: Celery.config_from_envvar(variable_name, - silent=False, force=False) - - Read configuration from environment variable. - - The value of the environment variable must be the name - of a module to import. - - .. code-block:: pycon - - >>> os.environ["CELERY_CONFIG_MODULE"] = "myapp.celeryconfig" - >>> celery.config_from_envvar("CELERY_CONFIG_MODULE") - - .. method:: Celery.autodiscover_tasks(packages, related_name="tasks") - - With a list of packages, try to import modules of a specific name (by - default 'tasks'). - - For example if you have an (imagined) directory tree like this:: - - foo/__init__.py - tasks.py - models.py - - bar/__init__.py - tasks.py - models.py - - baz/__init__.py - models.py - - Then calling ``app.autodiscover_tasks(['foo', bar', 'baz'])`` will - result in the modules ``foo.tasks`` and ``bar.tasks`` being imported. - - :param packages: List of packages to search. - This argument may also be a callable, in which case the - value returned is used (for lazy evaluation). - - :keyword related_name: The name of the module to find. Defaults - to "tasks", which means it look for "module.tasks" for every - module in ``packages``. - :keyword force: By default this call is lazy so that the actual - autodiscovery will not happen until an application imports the - default modules. Forcing will cause the autodiscovery to happen - immediately. - - - .. method:: Celery.add_defaults(d) - - Add default configuration from dict ``d``. - - If the argument is a callable function then it will be regarded - as a promise, and it won't be loaded until the configuration is - actually needed. - - This method can be compared to:: - - >>> celery.conf.update(d) - - with a difference that 1) no copy will be made and 2) the dict will - not be transferred when the worker spawns child processes, so - it's important that the same configuration happens at import time - when pickle restores the object on the other side. - - .. method:: Celery.setup_security(…) - - Setup the message-signing serializer. - This will affect all application instances (a global operation). - - Disables untrusted serializers and if configured to use the ``auth`` - serializer will register the auth serializer with the provided settings - into the Kombu serializer registry. - - :keyword allowed_serializers: List of serializer names, or content_types - that should be exempt from being disabled. - :keyword key: Name of private key file to use. - Defaults to the :setting:`CELERY_SECURITY_KEY` setting. - :keyword cert: Name of certificate file to use. - Defaults to the :setting:`CELERY_SECURITY_CERTIFICATE` setting. - :keyword store: Directory containing certificates. - Defaults to the :setting:`CELERY_SECURITY_CERT_STORE` setting. - :keyword digest: Digest algorithm used when signing messages. - Default is ``sha1``. - :keyword serializer: Serializer used to encode messages after - they have been signed. See :setting:`CELERY_TASK_SERIALIZER` for - the serializers supported. - Default is ``json``. - - .. method:: Celery.start(argv=None) - - Run :program:`celery` using `argv`. - - Uses :data:`sys.argv` if `argv` is not specified. - - .. method:: Celery.task(fun, …) - - Decorator to create a task class out of any callable. - - Examples: - - .. code-block:: python - - @app.task - def refresh_feed(url): - return … - - with setting extra options: - - .. code-block:: python - - @app.task(exchange="feeds") - def refresh_feed(url): - return … - - .. admonition:: App Binding - - For custom apps the task decorator will return a proxy - object, so that the act of creating the task is not performed - until the task is used or the task registry is accessed. - - If you are depending on binding to be deferred, then you must - not access any attributes on the returned object until the - application is fully set up (finalized). - - - .. method:: Celery.send_task(name[, args[, kwargs[, …]]]) - - Send task by name. - - :param name: Name of task to call (e.g. `"tasks.add"`). - :keyword result_cls: Specify custom result class. Default is - using :meth:`AsyncResult`. - - Otherwise supports the same arguments as :meth:`@-Task.apply_async`. - - .. attribute:: Celery.AsyncResult - - Create new result instance. See :class:`celery.result.AsyncResult`. - - .. attribute:: Celery.GroupResult - - Create new group result instance. - See :class:`celery.result.GroupResult`. - - .. method:: Celery.worker_main(argv=None) - - Run :program:`celery worker` using `argv`. - - Uses :data:`sys.argv` if `argv` is not specified. - - .. attribute:: Celery.Worker - - Worker application. See :class:`~@Worker`. - - .. attribute:: Celery.WorkController - - Embeddable worker. See :class:`~@WorkController`. - - .. attribute:: Celery.Beat - - Celerybeat scheduler application. - See :class:`~@Beat`. - - .. method:: Celery.connection(url=default, [ssl, [transport_options={}]]) - - Establish a connection to the message broker. - - :param url: Either the URL or the hostname of the broker to use. - - :keyword hostname: URL, Hostname/IP-address of the broker. - If an URL is used, then the other argument below will - be taken from the URL instead. - :keyword userid: Username to authenticate as. - :keyword password: Password to authenticate with - :keyword virtual_host: Virtual host to use (domain). - :keyword port: Port to connect to. - :keyword ssl: Defaults to the :setting:`BROKER_USE_SSL` setting. - :keyword transport: defaults to the :setting:`BROKER_TRANSPORT` - setting. - - :returns :class:`kombu.Connection`: - - .. method:: Celery.connection_or_acquire(connection=None) - - For use within a with-statement to get a connection from the pool - if one is not already provided. - - :keyword connection: If not provided, then a connection will be - acquired from the connection pool. - - .. method:: Celery.producer_or_acquire(producer=None) - - For use within a with-statement to get a producer from the pool - if one is not already provided - - :keyword producer: If not provided, then a producer will be - acquired from the producer pool. - - .. method:: Celery.mail_admins(subject, body, fail_silently=False) - - Sends an email to the admins in the :setting:`ADMINS` setting. - - .. method:: Celery.select_queues(queues=[]) - - Select a subset of queues, where queues must be a list of queue - names to keep. - - .. method:: Celery.now() - - Return the current time and date as a :class:`~datetime.datetime` - object. - - .. method:: Celery.set_current() - - Makes this the current app for this thread. - - .. method:: Celery.finalize() - - Finalizes the app by loading built-in tasks, - and evaluating pending task decorators - - .. data:: on_configure - - Signal sent when app is loading configuration. - - .. data:: on_after_configure - - Signal sent after app has prepared the configuration. - - .. data:: on_after_finalize - - Signal sent after app has been finalized. - - .. attribute:: Celery.Pickler - - Helper class used to pickle this application. + .. autodata:: on_after_finalize Canvas primitives ----------------- From c75dfcb2d779e511a88f274e3938303d37032b41 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 Oct 2015 17:50:11 -0700 Subject: [PATCH 0718/1103] Py2.7 json does not like str in dict keys. Closes #2033 --- celery/app/amqp.py | 26 +++++++++++++++++++++++++- celery/canvas.py | 2 +- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 343b4b72e..5ca88b1fa 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -6,7 +6,7 @@ Sending and receiving messages using Kombu. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import numbers @@ -22,6 +22,7 @@ from celery import signals from celery.five import items, string_t +from celery.local import try_import from celery.utils.saferepr import saferepr from celery.utils.text import indent as textindent from celery.utils.timeutils import to_utc @@ -30,6 +31,9 @@ __all__ = ['AMQP', 'Queues', 'task_message'] +# json in Python2.7 borks if dict contains byte keys. +JSON_NEEDS_UNICODE_KEYS = not try_import('simplejson') + #: Human readable queue declaration. QUEUE_FORMAT = """ .> {0.name:<16} exchange={0.exchange.name}({0.exchange.type}) \ @@ -40,6 +44,10 @@ ('headers', 'properties', 'body', 'sent_event')) +def utf8dict(d, encoding='utf-8'): + return {k.encode(encoding): v for k, v in items(d)} + + class Queues(dict): """Queue name⇒ declaration mapping. @@ -311,6 +319,14 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, argsrepr = saferepr(args) kwargsrepr = saferepr(kwargs) + if JSON_NEEDS_UNICODE_KEYS: + if callbacks: + callbacks = [utf8dict(callback) for callback in callbacks] + if errbacks: + errbacks = [utf8dict(errback) for errback in errbacks] + if chord: + chord = utf8dict(chord) + return task_message( headers={ 'lang': 'py', @@ -380,6 +396,14 @@ def as_task_v1(self, task_id, name, args=None, kwargs=None, eta = eta and eta.isoformat() expires = expires and expires.isoformat() + if JSON_NEEDS_UNICODE_KEYS: + if callbacks: + callbacks = [utf8dict(callback) for callback in callbacks] + if errbacks: + errbacks = [utf8dict(errback) for errback in errbacks] + if chord: + chord = utf8dict(chord) + return task_message( headers={}, properties={ diff --git a/celery/canvas.py b/celery/canvas.py index d012173dc..2e196fc0d 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -10,7 +10,7 @@ """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals from collections import MutableSequence, deque from copy import deepcopy From 20c3035b658be2fc7a1b24aaf9fb50bb8241bfed Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 Oct 2015 18:34:08 -0700 Subject: [PATCH 0719/1103] json bytes decoding for Python3 (Issue #2033) --- celery/app/amqp.py | 8 ++++++-- celery/canvas.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 5ca88b1fa..a5923edd6 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -9,6 +9,7 @@ from __future__ import absolute_import, unicode_literals import numbers +import sys from collections import Mapping, namedtuple from datetime import timedelta @@ -31,8 +32,10 @@ __all__ = ['AMQP', 'Queues', 'task_message'] +PY3 = sys.version_info[0] == 3 + # json in Python2.7 borks if dict contains byte keys. -JSON_NEEDS_UNICODE_KEYS = not try_import('simplejson') +JSON_NEEDS_UNICODE_KEYS = not PY3 and not try_import('simplejson') #: Human readable queue declaration. QUEUE_FORMAT = """ @@ -45,7 +48,8 @@ def utf8dict(d, encoding='utf-8'): - return {k.encode(encoding): v for k, v in items(d)} + return {k.decode(encoding) if isinstance(k, bytes) else k: v + for k, v in items(d)} class Queues(dict): diff --git a/celery/canvas.py b/celery/canvas.py index 2e196fc0d..adb7aa465 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -12,6 +12,8 @@ """ from __future__ import absolute_import, unicode_literals +import sys + from collections import MutableSequence, deque from copy import deepcopy from functools import partial as _partial, reduce @@ -21,6 +23,7 @@ from kombu.utils import cached_property, fxrange, reprcall, uuid from celery._state import current_app, get_current_worker_task +from celery.local import try_import from celery.result import GroupResult from celery.utils import abstract from celery.utils.functional import ( @@ -32,6 +35,11 @@ __all__ = ['Signature', 'chain', 'xmap', 'xstarmap', 'chunks', 'group', 'chord', 'signature', 'maybe_signature'] +PY3 = sys.version_info[0] == 3 + +# json in Python2.7 borks if dict contains byte keys. +JSON_NEEDS_UNICODE_KEYS = PY3 and not try_import('simplejson') + class _getitem_property(object): """Attribute -> dict key descriptor. @@ -323,6 +331,11 @@ def election(self): def __repr__(self): return self.reprcall() + if JSON_NEEDS_UNICODE_KEYS: + def items(self): + for k, v in dict.items(self): + yield k.decode() if isinstance(k, bytes) else k, v + @property def name(self): # for duck typing compatibility with Task.name From 730e00adf2cde3e735215e22d5e1a18149691238 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 Oct 2015 18:37:07 -0700 Subject: [PATCH 0720/1103] Adds Trove classifier for Python 3.5 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index e678ee7bb..009cd33e4 100644 --- a/setup.py +++ b/setup.py @@ -67,6 +67,7 @@ Programming Language :: Python :: 3 Programming Language :: Python :: 3.3 Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy Programming Language :: Python :: Implementation :: Jython From 0c8c7df4e3bb0ad51235067cd4fde1a5c8746b89 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 Oct 2015 18:41:46 -0700 Subject: [PATCH 0721/1103] OpenSSL tests works on Python 3 again. Closes #2040 --- celery/tests/security/case.py | 2 -- celery/tests/security/test_serialization.py | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/celery/tests/security/case.py b/celery/tests/security/case.py index ba421a9d5..4c9dcd516 100644 --- a/celery/tests/security/case.py +++ b/celery/tests/security/case.py @@ -8,8 +8,6 @@ class SecurityCase(AppCase): def setup(self): - if sys.version_info[0] == 3: - raise SkipTest('PyOpenSSL does not work on Python 3') try: from OpenSSL import crypto # noqa except ImportError: diff --git a/celery/tests/security/test_serialization.py b/celery/tests/security/test_serialization.py index 50bc4bfab..e66ae6fdc 100644 --- a/celery/tests/security/test_serialization.py +++ b/celery/tests/security/test_serialization.py @@ -4,6 +4,7 @@ import base64 from kombu.serialization import registry +from kombu.utils.encoding import bytes_to_str from celery.exceptions import SecurityError from celery.security.serialization import SecureSerializer, register_auth @@ -59,6 +60,6 @@ def test_register_auth(self): def test_lots_of_sign(self): for i in range(1000): - rdata = base64.urlsafe_b64encode(os.urandom(265)) + rdata = bytes_to_str(base64.urlsafe_b64encode(os.urandom(265))) s = self._get_s(KEY1, CERT1, [CERT1]) self.assertEqual(s.deserialize(s.serialize(rdata)), rdata) From 3377644364d31308cbec786a03a61ce4f972dc78 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 Oct 2015 18:44:57 -0700 Subject: [PATCH 0722/1103] No love for Python 3.3 --- .travis.yml | 1 - README.rst | 69 ++++++++++----------------- docs/getting-started/introduction.rst | 4 +- docs/includes/introduction.txt | 2 +- docs/whatsnew-4.0.rst | 2 +- setup.py | 11 ++--- tox.ini | 7 ++- 7 files changed, 36 insertions(+), 60 deletions(-) diff --git a/.travis.yml b/.travis.yml index 700106f3e..26d593ad5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,6 @@ env: PYTHONUNBUFFERED=yes matrix: - TOXENV=2.7 - - TOXENV=3.3 - TOXENV=3.4 - TOXENV=pypy - TOXENV=3.5 diff --git a/README.rst b/README.rst index af7b6e9b6..38671fab0 100644 --- a/README.rst +++ b/README.rst @@ -4,9 +4,7 @@ .. image:: http://cloud.github.com/downloads/celery/celery/celery_128.png -|build-status| |coverage-status| - -:Version: 4.0.0a1 (Cipater) +:Version: 4.0.0a1 (0today8) :Web: http://celeryproject.org/ :Download: http://pypi.python.org/pypi/celery/ :Source: http://github.com/celery/celery/ @@ -36,7 +34,7 @@ any language. So far there's RCelery_ for the Ruby programming language, and a `PHP client`, but language interoperability can also be achieved by using webhooks. -.. _RCelery: https://github.com/leapfrogonline/rcelery +.. _RCelery: http://leapfrogdevelopment.github.com/rcelery/ .. _`PHP client`: https://github.com/gjedeer/celery-php .. _`using webhooks`: http://docs.celeryproject.org/en/latest/userguide/remote-tasks.html @@ -46,7 +44,7 @@ What do I need? Celery version 3.0 runs on, -- Python (2.6, 2.7, 3.3, 3.4) +- Python (2.7, 3.4, 3.5) - PyPy (1.8, 1.9) - Jython (2.5, 2.7). @@ -166,26 +164,26 @@ Framework Integration Celery is easy to integrate with web frameworks, some of which even have integration packages: - +--------------------+----------------------------------------------------+ - | `Django`_ | not needed | - +--------------------+----------------------------------------------------+ - | `Pyramid`_ | `pyramid_celery`_ | - +--------------------+----------------------------------------------------+ - | `Pylons`_ | `celery-pylons`_ | - +--------------------+----------------------------------------------------+ - | `Flask`_ | not needed | - +--------------------+----------------------------------------------------+ - | `web2py`_ | `web2py-celery`_ | - +--------------------+----------------------------------------------------+ - | `Tornado`_ | `tornado-celery`_ | `another tornado-celery`_ | - +--------------------+----------------------------------------------------+ + +--------------------+------------------------+ + | `Django`_ | not needed | + +--------------------+------------------------+ + | `Pyramid`_ | `pyramid_celery`_ | + +--------------------+------------------------+ + | `Pylons`_ | `celery-pylons`_ | + +--------------------+------------------------+ + | `Flask`_ | not needed | + +--------------------+------------------------+ + | `web2py`_ | `web2py-celery`_ | + +--------------------+------------------------+ + | `Tornado`_ | `tornado-celery`_ | + +--------------------+------------------------+ The integration packages are not strictly necessary, but they can make development easier, and sometimes they add important hooks like closing database connections at ``fork``. .. _`Django`: http://djangoproject.com/ -.. _`Pylons`: http://www.pylonsproject.org/ +.. _`Pylons`: http://pylonsproject.org/ .. _`Flask`: http://flask.pocoo.org/ .. _`web2py`: http://web2py.com/ .. _`Bottle`: http://bottlepy.org/ @@ -196,7 +194,6 @@ database connections at ``fork``. .. _`web2py-celery`: http://code.google.com/p/web2py-celery/ .. _`Tornado`: http://www.tornadoweb.org/ .. _`tornado-celery`: http://github.com/mher/tornado-celery/ -.. _`another tornado-celery`: https://github.com/mayflaver/tornado-celery .. _celery-documentation: @@ -287,7 +284,10 @@ Transports and Backends for using memcached as a result backend. :celery[cassandra]: - for using Apache Cassandra as a result backend. + for using Apache Cassandra as a result backend with pycassa driver. + +:celery[new_cassandra]: + for using Apache Cassandra as a result backend with DataStax driver. :celery[couchdb]: for using CouchDB as a message transport (*experimental*). @@ -295,6 +295,9 @@ Transports and Backends :celery[couchbase]: for using CouchBase as a result backend. +:celery[riak]: + for using Riak as a result backend. + :celery[beanstalk]: for using Beanstalk as a message transport (*experimental*). @@ -396,26 +399,6 @@ Wiki http://wiki.github.com/celery/celery/ - -.. _maintainers: - -Maintainers -=========== - -- `@ask`_ (primary maintainer) -- `@thedrow`_ -- `@chrisgogreen`_ -- `@PMickael`_ -- `@malinoff`_ -- And you? We really need more: https://github.com/celery/celery/issues/2534 - -.. _`@ask`: http://github.com/ask -.. _`@thedrow`: http://github.com/thedrow -.. _`@chrisgogreen`: http://github.com/chrisgogreen -.. _`@PMickael`: http://github.com/PMickael -.. _`@malinoff`: http://github.com/malinoff - - .. _contributing-short: Contributing @@ -448,7 +431,3 @@ file in the top distribution directory for the full license text. :alt: Bitdeli badge :target: https://bitdeli.com/free -.. |build-status| image:: https://travis-ci.org/celery/celery.svg?branch=master - :target: https://travis-ci.org/celery/celery -.. |coverage-status| image:: https://codecov.io/gh/celery/celery/badge.svg - :target: https://codecov.io/gh/celery/celery diff --git a/docs/getting-started/introduction.rst b/docs/getting-started/introduction.rst index 05bb72632..f7d015932 100644 --- a/docs/getting-started/introduction.rst +++ b/docs/getting-started/introduction.rst @@ -37,9 +37,9 @@ What do I need? =============== .. sidebar:: Version Requirements - :subtitle: Celery version 3.0 runs on + :subtitle: Celery version 4.0 runs on - - Python ❨2.5, 2.6, 2.7, 3.2, 3.3, 3.4❩ + - Python ❨2.7, 3.4, 3.5❩ - PyPy ❨1.8, 1.9❩ - Jython ❨2.5, 2.7❩. diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 0aff1ea0b..16e2d2b59 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -38,7 +38,7 @@ What do I need? Celery version 3.0 runs on, -- Python (2.6, 2.7, 3.3, 3.4) +- Python (2.7, 3.4, 3.5) - PyPy (1.8, 1.9) - Jython (2.5, 2.7). diff --git a/docs/whatsnew-4.0.rst b/docs/whatsnew-4.0.rst index aed087003..7a8e808e5 100644 --- a/docs/whatsnew-4.0.rst +++ b/docs/whatsnew-4.0.rst @@ -28,7 +28,7 @@ To read more about Celery you should go read the :ref:`introduction `. While this version is backward compatible with previous versions it's important that you read the following section. -This version is officially supported on CPython 2.6, 2.7 and 3.3, +This version is officially supported on CPython 2.7, 3.4 and 3.5. and also supported on PyPy. .. _`website`: http://celeryproject.org/ diff --git a/setup.py b/setup.py index 009cd33e4..be8d51624 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,8 @@ if sys.version_info < (2, 7): raise Exception('Celery 4.0 requires Python 2.7 or higher.') +elif sys.version_info > (3, ) < (3, 4): + raise Exception('Celery 4.0 requires Python 3.4 or higher.') # -*- Upgrading from older versions -*- @@ -65,7 +67,6 @@ Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.3 Programming Language :: Python :: 3.4 Programming Language :: Python :: 3.5 Programming Language :: Python :: Implementation :: CPython @@ -173,18 +174,16 @@ def extras(*p): return reqs('extras', *p) # Celery specific -features = { +features = set([ 'auth', 'cassandra', 'memcache', 'couchbase', 'threads', 'eventlet', 'gevent', 'msgpack', 'yaml', 'redis', 'mongodb', 'sqs', 'couchdb', 'riak', 'beanstalk', 'zookeeper', 'zeromq', 'sqlalchemy', 'librabbitmq', 'pyro', 'slmq', 'new_cassandra', -} -extras_require = {x: extras(x + '.txt') for x in features} +]) +extras_require = dict((x, extras(x + '.txt')) for x in features) extra['extras_require'] = extras_require -print(tests_require) - # -*- %%% -*- setup( diff --git a/tox.ini b/tox.ini index 6c86d8064..9a087101b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = 2.7,pypy,3.3,3.4,3.5,pypy3 +envlist = 2.7,pypy,3.4,3.5,pypy3 [testenv] deps= @@ -8,8 +8,8 @@ deps= 2.7,pypy: -r{toxinidir}/requirements/test.txt 2.7: -r{toxinidir}/requirements/test-ci-default.txt - 3.3,3.4,3.5,pypy3: -r{toxinidir}/requirements/test3.txt - 3.3,3.4,3.5: -r{toxinidir}/requirements/test-ci-default.txt + 3.4,3.5,pypy3: -r{toxinidir}/requirements/test3.txt + 3.4,3.5: -r{toxinidir}/requirements/test-ci-default.txt pypy,pypy3: -r{toxinidir}/requirements/test-ci-base.txt pypy3: -r{toxinidir}/requirements/test-pypy3.txt @@ -21,7 +21,6 @@ commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] basepython = 2.7: python2.7 - 3.3: python3.3 3.4: python3.4 3.5: python3.5 pypy: pypy From 64bf8aebba92538ecf6f03e1c76b6e99828ee876 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 22 Oct 2015 18:46:06 -0700 Subject: [PATCH 0723/1103] Update versions in examples --- Changelog | 6 +++--- docs/getting-started/next-steps.rst | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Changelog b/Changelog index 201d85cd3..183a24ffb 100644 --- a/Changelog +++ b/Changelog @@ -4,9 +4,9 @@ Change history ================ -This document contains change notes for bugfix releases in the 4.0.x series -(Cipater), please see :ref:`whatsnew-4.0` for an overview of what's -new in Celery 4.0. +This document contains change notes for bugfix releases in +the 4.0.x series (0today8), please see :ref:`whatsnew-4.0` for +an overview of what's new in Celery 4.0. .. _version-4.0.0: diff --git a/docs/getting-started/next-steps.rst b/docs/getting-started/next-steps.rst index d93ec6e98..1cf98eb5b 100644 --- a/docs/getting-started/next-steps.rst +++ b/docs/getting-started/next-steps.rst @@ -78,7 +78,7 @@ The :program:`celery` program can be used to start the worker (you need to run t When the worker starts you should see a banner and some messages:: - -------------- celery@halcyon.local v3.1 (Cipater) + -------------- celery@halcyon.local v4.0 (0today8) ---- **** ----- --- * *** * -- [Configuration] -- * - **** --- . broker: amqp://guest@localhost:5672// @@ -152,7 +152,7 @@ start one or more workers in the background: .. code-block:: console $ celery multi start w1 -A proj -l info - celery multi v3.1.1 (Cipater) + celery multi v4.0.0 (0today8) > Starting nodes... > w1.halcyon.local: OK @@ -161,13 +161,13 @@ You can restart it too: .. code-block:: console $ celery multi restart w1 -A proj -l info - celery multi v3.1.1 (Cipater) + celery multi v4.0.0 (0today8) > Stopping nodes... > w1.halcyon.local: TERM -> 64024 > Waiting for 1 node..... > w1.halcyon.local: OK > Restarting node w1.halcyon.local: OK - celery multi v3.1.1 (Cipater) + celery multi v4.0.0 (0today8) > Stopping nodes... > w1.halcyon.local: TERM -> 64052 From 2f58c35340f64875d40b5d3a97c6d4bdad6f74ad Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 23 Oct 2015 13:33:11 -0700 Subject: [PATCH 0724/1103] Worker: Only start pidbox if transport supports fanout. Closes celery/kombu#387 --- celery/worker/__init__.py | 3 ++- celery/worker/consumer.py | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/celery/worker/__init__.py b/celery/worker/__init__.py index 416262cf1..1b86fd813 100644 --- a/celery/worker/__init__.py +++ b/celery/worker/__init__.py @@ -245,7 +245,8 @@ def signal_consumer_close(self): def should_use_eventloop(self): return (detect_environment() == 'default' and - self._conninfo.is_evented and not self.app.IS_WINDOWS) + self._conninfo.transport.implements.async and + not self.app.IS_WINDOWS) def stop(self, in_sighandler=False, exitcode=None): """Graceful shutdown of the worker server.""" diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 8077f954c..c10d576ba 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -176,9 +176,9 @@ def __init__(self, on_task_request, self.pool = pool self.timer = timer self.strategies = self.Strategies() - conninfo = self.app.connection() - self.connection_errors = conninfo.connection_errors - self.channel_errors = conninfo.channel_errors + self.conninfo = self.app.connection() + self.connection_errors = self.conninfo.connection_errors + self.channel_errors = self.conninfo.channel_errors self._restart_state = restart_state(maxR=5, maxT=1) self._does_info = logger.isEnabledFor(logging.INFO) @@ -685,7 +685,8 @@ def __init__(self, c, **kwargs): self.shutdown = self.box.shutdown def include_if(self, c): - return c.app.conf.CELERY_ENABLE_REMOTE_CONTROL + return (c.app.conf.CELERY_ENABLE_REMOTE_CONTROL and + 'fanout' in c.conninfo.transport.implements.exchange_type) class Gossip(bootsteps.ConsumerStep): From e7a4c85aa3d5acbc63cf0fb3dc9e30a39e57a4fc Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 23 Oct 2015 14:02:46 -0700 Subject: [PATCH 0725/1103] Redis transport now supports BROKER_USE_SSL (Issue celery/kombu#415) --- docs/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 15f952713..dcb8ab4f6 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1313,7 +1313,7 @@ will be performed every 5 seconds (twice the heartbeat sending rate). BROKER_USE_SSL ~~~~~~~~~~~~~~ -:transports supported: ``pyamqp`` +:transports supported: ``pyamqp``, ``redis`` Toggles SSL usage on broker connection and SSL settings. From fdf05eb3563681fb8318f76b3e15cb779859f0c0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 23 Oct 2015 15:57:34 -0700 Subject: [PATCH 0726/1103] ehm, Logic --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index be8d51624..aeec75afd 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ if sys.version_info < (2, 7): raise Exception('Celery 4.0 requires Python 2.7 or higher.') -elif sys.version_info > (3, ) < (3, 4): +elif sys.version_info > (3, ) and sys.version_info < (3, 4): raise Exception('Celery 4.0 requires Python 3.4 or higher.') # -*- Upgrading from older versions -*- From 018ea2ef9b9293508ef65788104a1792912167a3 Mon Sep 17 00:00:00 2001 From: Krzysztof Bujniewicz Date: Fri, 23 Oct 2015 12:59:48 +0200 Subject: [PATCH 0727/1103] Fix celery beat --detach in PyPy While running celery beat under PyPy, file descriptor pointing to /dev/urandom is closed while daemonizing. This makes shelve, and in turn beat's scheduler, unable to access it, hence the startup fails with OSError 9. This is fixed by /dev/urandom's fd to keep lsit passed to close_open_fds. --- CONTRIBUTORS.txt | 1 + celery/platforms.py | 45 ++++++++++++++++++++++++++-- celery/tests/utils/test_platforms.py | 10 +++++++ 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index c8991fbc9..4994ea119 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -194,3 +194,4 @@ Justin Patrin, 2015/08/06 Juan Rossi, 2015/08/10 Piotr Maślanka, 2015/08/24 Gerald Manipon, 2015/10/19 +Krzysztof Bujniewicz, 2015/10/21 diff --git a/celery/platforms.py b/celery/platforms.py index 047270406..6f7d9c6ea 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -46,7 +46,8 @@ 'close_open_fds', 'DaemonContext', 'detached', 'parse_uid', 'parse_gid', 'setgroups', 'initgroups', 'setgid', 'setuid', 'maybe_drop_privileges', 'signals', 'set_process_title', - 'set_mp_process_title', 'get_errno_name', 'ignore_errno'] + 'set_mp_process_title', 'get_errno_name', 'ignore_errno', + 'fd_by_path'] # exitcodes EX_OK = getattr(os, 'EX_OK', 0) @@ -247,6 +248,43 @@ def _create_pidlock(pidfile): pidlock.acquire() return pidlock +def fd_by_path(paths): + """ + Return a list of fds. + + This method returns list of fds corresponding to + file paths passed in paths variable. + + :keyword paths: List of file paths go get fd for. + + :returns: :list:. + + **Example**: + + .. code-block:: python + + keep = fd_by_path(['/dev/urandom', + '/my/precious/']) + """ + stats = set() + for path in paths: + try: + fd = os.open(path, os.O_RDONLY) + except OSError: + continue + try: + stats.add(os.fstat(fd)[1:3]) + finally: + os.close(fd) + + def fd_in_stats(fd): + try: + return os.fstat(fd)[1:3] in stats + except OSError: + return False + + return [fd for fd in range(get_fdmax(2048)) if fd_in_stats(fd)] + class DaemonContext(object): _is_open = False @@ -282,7 +320,10 @@ def open(self): self.after_chdir() if not self.fake: - close_open_fds(self.stdfds) + # We need to keep /dev/urandom from closing because + # shelve needs it, and Beat needs shelve to start. + keep = list(self.stdfds) + fd_by_path(['/dev/urandom']) + close_open_fds(keep) for fd in self.stdfds: self.redirect_to_null(maybe_fileno(fd)) if self.after_forkers and mputil is not None: diff --git a/celery/tests/utils/test_platforms.py b/celery/tests/utils/test_platforms.py index 02dd7bece..e8ac4d143 100644 --- a/celery/tests/utils/test_platforms.py +++ b/celery/tests/utils/test_platforms.py @@ -4,6 +4,7 @@ import os import sys import signal +import tempfile from celery import _find_option_with_arg from celery import platforms @@ -27,6 +28,7 @@ setgroups, _setgroups_hack, close_open_fds, + fd_by_path, ) try: @@ -54,6 +56,14 @@ def test_short_opt(self): 'bar' ) +class test_fd_by_path(Case): + + def test_finds(self): + test_file = tempfile.NamedTemporaryFile() + keep = fd_by_path([test_file.name]) + self.assertEqual(keep, [test_file.file.fileno()]) + test_file.close() + class test_close_open_fds(Case): From 1beb6a4c36f7fd9b4c8e77407e26179b4344eb53 Mon Sep 17 00:00:00 2001 From: Sergey Tikhonov Date: Mon, 26 Oct 2015 15:24:15 +0300 Subject: [PATCH 0728/1103] make EventDispatcher.send consistent with docstring --- celery/events/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/celery/events/__init__.py b/celery/events/__init__.py index 1fcf36ee7..b61d6be7f 100644 --- a/celery/events/__init__.py +++ b/celery/events/__init__.py @@ -216,7 +216,8 @@ def _publish(self, event, producer, routing_key, retry=False, raise self._outbound_buffer.append((event, routing_key, exc)) - def send(self, type, blind=False, utcoffset=utcoffset, **fields): + def send(self, type, retry=False, retry_policy=None, blind=False, + Event=Event, utcoffset=utcoffset, **fields): """Send event. :param type: Event type name, with group separated by dash (`-`). @@ -247,7 +248,9 @@ def send(self, type, blind=False, utcoffset=utcoffset, **fields): elif self.on_send_buffered: self.on_send_buffered() else: - return self.publish(type, fields, self.producer, blind) + return self.publish(type, fields, self.producer, retry=retry, + retry_policy=retry_policy, blind=blind, + Event=Event) def flush(self, errors=True, groups=True): """Flushes the outbound buffer.""" From 1919256eb52dc7a6b4287403a5cd77f9a6b430b5 Mon Sep 17 00:00:00 2001 From: Sergey Tikhonov Date: Mon, 26 Oct 2015 15:26:45 +0300 Subject: [PATCH 0729/1103] reorder args to be more backward compatible --- celery/events/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/celery/events/__init__.py b/celery/events/__init__.py index b61d6be7f..800a615a5 100644 --- a/celery/events/__init__.py +++ b/celery/events/__init__.py @@ -216,8 +216,8 @@ def _publish(self, event, producer, routing_key, retry=False, raise self._outbound_buffer.append((event, routing_key, exc)) - def send(self, type, retry=False, retry_policy=None, blind=False, - Event=Event, utcoffset=utcoffset, **fields): + def send(self, type, blind=False, utcoffset=utcoffset, retry=False, + retry_policy=None, Event=Event, **fields): """Send event. :param type: Event type name, with group separated by dash (`-`). @@ -248,9 +248,9 @@ def send(self, type, retry=False, retry_policy=None, blind=False, elif self.on_send_buffered: self.on_send_buffered() else: - return self.publish(type, fields, self.producer, retry=retry, - retry_policy=retry_policy, blind=blind, - Event=Event) + return self.publish(type, fields, self.producer, blind=blind, + Event=Event, retry=retry, + retry_policy=retry_policy) def flush(self, errors=True, groups=True): """Flushes the outbound buffer.""" From f80e5da20487a133992d8429c5a792adeda1c17d Mon Sep 17 00:00:00 2001 From: sukrit007 Date: Tue, 4 Aug 2015 17:12:42 -0700 Subject: [PATCH 0730/1103] Fix for https://github.com/celery/celery/issues/2743 Fixes celery issue for pymongo 3+ with gevent --- celery/backends/mongodb.py | 4 --- celery/tests/backends/test_mongodb.py | 45 +++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index adf535c43..5a57ffccc 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -10,7 +10,6 @@ from datetime import datetime, timedelta -from kombu.syn import detect_environment from kombu.utils import cached_property from kombu.exceptions import EncodeError from celery import states @@ -158,9 +157,6 @@ def _get_connection(self): conf = dict(self.options) conf['host'] = host - if detect_environment() != 'default': - conf['use_greenlets'] = True - self._connection = MongoClient(**conf) return self._connection diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index 2d656a6d5..1d073ec3f 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -310,9 +310,15 @@ def test_restore_group(self, mock_get_database): mock_get_database.assert_called_once_with() mock_collection.find_one.assert_called_once_with( {'_id': sentinel.taskset_id}) +<<<<<<< HEAD self.assertEqual( list(sorted(['date_done', 'result', 'task_id'])), list(sorted(ret_val.keys())), +======= + self.assertItemsEqual( + ['date_done', 'result', 'task_id'], + list(ret_val.keys()), +>>>>>>> e758762... Fix for https://github.com/celery/celery/issues/2743 ) @patch('celery.backends.mongodb.MongoBackend._get_database') @@ -380,3 +386,42 @@ def test_get_database_authfailure(self): with self.assertRaises(ImproperlyConfigured): x._get_database() db.authenticate.assert_called_with('jerry', 'cere4l') + + @patch('celery.backends.mongodb.detect_environment') + def test_prepare_client_options_for_ver_2(self, m_detect_env): + m_detect_env.return_value = 'default' + with patch('pymongo.version_tuple', new=(2, 6, 3)): + options = self.backend._prepare_client_options() + self.assertDictEqual(options, { + 'max_pool_size': self.backend.max_pool_size, + 'auto_start_request': False + }) + + @patch('celery.backends.mongodb.detect_environment') + def test_prepare_client_options_for_ver_2_with_gevent(self, m_detect_env): + m_detect_env.return_value = 'gevent' + with patch('pymongo.version_tuple', new=(2, 6, 3)): + options = self.backend._prepare_client_options() + self.assertDictEqual(options, { + 'max_pool_size': self.backend.max_pool_size, + 'auto_start_request': False, + 'use_greenlets': True + }) + + @patch('celery.backends.mongodb.detect_environment') + def test_prepare_client_options_for_ver_3(self, m_detect_env): + m_detect_env.return_value = 'default' + with patch('pymongo.version_tuple', new=(3, 0, 3)): + options = self.backend._prepare_client_options() + self.assertDictEqual(options, { + 'maxPoolSize': self.backend.max_pool_size + }) + + @patch('celery.backends.mongodb.detect_environment') + def test_prepare_client_options_for_ver_3_with_gevent(self, m_detect_env): + m_detect_env.return_value = 'gevent' + with patch('pymongo.version_tuple', new=(3, 0, 3)): + options = self.backend._prepare_client_options() + self.assertDictEqual(options, { + 'maxPoolSize': self.backend.max_pool_size + }) From 0659e5bd5994de769fa10878ea3dc3dd933cf492 Mon Sep 17 00:00:00 2001 From: sukrit007 Date: Tue, 4 Aug 2015 17:20:59 -0700 Subject: [PATCH 0731/1103] Fix broken tests --- celery/tests/backends/test_mongodb.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index 1d073ec3f..595fbaa23 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -310,15 +310,9 @@ def test_restore_group(self, mock_get_database): mock_get_database.assert_called_once_with() mock_collection.find_one.assert_called_once_with( {'_id': sentinel.taskset_id}) -<<<<<<< HEAD - self.assertEqual( - list(sorted(['date_done', 'result', 'task_id'])), - list(sorted(ret_val.keys())), -======= self.assertItemsEqual( ['date_done', 'result', 'task_id'], list(ret_val.keys()), ->>>>>>> e758762... Fix for https://github.com/celery/celery/issues/2743 ) @patch('celery.backends.mongodb.MongoBackend._get_database') From 718892665de4697aa1a614ef2e61075d9b1a3245 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 26 Oct 2015 12:12:25 -0700 Subject: [PATCH 0732/1103] flakes --- celery/app/base.py | 4 ++-- celery/app/trace.py | 4 +++- celery/tests/security/case.py | 2 -- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 7fd8c2a37..40d4afc26 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -502,8 +502,8 @@ def setup_security(self, allowed_serializers=None, key=None, cert=None, serializer will register the auth serializer with the provided settings into the Kombu serializer registry. - :keyword allowed_serializers: List of serializer names, or content_types - that should be exempt from being disabled. + :keyword allowed_serializers: List of serializer names, or + content_types that should be exempt from being disabled. :keyword key: Name of private key file to use. Defaults to the :setting:`CELERY_SECURITY_KEY` setting. :keyword cert: Name of certificate file to use. diff --git a/celery/app/trace.py b/celery/app/trace.py index 393aeb461..97860f817 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -394,7 +394,9 @@ def trace_task(uuid, args, kwargs, request=None): group(sigs).apply_async((retval,)) else: signature(callbacks[0], app=app).delay(retval) - mark_as_done(uuid, retval, task_request, publish_result) + mark_as_done( + uuid, retval, task_request, publish_result, + ) except EncodeError as exc: I, R, state, retval = on_error(task_request, exc, uuid) else: diff --git a/celery/tests/security/case.py b/celery/tests/security/case.py index 4c9dcd516..4440f4963 100644 --- a/celery/tests/security/case.py +++ b/celery/tests/security/case.py @@ -2,8 +2,6 @@ from celery.tests.case import AppCase, SkipTest -import sys - class SecurityCase(AppCase): From 91d89d5691e82f8551c4938a160e60ba870b9df7 Mon Sep 17 00:00:00 2001 From: Sukrit Khera Date: Mon, 26 Oct 2015 12:42:02 -0700 Subject: [PATCH 0733/1103] Adding Sukrit Khera to contributor list Adding Sukrit Khera to contributor list --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 4994ea119..bfc00f31b 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -195,3 +195,4 @@ Juan Rossi, 2015/08/10 Piotr Maślanka, 2015/08/24 Gerald Manipon, 2015/10/19 Krzysztof Bujniewicz, 2015/10/21 +Sukrit Khera, 2015/10/26 From 0196f0682b6eedf0f526297f1edf99af85d1912f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 26 Oct 2015 13:04:32 -0700 Subject: [PATCH 0734/1103] flakes --- celery/platforms.py | 6 +++--- celery/tests/utils/test_platforms.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/celery/platforms.py b/celery/platforms.py index 6f7d9c6ea..75d71db85 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -248,9 +248,9 @@ def _create_pidlock(pidfile): pidlock.acquire() return pidlock + def fd_by_path(paths): - """ - Return a list of fds. + """Return a list of fds. This method returns list of fds corresponding to file paths passed in paths variable. @@ -283,7 +283,7 @@ def fd_in_stats(fd): except OSError: return False - return [fd for fd in range(get_fdmax(2048)) if fd_in_stats(fd)] + return [_fd for _fd in range(get_fdmax(2048)) if fd_in_stats(_fd)] class DaemonContext(object): diff --git a/celery/tests/utils/test_platforms.py b/celery/tests/utils/test_platforms.py index e8ac4d143..5c4e568d5 100644 --- a/celery/tests/utils/test_platforms.py +++ b/celery/tests/utils/test_platforms.py @@ -56,6 +56,7 @@ def test_short_opt(self): 'bar' ) + class test_fd_by_path(Case): def test_finds(self): From 34538d62017612ad6b6944614879bcf058cd2287 Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Mon, 26 Oct 2015 18:00:24 -0400 Subject: [PATCH 0735/1103] Added a missing comma in the docs. --- docs/getting-started/first-steps-with-celery.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/first-steps-with-celery.rst b/docs/getting-started/first-steps-with-celery.rst index 23d1df848..0231137de 100644 --- a/docs/getting-started/first-steps-with-celery.rst +++ b/docs/getting-started/first-steps-with-celery.rst @@ -275,7 +275,7 @@ See :mod:`celery.result` for the complete result object reference. Configuration ============= -Celery, like a consumer appliance doesn't need much to be operated. +Celery, like a consumer appliance, doesn't need much to be operated. It has an input and an output, where you must connect the input to a broker and maybe the output to a result backend if so wanted. But if you look closely at the back there's a lid revealing loads of sliders, dials and buttons: this is the configuration. From 98673ca473ca6ad892bd4993838e79d7ebcedf82 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 26 Oct 2015 15:58:49 -0700 Subject: [PATCH 0736/1103] Remove settings *_LOG_LEVEL, *_LOG_FILE scheduled for removal in 4.0. --- celery/app/defaults.py | 38 ++++++++----------------------- celery/app/log.py | 2 +- celery/apps/beat.py | 6 ++--- celery/bin/beat.py | 2 +- celery/bin/worker.py | 2 +- celery/tests/app/test_defaults.py | 7 ------ celery/worker/__init__.py | 4 ++-- 7 files changed, 17 insertions(+), 44 deletions(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 2b8753919..87a794d08 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -39,11 +39,6 @@ DEFAULT_TASK_LOG_FMT = """[%(asctime)s: %(levelname)s/%(processName)s] \ %(task_name)s[%(task_id)s]: %(message)s""" -_BROKER_OLD = {'deprecate_by': '2.5', 'remove_by': '4.0', - 'alt': 'BROKER_URL setting'} -_REDIS_OLD = {'deprecate_by': '2.5', 'remove_by': '4.0', - 'alt': 'URL form of CELERY_RESULT_BACKEND'} - searchresult = namedtuple('searchresult', ('namespace', 'key', 'type')) @@ -81,11 +76,11 @@ def __repr__(self): 'USE_SSL': Option(False, type='bool'), 'TRANSPORT': Option(type='string'), 'TRANSPORT_OPTIONS': Option({}, type='dict'), - 'HOST': Option(type='string', **_BROKER_OLD), - 'PORT': Option(type='int', **_BROKER_OLD), - 'USER': Option(type='string', **_BROKER_OLD), - 'PASSWORD': Option(type='string', **_BROKER_OLD), - 'VHOST': Option(type='string', **_BROKER_OLD), + 'HOST': Option(type='string'), + 'PORT': Option(type='int'), + 'USER': Option(type='string'), + 'PASSWORD': Option(type='string'), + 'VHOST': Option(type='string'), }, 'CASSANDRA': { 'COLUMN_FAMILY': Option(type='string'), @@ -129,10 +124,10 @@ def __repr__(self): 'MAX_CACHED_RESULTS': Option(100, type='int'), 'MESSAGE_COMPRESSION': Option(type='string'), 'MONGODB_BACKEND_SETTINGS': Option(type='dict'), - 'REDIS_HOST': Option(type='string', **_REDIS_OLD), - 'REDIS_PORT': Option(type='int', **_REDIS_OLD), - 'REDIS_DB': Option(type='int', **_REDIS_OLD), - 'REDIS_PASSWORD': Option(type='string', **_REDIS_OLD), + 'REDIS_HOST': Option(type='string'), + 'REDIS_PORT': Option(type='int'), + 'REDIS_DB': Option(type='int'), + 'REDIS_PASSWORD': Option(type='string'), 'REDIS_MAX_CONNECTIONS': Option(type='int'), 'REJECT_ON_WORKER_LOST': Option(type='bool'), 'RESULT_BACKEND': Option(type='string'), @@ -183,10 +178,6 @@ def __repr__(self): 'CONSUMER': Option('celery.worker.consumer:Consumer', type='string'), 'LOG_FORMAT': Option(DEFAULT_PROCESS_LOG_FMT), 'LOG_COLOR': Option(type='bool'), - 'LOG_LEVEL': Option('WARN', deprecate_by='2.4', remove_by='4.0', - alt='--loglevel argument'), - 'LOG_FILE': Option(deprecate_by='2.4', remove_by='4.0', - alt='--logfile argument'), 'MAX_TASKS_PER_CHILD': Option(type='int'), 'POOL': Option(DEFAULT_POOL), 'POOL_PUTLOCKS': Option(True, type='bool'), @@ -204,17 +195,6 @@ def __repr__(self): 'SCHEDULE_FILENAME': Option('celerybeat-schedule'), 'SYNC_EVERY': Option(0, type='int'), 'MAX_LOOP_INTERVAL': Option(0, type='float'), - 'LOG_LEVEL': Option('INFO', deprecate_by='2.4', remove_by='4.0', - alt='--loglevel argument'), - 'LOG_FILE': Option(deprecate_by='2.4', remove_by='4.0', - alt='--logfile argument'), - }, - 'CELERYMON': { - 'LOG_LEVEL': Option('INFO', deprecate_by='2.4', remove_by='4.0', - alt='--loglevel argument'), - 'LOG_FILE': Option(deprecate_by='2.4', remove_by='4.0', - alt='--logfile argument'), - 'LOG_FORMAT': Option(DEFAULT_LOG_FMT), }, 'EMAIL': { 'HOST': Option('localhost'), diff --git a/celery/app/log.py b/celery/app/log.py index 372bc1ed6..3f6261b6a 100644 --- a/celery/app/log.py +++ b/celery/app/log.py @@ -58,7 +58,7 @@ class Logging(object): def __init__(self, app): self.app = app - self.loglevel = mlevel(self.app.conf.CELERYD_LOG_LEVEL) + self.loglevel = mlevel(logging.WARN) self.format = self.app.conf.CELERYD_LOG_FORMAT self.task_format = self.app.conf.CELERYD_TASK_LOG_FORMAT self.colorize = self.app.conf.CELERYD_LOG_COLOR diff --git a/celery/apps/beat.py b/celery/apps/beat.py index 3daecd11f..727d7d4f4 100644 --- a/celery/apps/beat.py +++ b/celery/apps/beat.py @@ -43,13 +43,13 @@ class Beat(object): def __init__(self, max_interval=None, app=None, socket_timeout=30, pidfile=None, no_color=None, - loglevel=None, logfile=None, schedule=None, + loglevel='WARN', logfile=None, schedule=None, scheduler_cls=None, redirect_stdouts=None, redirect_stdouts_level=None, **kwargs): """Starts the beat task scheduler.""" self.app = app = app or self.app - self.loglevel = self._getopt('log_level', loglevel) - self.logfile = self._getopt('log_file', logfile) + self.loglevel = loglevel + self.logfile = logfile self.schedule = self._getopt('schedule_filename', schedule) self.scheduler_cls = self._getopt('scheduler', scheduler_cls) self.redirect_stdouts = self._getopt( diff --git a/celery/bin/beat.py b/celery/bin/beat.py index 4bcbc626b..c8041217b 100644 --- a/celery/bin/beat.py +++ b/celery/bin/beat.py @@ -87,7 +87,7 @@ def get_options(self): default=c.CELERYBEAT_SCHEDULE_FILENAME), Option('--max-interval', type='float'), Option('-S', '--scheduler', dest='scheduler_cls'), - Option('-l', '--loglevel', default=c.CELERYBEAT_LOG_LEVEL)) + + Option('-l', '--loglevel', default='WARN')) + daemon_options(default_pidfile='celerybeat.pid') + tuple(self.app.user_options['beat']) ) diff --git a/celery/bin/worker.py b/celery/bin/worker.py index d01be1097..9426baddc 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -226,7 +226,7 @@ def get_options(self): default=conf.CELERYD_CONCURRENCY, type='int'), Option('-P', '--pool', default=conf.CELERYD_POOL, dest='pool_cls'), Option('--purge', '--discard', default=False, action='store_true'), - Option('-l', '--loglevel', default=conf.CELERYD_LOG_LEVEL), + Option('-l', '--loglevel', default='WARN'), Option('-n', '--hostname'), Option('-B', '--beat', action='store_true'), Option('-s', '--schedule', dest='schedule_filename', diff --git a/celery/tests/app/test_defaults.py b/celery/tests/app/test_defaults.py index bf87f80ae..9d0c2071e 100644 --- a/celery/tests/app/test_defaults.py +++ b/celery/tests/app/test_defaults.py @@ -37,13 +37,6 @@ def test_default_pool_pypy_15(self): with pypy_version((1, 5, 0)): self.assertEqual(self.defaults.DEFAULT_POOL, 'prefork') - def test_deprecated(self): - source = Mock() - source.CELERYD_LOG_LEVEL = 2 - with patch('celery.utils.warn_deprecated') as warn: - self.defaults.find_deprecated_settings(source) - self.assertTrue(warn.called) - def test_default_pool_jython(self): with sys_platform('java 1.6.51'): self.assertEqual(self.defaults.DEFAULT_POOL, 'threads') diff --git a/celery/worker/__init__.py b/celery/worker/__init__.py index 1b86fd813..444dab4d3 100644 --- a/celery/worker/__init__.py +++ b/celery/worker/__init__.py @@ -352,9 +352,9 @@ def setup_defaults(self, concurrency=None, loglevel=None, logfile=None, task_time_limit=None, task_soft_time_limit=None, max_tasks_per_child=None, prefetch_multiplier=None, disable_rate_limits=None, worker_lost_wait=None, **_kw): + self.loglevel = loglevel + self.logfile = logfile self.concurrency = self._getopt('concurrency', concurrency) - self.loglevel = self._getopt('log_level', loglevel) - self.logfile = self._getopt('log_file', logfile) self.send_events = self._getopt('send_events', send_events) self.pool_cls = self._getopt('pool', pool_cls) self.consumer_cls = self._getopt('consumer', consumer_cls) From de219813e86a9a4850f0723953ae1201f3a184c4 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 26 Oct 2015 22:05:22 -0700 Subject: [PATCH 0737/1103] Fixes MongoDB tests --- celery/tests/backends/test_mongodb.py | 34 +-------------------------- 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index 595fbaa23..1ade2e8f5 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -381,39 +381,7 @@ def test_get_database_authfailure(self): x._get_database() db.authenticate.assert_called_with('jerry', 'cere4l') - @patch('celery.backends.mongodb.detect_environment') - def test_prepare_client_options_for_ver_2(self, m_detect_env): - m_detect_env.return_value = 'default' - with patch('pymongo.version_tuple', new=(2, 6, 3)): - options = self.backend._prepare_client_options() - self.assertDictEqual(options, { - 'max_pool_size': self.backend.max_pool_size, - 'auto_start_request': False - }) - - @patch('celery.backends.mongodb.detect_environment') - def test_prepare_client_options_for_ver_2_with_gevent(self, m_detect_env): - m_detect_env.return_value = 'gevent' - with patch('pymongo.version_tuple', new=(2, 6, 3)): - options = self.backend._prepare_client_options() - self.assertDictEqual(options, { - 'max_pool_size': self.backend.max_pool_size, - 'auto_start_request': False, - 'use_greenlets': True - }) - - @patch('celery.backends.mongodb.detect_environment') - def test_prepare_client_options_for_ver_3(self, m_detect_env): - m_detect_env.return_value = 'default' - with patch('pymongo.version_tuple', new=(3, 0, 3)): - options = self.backend._prepare_client_options() - self.assertDictEqual(options, { - 'maxPoolSize': self.backend.max_pool_size - }) - - @patch('celery.backends.mongodb.detect_environment') - def test_prepare_client_options_for_ver_3_with_gevent(self, m_detect_env): - m_detect_env.return_value = 'gevent' + def test_prepare_client_options(self): with patch('pymongo.version_tuple', new=(3, 0, 3)): options = self.backend._prepare_client_options() self.assertDictEqual(options, { From 35b99e4ad17a71fde29d67155bd00e0621041dcd Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 26 Oct 2015 22:05:45 -0700 Subject: [PATCH 0738/1103] Use kombu.Connection.supports_exchange_type (requires kombu master) --- celery/worker/consumer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index c10d576ba..a5bb52013 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -686,7 +686,7 @@ def __init__(self, c, **kwargs): def include_if(self, c): return (c.app.conf.CELERY_ENABLE_REMOTE_CONTROL and - 'fanout' in c.conninfo.transport.implements.exchange_type) + c.conninfo.supports_exchange_type('fanout')) class Gossip(bootsteps.ConsumerStep): From 8fb23c6a92a2ba4d4a972d13b64fc12bf75e1924 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 26 Oct 2015 22:12:49 -0700 Subject: [PATCH 0739/1103] Attempt to fix Py3 tests --- celery/worker/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/worker/__init__.py b/celery/worker/__init__.py index 444dab4d3..cf9106112 100644 --- a/celery/worker/__init__.py +++ b/celery/worker/__init__.py @@ -342,7 +342,7 @@ def __str__(self): def state(self): return state - def setup_defaults(self, concurrency=None, loglevel=None, logfile=None, + def setup_defaults(self, concurrency=None, loglevel='WARN', logfile=None, send_events=None, pool_cls=None, consumer_cls=None, timer_cls=None, timer_precision=None, autoscaler_cls=None, autoreloader_cls=None, From 5ed905723aebc722ec25cf01f5b9185674965bbf Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 26 Oct 2015 22:43:14 -0700 Subject: [PATCH 0740/1103] Fixes setup.py version check for PyPy3 --- setup.py | 46 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index aeec75afd..a414b4b4f 100644 --- a/setup.py +++ b/setup.py @@ -8,12 +8,46 @@ import sys import codecs +try: + import platform + _pyimp = platform.python_implementation +except (AttributeError, ImportError): + def _pyimp(): + return 'Python' + +E_UNSUPPORTED_PYTHON = """ +---------------------------------------- + Celery 4.0 requires %s %s or later! +---------------------------------------- + +- For CPython 2.6, PyPy 1.x, Jython 2.6, CPython 3.2->3.3; use Celery 3.1: + + $ pip install 'celery<4' + +- For CPython 2.5, Jython 2.5; use Celery 3.0: + + $ pip install 'celery<3.1' + +- For CPython 2.4; use Celery 2.2: + + $ pip install 'celery<2.3' +""" + +PYIMP = _pyimp() +PY26_OR_LESS = sys.version_info < (2, 7) +PY3 = sys.version_info[0] == 3 +PY33_OR_LESS = PY3 and sys.version_info < (3, 4) +JYTHON = sys.platform.startswith('java') +PYPY_VERSION = getattr(sys, 'pypy_version_info', None) +PYPY = PYPY_VERSION is not None +PYPY24_ATLEAST = PYPY_VERSION and PYPY_VERSION >= (2, 4) + CELERY_COMPAT_PROGRAMS = int(os.environ.get('CELERY_COMPAT_PROGRAMS', 1)) -if sys.version_info < (2, 7): - raise Exception('Celery 4.0 requires Python 2.7 or higher.') -elif sys.version_info > (3, ) and sys.version_info < (3, 4): - raise Exception('Celery 4.0 requires Python 3.4 or higher.') +if PY26_OR_LESS: + raise Exception(E_UNSUPPORTED_PYTHON % (PYIMP, '2.7')) +elif PY33_OR_LESS and not PYPY24_ATLEAST: + raise Exception(E_UNSUPPORTED_PYTHON % (PYIMP, '3.4')) # -*- Upgrading from older versions -*- @@ -48,10 +82,6 @@ finally: sys.path[:] = orig_path -PY3 = sys.version_info[0] == 3 -JYTHON = sys.platform.startswith('java') -PYPY = hasattr(sys, 'pypy_version_info') - NAME = 'celery' entrypoints = {} extra = {} From 9836f1841feba1c0a080d469163e7304f5bf069a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 27 Oct 2015 10:47:37 -0700 Subject: [PATCH 0741/1103] Fixes broken chords in master. Closes #2885 --- celery/backends/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 4b7ae24d4..e03432f30 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -118,7 +118,7 @@ def mark_as_done(self, task_id, result, if store_result: self.store_result(task_id, result, status=state, request=request) if request and request.chord: - self.on_chord_part_return(request, state) + self.on_chord_part_return(request, state, result) def mark_as_failure(self, task_id, exc, traceback=None, request=None, store_result=True, From 5cae0e754128750a893524dcba4ae030c414de33 Mon Sep 17 00:00:00 2001 From: Dave Smith Date: Tue, 11 Nov 2014 15:00:36 -0700 Subject: [PATCH 0742/1103] Adds the CELERYD_MAX_MEMORY_PER_CHILD setting This allows users to specify the maximum amount of resident memory that may be consumed by a child process before it will be replaced by a new child process. If a single task causes a child process to exceed this limit, the task will be completed and the child process will be replaced afterwards. This commit depends on the corresponding commit in the billiard project that enables this setting. --- celery/app/defaults.py | 1 + celery/bin/worker.py | 10 ++++++++++ celery/tests/worker/test_components.py | 12 ++++++++++++ celery/worker/__init__.py | 6 +++++- celery/worker/components.py | 1 + docs/configuration.rst | 11 +++++++++++ docs/userguide/workers.rst | 16 ++++++++++++++++ 7 files changed, 56 insertions(+), 1 deletion(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 87a794d08..e64716269 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -179,6 +179,7 @@ def __repr__(self): 'LOG_FORMAT': Option(DEFAULT_PROCESS_LOG_FMT), 'LOG_COLOR': Option(type='bool'), 'MAX_TASKS_PER_CHILD': Option(type='int'), + 'MAX_MEMORY_PER_CHILD': Option(type='int'), 'POOL': Option(DEFAULT_POOL), 'POOL_PUTLOCKS': Option(True, type='bool'), 'POOL_RESTARTS': Option(False, type='bool'), diff --git a/celery/bin/worker.py b/celery/bin/worker.py index 9426baddc..b3492cb0c 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -109,6 +109,14 @@ Maximum number of tasks a pool worker can execute before it's terminated and replaced by a new worker. +.. cmdoption:: --maxmemperchild + + Maximum amount of resident memory, in KiB, that may be consumed by a + child process before it will be replaced by a new one. If a single + task causes a child process to exceed this limit, the task will be + completed and the child process will be replaced afterwards. + Default: no limit. + .. cmdoption:: --pidfile Optional file used to store the workers pid. @@ -244,6 +252,8 @@ def get_options(self): default=conf.CELERYD_MAX_TASKS_PER_CHILD, type='int'), Option('--prefetch-multiplier', dest='prefetch_multiplier', default=conf.CELERYD_PREFETCH_MULTIPLIER, type='int'), + Option('--maxmemperchild', dest='max_memory_per_child', + default=conf.CELERYD_MAX_MEMORY_PER_CHILD, type='int'), Option('--queues', '-Q', default=[]), Option('--exclude-queues', '-X', default=[]), Option('--include', '-I', default=[]), diff --git a/celery/tests/worker/test_components.py b/celery/tests/worker/test_components.py index c11d48d8e..4a5f898bf 100644 --- a/celery/tests/worker/test_components.py +++ b/celery/tests/worker/test_components.py @@ -34,3 +34,15 @@ def test_create_when_eventloop(self): w.pool = Mock() comp.create(w) self.assertIs(w.process_task, w._process_task_sem) + + def test_create_calls_instantiate_with_max_memory(self): + w = Mock() + w.use_eventloop = w.pool_putlocks = w.pool_cls.uses_semaphore = True + comp = Pool(w) + comp.instantiate = Mock() + w.max_memory_per_child = 32 + + comp.create(w) + + self.assertEqual( + comp.instantiate.call_args[1]['max_memory_per_child'], 32) diff --git a/celery/worker/__init__.py b/celery/worker/__init__.py index cf9106112..c006c5280 100644 --- a/celery/worker/__init__.py +++ b/celery/worker/__init__.py @@ -351,7 +351,8 @@ def setup_defaults(self, concurrency=None, loglevel='WARN', logfile=None, schedule_filename=None, scheduler_cls=None, task_time_limit=None, task_soft_time_limit=None, max_tasks_per_child=None, prefetch_multiplier=None, - disable_rate_limits=None, worker_lost_wait=None, **_kw): + disable_rate_limits=None, worker_lost_wait=None, + max_memory_per_child=None, **_kw): self.loglevel = loglevel self.logfile = logfile self.concurrency = self._getopt('concurrency', concurrency) @@ -381,6 +382,9 @@ def setup_defaults(self, concurrency=None, loglevel='WARN', logfile=None, self.max_tasks_per_child = self._getopt( 'max_tasks_per_child', max_tasks_per_child, ) + self.max_memory_per_child = self._getopt( + 'max_memory_per_child', max_memory_per_child, + ) self.prefetch_multiplier = int(self._getopt( 'prefetch_multiplier', prefetch_multiplier, )) diff --git a/celery/worker/components.py b/celery/worker/components.py index d3f219da1..2c09156ff 100644 --- a/celery/worker/components.py +++ b/celery/worker/components.py @@ -156,6 +156,7 @@ def create(self, w, semaphore=None, max_restarts=None): w.pool_cls, w.min_concurrency, initargs=(w.app, w.hostname), maxtasksperchild=w.max_tasks_per_child, + max_memory_per_child=w.max_memory_per_child, timeout=w.task_time_limit, soft_timeout=w.task_soft_time_limit, putlocks=w.pool_putlocks and threaded, diff --git a/docs/configuration.rst b/docs/configuration.rst index dcb8ab4f6..0b48c3080 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1665,6 +1665,17 @@ CELERYD_MAX_TASKS_PER_CHILD Maximum number of tasks a pool worker process can execute before it's replaced with a new one. Default is no limit. +.. setting:: CELERYD_MAX_MEMORY_PER_CHILD + +CELERYD_MAX_MEMORY_PER_CHILD +~~~~~~~~~~~~~~~~~~~~~ + +Maximum amount of resident memory that may be consumed by a +worker before it will be replaced by a new worker. If a single +task causes a worker to exceed this limit, the task will be +completed, and the worker will be replaced afterwards. Default: +no limit. + .. setting:: CELERYD_TASK_TIME_LIMIT CELERYD_TASK_TIME_LIMIT diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index b12852a8d..d9332b2c9 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -528,6 +528,22 @@ for example from closed source C extensions. The option can be set using the workers `--maxtasksperchild` argument or using the :setting:`CELERYD_MAX_TASKS_PER_CHILD` setting. +Max memory per child setting +============================ + +.. versionadded:: TODO + +pool support: *prefork* + +With this option you can configure the maximum amount of resident +memory a worker can execute before it's replaced by a new process. + +This is useful if you have memory leaks you have no control over +for example from closed source C extensions. + +The option can be set using the workers `--maxmemperchild` argument +or using the :setting:`CELERYD_MAX_MEMORY_PER_CHILD` setting. + .. _worker-autoscaling: Autoscaling From 7053f79bba62054e4aa3e92030e5caa6f306ba21 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 27 Oct 2015 13:18:02 -0700 Subject: [PATCH 0743/1103] flakes --- celery/tests/app/test_defaults.py | 2 +- celery/worker/request.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/celery/tests/app/test_defaults.py b/celery/tests/app/test_defaults.py index 9d0c2071e..61dd4ba33 100644 --- a/celery/tests/app/test_defaults.py +++ b/celery/tests/app/test_defaults.py @@ -7,7 +7,7 @@ from celery.app.defaults import NAMESPACES from celery.tests.case import ( - AppCase, Mock, patch, pypy_version, sys_platform, + AppCase, pypy_version, sys_platform, ) diff --git a/celery/worker/request.py b/celery/worker/request.py index 3a57b16ec..c47ae81d5 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -356,8 +356,10 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): # (acks_late) acknowledge after result stored. if self.task.acks_late: requeue = self.delivery_info.get('redelivered', None) is False - reject = (self.task.reject_on_worker_lost and - isinstance(exc, WorkerLostError)) + reject = ( + self.task.reject_on_worker_lost and + isinstance(exc, WorkerLostError) + ) if reject: self.reject(requeue=requeue) else: From 41ac67b33f1b0e7f2f28b2f9dfd5779c2a633972 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 27 Oct 2015 13:19:02 -0700 Subject: [PATCH 0744/1103] Removes CELERYMON* settings --- docs/configuration.rst | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 0b48c3080..8373b2ecd 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -2215,22 +2215,3 @@ changes to the schedule into account. Also when running celery beat embedded (:option:`-B`) on Jython as a thread the max interval is overridden and set to 1 so that it's possible to shut down in a timely manner. - - -.. _conf-celerymon: - -Monitor Server: celerymon -------------------------- - - -.. setting:: CELERYMON_LOG_FORMAT - -CELERYMON_LOG_FORMAT -~~~~~~~~~~~~~~~~~~~~ - -The format to use for log messages. - -Default is `[%(asctime)s: %(levelname)s/%(processName)s] %(message)s` - -See the Python :mod:`logging` module for more information about log -formats. From baf8f4df5dca9bf4ee2a9fb1395f425fc10123e0 Mon Sep 17 00:00:00 2001 From: Dave Smith Date: Tue, 27 Oct 2015 15:20:56 -0600 Subject: [PATCH 0745/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index bfc00f31b..b62f1915c 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -196,3 +196,4 @@ Piotr Maślanka, 2015/08/24 Gerald Manipon, 2015/10/19 Krzysztof Bujniewicz, 2015/10/21 Sukrit Khera, 2015/10/26 +Dave Smith, 2015/10/27 From 149de1291e5bd67618d3b1e0c36534192a430ce3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 27 Oct 2015 23:00:07 -0700 Subject: [PATCH 0746/1103] Removes compat programs celeryd, celerybeat and celeryd-multi (scheduled for removal in 4.0) --- celery/__main__.py | 36 ------------------------------------ setup.py | 9 --------- 2 files changed, 45 deletions(-) diff --git a/celery/__main__.py b/celery/__main__.py index 572f7c3c9..590c94766 100644 --- a/celery/__main__.py +++ b/celery/__main__.py @@ -2,26 +2,10 @@ import sys -from os.path import basename - from . import maybe_patch_concurrency __all__ = ['main'] -DEPRECATED_FMT = """ -The {old!r} command is deprecated, please use {new!r} instead: - -$ {new_argv} - -""" - - -def _warn_deprecated(new): - print(DEPRECATED_FMT.format( - old=basename(sys.argv[0]), new=new, - new_argv=' '.join([new] + sys.argv[1:])), - ) - def main(): if 'multi' not in sys.argv: @@ -30,25 +14,5 @@ def main(): main() -def _compat_worker(): - maybe_patch_concurrency() - _warn_deprecated('celery worker') - from celery.bin.worker import main - main() - - -def _compat_multi(): - _warn_deprecated('celery multi') - from celery.bin.multi import main - main() - - -def _compat_beat(): - maybe_patch_concurrency() - _warn_deprecated('celery beat') - from celery.bin.beat import main - main() - - if __name__ == '__main__': # pragma: no cover main() diff --git a/setup.py b/setup.py index a414b4b4f..4a9d9679b 100644 --- a/setup.py +++ b/setup.py @@ -42,8 +42,6 @@ def _pyimp(): PYPY = PYPY_VERSION is not None PYPY24_ATLEAST = PYPY_VERSION and PYPY_VERSION >= (2, 4) -CELERY_COMPAT_PROGRAMS = int(os.environ.get('CELERY_COMPAT_PROGRAMS', 1)) - if PY26_OR_LESS: raise Exception(E_UNSUPPORTED_PYTHON % (PYIMP, '2.7')) elif PY33_OR_LESS and not PYPY24_ATLEAST: @@ -190,13 +188,6 @@ def reqs(*f): 'celery = celery.__main__:main', ] -if CELERY_COMPAT_PROGRAMS: - console_scripts.extend([ - 'celeryd = celery.__main__:_compat_worker', - 'celerybeat = celery.__main__:_compat_beat', - 'celeryd-multi = celery.__main__:_compat_multi', - ]) - # -*- Extras -*- From a43653a15c5810c449acb168b24e010db82c3b36 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 28 Oct 2015 11:35:34 -0700 Subject: [PATCH 0747/1103] Worker cannot drain without timeout at startup --- celery/worker/loops.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/celery/worker/loops.py b/celery/worker/loops.py index 223c15378..8dcc9be62 100644 --- a/celery/worker/loops.py +++ b/celery/worker/loops.py @@ -7,6 +7,7 @@ """ from __future__ import absolute_import +import errno import socket from celery.bootsteps import RUN @@ -21,6 +22,15 @@ error = logger.error +def _quick_drain(connection, timeout=0.1): + try: + connection.drain_events(timeout=timeout) + except Exception as exc: + exc_errno = getattr(exc, 'errno', None) + if exc_errno is not None and exc_errno != errno.EAGAIN: + raise + + def asynloop(obj, connection, consumer, blueprint, hub, qos, heartbeat, clock, hbrate=2.0, RUN=RUN): """Non-blocking event loop consuming messages until connection is lost, @@ -51,7 +61,7 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos, # limit - drain an event so we are in a clean state # prior to starting our event loop. if connection.transport.driver_type == 'amqp': - hub.call_soon(connection.drain_events) + hub.call_soon(_quick_drain, connection) # FIXME: Use loop.run_forever # Tried and works, but no time to test properly before release. From 5101fe63911cf532ab91653539bd315edcd8b3c5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 28 Oct 2015 11:36:20 -0700 Subject: [PATCH 0748/1103] Command line arguments may be in the form of '--key value' --- celery/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/celery/__init__.py b/celery/__init__.py index d94678441..10844a7b8 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -68,18 +68,18 @@ def debug_import(name, locals=None, globals=None, def _find_option_with_arg(argv, short_opts=None, long_opts=None): - """Search argv for option specifying its short and longopt - alternatives. + """Search argv for options specifying short and longopt alternatives. - Return the value of the option if found. + :returns: value for option found + :raises KeyError: if option not found. """ for i, arg in enumerate(argv): if arg.startswith('-'): if long_opts and arg.startswith('--'): - name, _, val = arg.partition('=') + name, sep, val = arg.partition('=') if name in long_opts: - return val + return val if sep else argv[i + 1] if short_opts and arg in short_opts: return argv[i + 1] raise KeyError('|'.join(short_opts or [] + long_opts or [])) From a6b7aca6719c262c76e6b844cb137298a43b837d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 28 Oct 2015 11:36:59 -0700 Subject: [PATCH 0749/1103] Cosmetics --- celery/__init__.py | 33 +++++++++++++++++++-------------- celery/_state.py | 8 +++++--- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/celery/__init__.py b/celery/__init__.py index 10844a7b8..9c189493f 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -19,27 +19,30 @@ SERIES = '0today8' VERSION = version_info_t(4, 0, 0, 'a1', '') + __version__ = '{0.major}.{0.minor}.{0.micro}{0.releaselevel}'.format(VERSION) __author__ = 'Ask Solem' __contact__ = 'ask@celeryproject.org' __homepage__ = 'http://celeryproject.org' __docformat__ = 'restructuredtext' + +# -eof meta- + __all__ = [ 'Celery', 'bugreport', 'shared_task', 'task', 'current_app', 'current_task', 'maybe_signature', 'chain', 'chord', 'chunks', 'group', 'signature', 'xmap', 'xstarmap', 'uuid', 'version', '__version__', ] + VERSION_BANNER = '{0} ({1})'.format(__version__, SERIES) -# -eof meta- if os.environ.get('C_IMPDEBUG'): # pragma: no cover from .five import builtins - real_import = builtins.__import__ def debug_import(name, locals=None, globals=None, - fromlist=None, level=-1): + fromlist=None, level=-1, real_import=builtins.__import__): glob = globals or getattr(sys, 'emarfteg_'[::-1])(1).f_globals importer_name = glob and glob.get('__name__') or 'unknown' print('-- {0} imports {1}'.format(importer_name, name)) @@ -88,21 +91,20 @@ def _find_option_with_arg(argv, short_opts=None, long_opts=None): def _patch_eventlet(): import eventlet import eventlet.debug - eventlet.monkey_patch() - EVENTLET_DBLOCK = int(os.environ.get('EVENTLET_NOBLOCK', 0)) - if EVENTLET_DBLOCK: - eventlet.debug.hub_blocking_detection(EVENTLET_DBLOCK) + eventlet.monkey_patch() + blockdetect = float(os.environ.get('EVENTLET_NOBLOCK', 0)) + eventlet.debug.hub_blocking_detection(blockdetect, blockdetect) def _patch_gevent(): - from gevent import monkey, version_info + from gevent import monkey, signal as gsignal, version_info + monkey.patch_all() if version_info[0] == 0: # pragma: no cover # Signals aren't working in gevent versions <1.0, # and are not monkey patched by patch_all() - from gevent import signal as _gevent_signal _signal = __import__('signal') - _signal.signal = _gevent_signal + _signal.signal = gsignal def maybe_patch_concurrency(argv=sys.argv, @@ -124,7 +126,8 @@ def maybe_patch_concurrency(argv=sys.argv, pass else: patcher() - # set up eventlet/gevent environments ASAP. + + # set up eventlet/gevent environments ASAP from celery import concurrency concurrency.get_implementation(pool) @@ -137,9 +140,11 @@ def maybe_patch_concurrency(argv=sys.argv, 'celery.app': ['Celery', 'bugreport', 'shared_task'], 'celery.app.task': ['Task'], 'celery._state': ['current_app', 'current_task'], - 'celery.canvas': ['chain', 'chord', 'chunks', 'group', - 'signature', 'maybe_signature', 'subtask', - 'xmap', 'xstarmap'], + 'celery.canvas': [ + 'chain', 'chord', 'chunks', 'group', + 'signature', 'maybe_signature', 'subtask', + 'xmap', 'xstarmap', + ], 'celery.utils': ['uuid'], }, direct={'task': 'celery.task'}, diff --git a/celery/_state.py b/celery/_state.py index 9ed62b89d..1fec88973 100644 --- a/celery/_state.py +++ b/celery/_state.py @@ -19,9 +19,11 @@ from celery.local import Proxy from celery.utils.threads import LocalStack -__all__ = ['set_default_app', 'get_current_app', 'get_current_task', - 'get_current_worker_task', 'current_app', 'current_task', - 'connect_on_app_finalize'] +__all__ = [ + 'set_default_app', 'get_current_app', 'get_current_task', + 'get_current_worker_task', 'current_app', 'current_task', + 'connect_on_app_finalize', +] #: Global default app used when no current app. default_app = None From 6ccffeba37091c664e74ec3ca84ee37a5b9fb1f6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 28 Oct 2015 11:37:43 -0700 Subject: [PATCH 0750/1103] Prefork: _timeout_handler can be None in billiard master --- celery/concurrency/asynpool.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 76a5c8da4..9aa819274 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -43,6 +43,7 @@ from kombu.utils import fxrange from kombu.utils.eventio import SELECT_BAD_FD from celery.five import Counter, items, values +from celery.utils.functional import noop from celery.utils.log import get_logger from celery.worker import state as worker_state @@ -417,8 +418,13 @@ def __init__(self, processes=None, synack=False, # as processes are recycled, or found lost elsewhere. self._fileno_to_outq[proc.outqR_fd] = proc self._fileno_to_synq[proc.synqW_fd] = proc - self.on_soft_timeout = self._timeout_handler.on_soft_timeout - self.on_hard_timeout = self._timeout_handler.on_hard_timeout + + self.on_soft_timeout = getattr( + self._timeout_handler, 'on_soft_timeout', noop, + ) + self.on_hard_timeout = getattr( + self._timeout_handler, 'on_hard_timeout', noop, + ) def _event_process_exit(self, hub, fd): # This method is called whenever the process sentinel is readable. From fe793b2074bf508c02f495f9c8c5fea55ec4ed82 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 28 Oct 2015 12:10:11 -0700 Subject: [PATCH 0751/1103] RabbitMQ supports priorities now. Closes #2835 --- docs/faq.rst | 14 ++++++++------ docs/getting-started/next-steps.rst | 2 +- docs/userguide/calling.rst | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 4ca99c601..7efb678d5 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -746,13 +746,15 @@ create a new schedule subclass and override Does celery support task priorities? ------------------------------------ -**Answer**: No. In theory, yes, as AMQP supports priorities. However -RabbitMQ doesn't implement them yet. +**Answer**: Yes. -The usual way to prioritize work in Celery, is to route high priority tasks -to different servers. In the real world this may actually work better than per message -priorities. You can use this in combination with rate limiting to achieve a -highly responsive system. +RabbitMQ supports priorities since version 3.5.0. +Redis transport emulates support of priorities. + +You can also prioritize work by routing high priority tasks +to different workers. In the real world this may actually work better +than per message priorities. You can use this in combination with rate +limiting to achieve a responsive system. .. _faq-acks_late-vs-retry: diff --git a/docs/getting-started/next-steps.rst b/docs/getting-started/next-steps.rst index 1cf98eb5b..981b096a5 100644 --- a/docs/getting-started/next-steps.rst +++ b/docs/getting-started/next-steps.rst @@ -122,7 +122,7 @@ the :ref:`Monitoring and Management guide `. tasks from. The worker can be told to consume from several queues at once, and this is used to route messages to specific workers as a means for Quality of Service, separation of concerns, -and emulating priorities, all described in the :ref:`Routing Guide +and prioritization, all described in the :ref:`Routing Guide `. You can get a complete list of command-line arguments diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index 8042379e3..e33e2aa9d 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -497,6 +497,6 @@ AMQP's full routing capabilities. Interested parties may read the - priority - A number between `0` and `9`, where `0` is the highest priority. + A number between `0` and `255`, where `255` is the highest priority. - Supported by: redis, beanstalk + Supported by: rabbitmq, redis (priority reversed, 0 is highest), beanstalk From ef107f05e042876efaab7c75f71262727a8387e3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 28 Oct 2015 13:41:26 -0700 Subject: [PATCH 0752/1103] Documentation now licensed under the CC BY-SA 4.0 license (Issue #2890) --- LICENSE | 6 +++--- docs/copyright.rst | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/LICENSE b/LICENSE index 736d82a97..92a530c9b 100644 --- a/LICENSE +++ b/LICENSE @@ -40,9 +40,9 @@ Documentation License The documentation portion of Celery (the rendered contents of the "docs" directory of a software distribution or checkout) is supplied -under the Creative Commons Attribution-Noncommercial-Share Alike 3.0 -United States License as described by -http://creativecommons.org/licenses/by-nc-sa/3.0/us/ +under the "Creative Commons Attribution-ShareAlike 4.0 +International" (CC BY-SA 4.0) License as described by +http://creativecommons.org/licenses/by-sa/4.0/ Footnotes ========= diff --git a/docs/copyright.rst b/docs/copyright.rst index cf2885186..a81d5cb8d 100644 --- a/docs/copyright.rst +++ b/docs/copyright.rst @@ -11,17 +11,18 @@ Copyright |copy| 2009-2015, Ask Solem. All rights reserved. This material may be copied or distributed only subject to the terms and conditions set forth in the `Creative Commons -Attribution-Noncommercial-Share Alike 3.0 United States License -`_. You must -give the original author credit. You may not use this work for -commercial purposes. If you alter, transform, or build upon this -work, you may distribute the resulting work only under the same or -similar license to this one. +Attribution-ShareAlike 4.0 International` +`_ license. + +You may share and adapt the material, even for commercial purposes, but +you must give the original author credit. +If you alter, transform, or build upon this +work, you may distribute the resulting work only under the same license or +a license compatible to this one. .. note:: While the *Celery* documentation is offered under the - Creative Commons *attribution-nonconmmercial-share alike 3.0 united - states* license, the Celery *software* is offered under the - less restrictive + Creative Commons *Attribution-ShareAlike 4.0 International* license + the Celery *software* is offered under the `BSD License (3 Clause) `_ From 50185a4fc0952f741ca92430ba5c92a654138679 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 28 Oct 2015 15:40:02 -0700 Subject: [PATCH 0753/1103] Fixes tests --- celery/tests/bin/test_celery.py | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/celery/tests/bin/test_celery.py b/celery/tests/bin/test_celery.py index 573810eec..26e5b473a 100644 --- a/celery/tests/bin/test_celery.py +++ b/celery/tests/bin/test_celery.py @@ -36,12 +36,6 @@ class test__main__(AppCase): - def test_warn_deprecated(self): - with override_stdouts() as (stdout, _): - __main__._warn_deprecated('YADDA YADDA') - self.assertIn('command is deprecated', stdout.getvalue()) - self.assertIn('YADDA YADDA', stdout.getvalue()) - def test_main(self): with patch('celery.__main__.maybe_patch_concurrency') as mpc: with patch('celery.bin.celery.main') as main: @@ -49,33 +43,6 @@ def test_main(self): mpc.assert_called_with() main.assert_called_with() - def test_compat_worker(self): - with patch('celery.__main__.maybe_patch_concurrency') as mpc: - with patch('celery.__main__._warn_deprecated') as depr: - with patch('celery.bin.worker.main') as main: - __main__._compat_worker() - mpc.assert_called_with() - depr.assert_called_with('celery worker') - main.assert_called_with() - - def test_compat_multi(self): - with patch('celery.__main__.maybe_patch_concurrency') as mpc: - with patch('celery.__main__._warn_deprecated') as depr: - with patch('celery.bin.multi.main') as main: - __main__._compat_multi() - self.assertFalse(mpc.called) - depr.assert_called_with('celery multi') - main.assert_called_with() - - def test_compat_beat(self): - with patch('celery.__main__.maybe_patch_concurrency') as mpc: - with patch('celery.__main__._warn_deprecated') as depr: - with patch('celery.bin.beat.main') as main: - __main__._compat_beat() - mpc.assert_called_with() - depr.assert_called_with('celery beat') - main.assert_called_with() - class test_Command(AppCase): From 4d998d17d964264d8da04755b91c845931479ffe Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 Oct 2015 11:23:02 -0700 Subject: [PATCH 0754/1103] Fixes tests --- celery/__init__.py | 3 ++- celery/tests/worker/test_loops.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/celery/__init__.py b/celery/__init__.py index 9c189493f..84f3fa6de 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -94,7 +94,8 @@ def _patch_eventlet(): eventlet.monkey_patch() blockdetect = float(os.environ.get('EVENTLET_NOBLOCK', 0)) - eventlet.debug.hub_blocking_detection(blockdetect, blockdetect) + if blockdetect: + eventlet.debug.hub_blocking_detection(blockdetect, blockdetect) def _patch_gevent(): from gevent import monkey, signal as gsignal, version_info diff --git a/celery/tests/worker/test_loops.py b/celery/tests/worker/test_loops.py index f70ccf41b..306a61c7e 100644 --- a/celery/tests/worker/test_loops.py +++ b/celery/tests/worker/test_loops.py @@ -10,7 +10,7 @@ from celery.platforms import EX_FAILURE from celery.worker import state from celery.worker.consumer import Consumer -from celery.worker.loops import asynloop, synloop +from celery.worker.loops import _quick_drain, asynloop, synloop from celery.tests.case import AppCase, Mock, task_message_from_sig @@ -126,7 +126,7 @@ def add(x, y): def test_drain_after_consume(self): x, _ = get_task_callback(self.app, transport_driver_type='amqp') self.assertIn( - x.connection.drain_events, [p.fun for p in x.hub._ready], + _quick_drain, [p.fun for p in x.hub._ready], ) def test_setup_heartbeat(self): From a7d3fcfcc438000e576f79c494526e6dd1270927 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 Oct 2015 11:25:36 -0700 Subject: [PATCH 0755/1103] flakes --- celery/__init__.py | 1 + celery/tests/bin/test_celery.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/celery/__init__.py b/celery/__init__.py index 84f3fa6de..48ac71dd7 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -97,6 +97,7 @@ def _patch_eventlet(): if blockdetect: eventlet.debug.hub_blocking_detection(blockdetect, blockdetect) + def _patch_gevent(): from gevent import monkey, signal as gsignal, version_info diff --git a/celery/tests/bin/test_celery.py b/celery/tests/bin/test_celery.py index 26e5b473a..4139750a2 100644 --- a/celery/tests/bin/test_celery.py +++ b/celery/tests/bin/test_celery.py @@ -29,9 +29,7 @@ command, ) -from celery.tests.case import ( - AppCase, Mock, WhateverIO, override_stdouts, patch, -) +from celery.tests.case import AppCase, Mock, WhateverIO, patch class test__main__(AppCase): From 53b5fdf3c504ca667ffc8d606d2c6d6fa6f21cf6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 26 Oct 2015 18:17:07 -0700 Subject: [PATCH 0756/1103] Lowercase settings and settings cleanup (radical, but backwards compatible) All settings are now in lowercase, and most of them have been renamed. When loading settings the loader will look at the settings in the config and decide if it's using old or new settings. The settings will autmatically convert between old and new settings keys, depending on the format the settings is in. - It's not legal to mix new setting names and old setting names, that is unless the setting have two alternatives (old and new). An ImproperlyConfigured exceptions is rasised in this case, with help telling user exactly how to fix the problem. - To support loading configuration from Django settings a new ``namespace`` argument has been added to ``Celery`` and ``config_from_object``. This can be used from Django:: app = Celery() app.config_from_object('django.conf:settings', namespace='CELERY_') # settings.py: CELERY_BROKER_URL = 'amqp://' CELERY_TASK_PROTOCOL = 2 CELERY_TASK_ALWAYS_EAGER = True Or other apps wanting a prefix for some reason:: app = Celery(namespace='celery_') app.conf.celery_task_always_eager = True app.conf.celery_task_routes = {'proj.tasks.add': 'math.yo'} - Initial configuration directly on the app object is now lazy! You can set keys on an unfinalized app, without causing the tasks or the rest of the app to be evaluated: app = Celery() app.conf.update( task_default_delivery_mode=1, task_default_queue='default', task_default_exchange='default', task_default_routing_key='default', ) app.conf.task_always_eager = True assert not app.configured # <-- still not finalized app.config_from_object('celeryconfig') assert not app.configured # <-- even now app.finalize() assert app.finalized # <-- but now we are # and the config done first remains, unlike older versions of Celery. assert app.conf.task.default_queue == 'default' app.config_from_object(object()) # but calling config_from_* again will reset everything. assert app.conf.task_default_queue == 'celery' - ``config_from_*`` methods no longer override configuration set manually before the app was finalized. But calling again after the app is finalized, will clean out old configuration. --- celery/app/amqp.py | 42 +- celery/app/annotations.py | 4 +- celery/app/base.py | 184 ++- celery/app/builtins.py | 2 +- celery/app/defaults.py | 440 +++-- celery/app/log.py | 8 +- celery/app/routes.py | 8 +- celery/app/task.py | 49 +- celery/app/trace.py | 4 +- celery/app/utils.py | 135 +- celery/apps/beat.py | 18 +- celery/apps/worker.py | 17 +- celery/backends/amqp.py | 6 +- celery/backends/base.py | 12 +- celery/backends/cache.py | 4 +- celery/backends/cassandra.py | 16 +- celery/backends/couchbase.py | 2 +- celery/backends/database/__init__.py | 12 +- celery/backends/mongodb.py | 2 +- celery/backends/new_cassandra.py | 16 +- celery/backends/redis.py | 23 +- celery/backends/riak.py | 2 +- celery/beat.py | 18 +- celery/bin/base.py | 2 +- celery/bin/beat.py | 2 +- celery/bin/celery.py | 1 - celery/bin/graph.py | 2 +- celery/bin/worker.py | 24 +- celery/canvas.py | 8 +- celery/contrib/batches.py | 2 +- celery/contrib/migrate.py | 2 +- celery/datastructures.py | 52 +- celery/events/__init__.py | 10 +- celery/events/cursesmon.py | 2 +- celery/exceptions.py | 4 +- celery/loaders/base.py | 17 +- celery/schedules.py | 4 +- celery/security/__init__.py | 14 +- celery/states.py | 2 +- celery/task/base.py | 10 +- celery/task/sets.py | 2 +- celery/tests/app/test_amqp.py | 6 +- celery/tests/app/test_app.py | 210 ++- celery/tests/app/test_beat.py | 16 +- celery/tests/app/test_builtins.py | 4 +- celery/tests/app/test_defaults.py | 29 +- celery/tests/app/test_loaders.py | 15 +- celery/tests/app/test_log.py | 2 +- celery/tests/app/test_routes.py | 14 +- celery/tests/backends/test_amqp.py | 2 +- celery/tests/backends/test_base.py | 2 +- celery/tests/backends/test_cache.py | 4 +- celery/tests/backends/test_cassandra.py | 12 +- celery/tests/backends/test_couchbase.py | 14 +- celery/tests/backends/test_database.py | 4 +- celery/tests/backends/test_mongodb.py | 8 +- celery/tests/backends/test_new_cassandra.py | 12 +- celery/tests/backends/test_redis.py | 12 +- celery/tests/backends/test_riak.py | 26 +- celery/tests/bin/test_base.py | 6 +- celery/tests/bin/test_worker.py | 8 +- celery/tests/case.py | 22 +- celery/tests/compat_modules/test_http.py | 4 +- celery/tests/compat_modules/test_sets.py | 4 +- celery/tests/events/test_events.py | 2 +- celery/tests/security/test_security.py | 6 +- celery/tests/tasks/test_canvas.py | 4 +- celery/tests/tasks/test_chord.py | 6 +- celery/tests/tasks/test_result.py | 4 +- celery/tests/tasks/test_tasks.py | 4 +- celery/tests/utils/test_datastructures.py | 12 +- celery/tests/worker/test_consumer.py | 17 +- celery/tests/worker/test_control.py | 6 +- celery/tests/worker/test_request.py | 2 +- celery/tests/worker/test_worker.py | 8 +- celery/utils/functional.py | 11 +- celery/worker/__init__.py | 77 +- celery/worker/components.py | 4 +- celery/worker/consumer.py | 20 +- celery/worker/control.py | 2 +- celery/worker/request.py | 2 +- docs/configuration.rst | 1456 ++++++++--------- docs/django/first-steps-with-django.rst | 16 +- docs/faq.rst | 10 +- docs/getting-started/brokers/beanstalk.rst | 2 +- docs/getting-started/brokers/couchdb.rst | 2 +- docs/getting-started/brokers/django.rst | 2 +- docs/getting-started/brokers/ironmq.rst | 2 +- docs/getting-started/brokers/mongodb.rst | 2 +- docs/getting-started/brokers/rabbitmq.rst | 2 +- docs/getting-started/brokers/redis.rst | 14 +- docs/getting-started/brokers/sqlalchemy.rst | 12 +- docs/getting-started/brokers/sqs.rst | 20 +- .../first-steps-with-celery.rst | 38 +- docs/getting-started/next-steps.rst | 10 +- docs/glossary.rst | 2 +- docs/internals/app-overview.rst | 10 +- docs/internals/deprecation.rst | 46 +- docs/userguide/application.rst | 16 +- docs/userguide/calling.rst | 14 +- docs/userguide/canvas.rst | 4 +- docs/userguide/monitoring.rst | 2 +- docs/userguide/optimizing.rst | 18 +- docs/userguide/periodic-tasks.rst | 20 +- docs/userguide/remote-tasks.rst | 2 +- docs/userguide/routing.rst | 75 +- docs/userguide/security.rst | 26 +- docs/userguide/signals.rst | 8 +- docs/userguide/tasks.rst | 36 +- docs/userguide/workers.rst | 20 +- examples/celery_http_gateway/settings.py | 1 - examples/django/proj/celery.py | 2 +- examples/django/proj/settings.py | 2 +- examples/eventlet/celeryconfig.py | 12 +- examples/gevent/celeryconfig.py | 11 +- examples/next-steps/proj/celery.py | 2 +- extra/release/verify_config_reference.py | 27 +- funtests/benchmarks/bench_worker.py | 18 +- funtests/stress/stress/app.py | 2 + funtests/stress/stress/templates.py | 66 +- funtests/suite/config.py | 18 +- 121 files changed, 2060 insertions(+), 1795 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index a5923edd6..b8b5a9e27 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -245,7 +245,7 @@ def __init__(self, app): @cached_property def create_task_message(self): - return self.task_protocols[self.app.conf.CELERY_TASK_PROTOCOL] + return self.task_protocols[self.app.conf.task_protocol] @cached_property def send_task_message(self): @@ -257,15 +257,15 @@ def Queues(self, queues, create_missing=None, ha_policy=None, from the current configuration.""" conf = self.app.conf if create_missing is None: - create_missing = conf.CELERY_CREATE_MISSING_QUEUES + create_missing = conf.task_create_missing_queues if ha_policy is None: - ha_policy = conf.CELERY_QUEUE_HA_POLICY + ha_policy = conf.task_queue_ha_policy if max_priority is None: - max_priority = conf.CELERY_QUEUE_MAX_PRIORITY - if not queues and conf.CELERY_DEFAULT_QUEUE: - queues = (Queue(conf.CELERY_DEFAULT_QUEUE, + max_priority = conf.task_queue_max_priority + if not queues and conf.task_default_queue: + queues = (Queue(conf.task_default_queue, exchange=self.default_exchange, - routing_key=conf.CELERY_DEFAULT_ROUTING_KEY),) + routing_key=conf.task_default_routing_key),) autoexchange = (self.autoexchange if autoexchange is None else autoexchange) return self.queues_cls( @@ -276,15 +276,15 @@ def Queues(self, queues, create_missing=None, ha_policy=None, def Router(self, queues=None, create_missing=None): """Return the current task router.""" return _routes.Router(self.routes, queues or self.queues, - self.app.either('CELERY_CREATE_MISSING_QUEUES', + self.app.either('task_create_missing_queues', create_missing), app=self.app) def flush_routes(self): - self._rtable = _routes.prepare(self.app.conf.CELERY_ROUTES) + self._rtable = _routes.prepare(self.app.conf.task_routes) def TaskConsumer(self, channel, queues=None, accept=None, **kw): if accept is None: - accept = self.app.conf.CELERY_ACCEPT_CONTENT + accept = self.app.conf.accept_content return self.Consumer( channel, accept=accept, queues=queues or list(self.queues.consume_from.values()), @@ -442,9 +442,9 @@ def as_task_v1(self, task_id, name, args=None, kwargs=None, ) def _create_task_sender(self): - default_retry = self.app.conf.CELERY_TASK_PUBLISH_RETRY - default_policy = self.app.conf.CELERY_TASK_PUBLISH_RETRY_POLICY - default_delivery_mode = self.app.conf.CELERY_DEFAULT_DELIVERY_MODE + default_retry = self.app.conf.task_publish_retry + default_policy = self.app.conf.task_publish_retry_policy + default_delivery_mode = self.app.conf.task_default_delivery_mode default_queue = self.default_queue queues = self.queues send_before_publish = signals.before_task_publish.send @@ -458,9 +458,9 @@ def _create_task_sender(self): default_evd = self._event_dispatcher default_exchange = self.default_exchange - default_rkey = self.app.conf.CELERY_DEFAULT_ROUTING_KEY - default_serializer = self.app.conf.CELERY_TASK_SERIALIZER - default_compressor = self.app.conf.CELERY_MESSAGE_COMPRESSION + default_rkey = self.app.conf.task_default_routing_key + default_serializer = self.app.conf.task_serializer + default_compressor = self.app.conf.result_compression def publish_task(producer, name, message, exchange=None, routing_key=None, queue=None, @@ -541,12 +541,12 @@ def publish_task(producer, name, message, @cached_property def default_queue(self): - return self.queues[self.app.conf.CELERY_DEFAULT_QUEUE] + return self.queues[self.app.conf.task_default_queue] @cached_property def queues(self): """Queue name⇒ declaration mapping.""" - return self.Queues(self.app.conf.CELERY_QUEUES) + return self.Queues(self.app.conf.task_queues) @queues.setter # noqa def queues(self, queues): @@ -575,12 +575,12 @@ def producer_pool(self): @cached_property def default_exchange(self): - return Exchange(self.app.conf.CELERY_DEFAULT_EXCHANGE, - self.app.conf.CELERY_DEFAULT_EXCHANGE_TYPE) + return Exchange(self.app.conf.task_default_exchange, + self.app.conf.task_default_exchange_type) @cached_property def utc(self): - return self.app.conf.CELERY_ENABLE_UTC + return self.app.conf.enable_utc @cached_property def _event_dispatcher(self): diff --git a/celery/app/annotations.py b/celery/app/annotations.py index 6535aa81b..9ae1aea70 100644 --- a/celery/app/annotations.py +++ b/celery/app/annotations.py @@ -7,7 +7,7 @@ task classes in the configuration. This prepares and performs the annotations in the - :setting:`CELERY_ANNOTATIONS` setting. + :setting:`task_annotations` setting. """ from __future__ import absolute_import @@ -38,7 +38,7 @@ def annotate(self, task): def prepare(annotations): - """Expands the :setting:`CELERY_ANNOTATIONS` setting.""" + """Expands the :setting:`task_annotations` setting.""" def expand_annotation(annotation): if isinstance(annotation, dict): diff --git a/celery/app/base.py b/celery/app/base.py index 40d4afc26..eb91173f8 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -13,11 +13,10 @@ import warnings from collections import defaultdict, deque -from copy import deepcopy from operator import attrgetter from functools import wraps -from amqp import promise +from amqp import starpromise try: from billiard.util import register_after_fork except ImportError: @@ -33,8 +32,9 @@ _register_app, get_current_worker_task, connect_on_app_finalize, _announce_app_finalized, ) +from celery.datastructures import AttributeDictMixin from celery.exceptions import AlwaysEagerIgnored, ImproperlyConfigured -from celery.five import items, values +from celery.five import UserDict, values from celery.loaders import get_loader_cls from celery.local import PromiseProxy, maybe_evaluate from celery.utils import abstract @@ -45,10 +45,11 @@ from celery.utils.objects import FallbackContext, mro_lookup from .annotations import prepare as prepare_annotations -from .defaults import DEFAULTS, find_deprecated_settings +from .defaults import find_deprecated_settings from .registry import TaskRegistry from .utils import ( - AppPickler, Settings, bugreport, _unpickle_app, _unpickle_app_v2, appstr, + AppPickler, Settings, + bugreport, _unpickle_app, _unpickle_app_v2, appstr, detect_settings, ) # Load all builtin tasks @@ -107,6 +108,18 @@ def _ensure_after_fork(): register_after_fork(_global_after_fork, _global_after_fork) +class PendingConfiguration(UserDict, AttributeDictMixin): + callback = None + data = None + + def __init__(self, conf, callback): + object.__setattr__(self, 'data', conf) + object.__setattr__(self, 'callback', callback) + + def __getitem__(self, key): + return self.callback(key) + + class Celery(object): """Celery application. @@ -117,7 +130,7 @@ class Celery(object): Default is :class:`celery.loaders.app.AppLoader`. :keyword backend: The result store backend class, or the name of the backend class to use. Default is the value of the - :setting:`CELERY_RESULT_BACKEND` setting. + :setting:`result_backend` setting. :keyword amqp: AMQP object or class name. :keyword events: Events object or class name. :keyword log: Log object or class name. @@ -181,7 +194,7 @@ def __init__(self, main=None, loader=None, backend=None, amqp=None, events=None, log=None, control=None, set_as_current=True, tasks=None, broker=None, include=None, changes=None, config_source=None, fixups=None, task_cls=None, - autofinalize=True, **kwargs): + autofinalize=True, namespace=None, **kwargs): self.clock = LamportClock() self.main = main self.amqp_cls = amqp or self.amqp_cls @@ -195,6 +208,7 @@ def __init__(self, main=None, loader=None, backend=None, self.user_options = defaultdict(set) self.steps = defaultdict(set) self.autofinalize = autofinalize + self.namespace = namespace self.configured = False self._config_source = config_source @@ -216,12 +230,15 @@ def __init__(self, main=None, loader=None, backend=None, # these options are moved to the config to # simplify pickling of the app object. self._preconf = changes or {} - if broker: - self._preconf['BROKER_URL'] = broker - if backend: - self._preconf['CELERY_RESULT_BACKEND'] = backend - if include: - self._preconf['CELERY_IMPORTS'] = include + self._preconf_set_by_auto = set() + self.__autoset('broker_url', broker) + self.__autoset('result_backend', backend) + self.__autoset('include', include) + self._conf = Settings( + PendingConfiguration( + self._preconf, self._get_from_conf_and_finalize), + prefix=self.namespace, + ) # - Apply fixups. self.fixups = set(self.builtin_fixups) if fixups is None else fixups @@ -241,6 +258,11 @@ def __init__(self, main=None, loader=None, backend=None, self.on_init() _register_app(self) + def __autoset(self, key, value): + if value: + self._preconf[key] = value + self._preconf_set_by_auto.add(key) + def set_current(self): """Makes this the current app for this thread.""" _set_current_app(self) @@ -445,7 +467,8 @@ def add_defaults(self, fun): return self._conf.add_defaults(fun()) self._pending_defaults.append(fun) - def config_from_object(self, obj, silent=False, force=False): + def config_from_object(self, obj, + silent=False, force=False, namespace=None): """Reads configuration from object, where object is either an object or the name of a module to import. @@ -463,9 +486,11 @@ def config_from_object(self, obj, silent=False, force=False): """ self._config_source = obj + self.namespace = namespace or self.namespace if force or self.configured: self._conf = None - return self.loader.config_from_object(obj, silent=silent) + if self.loader.config_from_object(obj, silent=silent): + return self.conf def config_from_envvar(self, variable_name, silent=False, force=False): """Read configuration from environment variable. @@ -488,7 +513,7 @@ def config_from_envvar(self, variable_name, silent=False, force=False): return self.config_from_object(module_name, silent=silent, force=force) def config_from_cmdline(self, argv, namespace='celery'): - (self._conf if self.configured else self.conf).update( + self._conf.update( self.loader.cmdline_config_parser(argv, namespace) ) @@ -505,15 +530,15 @@ def setup_security(self, allowed_serializers=None, key=None, cert=None, :keyword allowed_serializers: List of serializer names, or content_types that should be exempt from being disabled. :keyword key: Name of private key file to use. - Defaults to the :setting:`CELERY_SECURITY_KEY` setting. + Defaults to the :setting:`security_key` setting. :keyword cert: Name of certificate file to use. - Defaults to the :setting:`CELERY_SECURITY_CERTIFICATE` setting. + Defaults to the :setting:`security_certificate` setting. :keyword store: Directory containing certificates. - Defaults to the :setting:`CELERY_SECURITY_CERT_STORE` setting. + Defaults to the :setting:`security_cert_store` setting. :keyword digest: Digest algorithm used when signing messages. Default is ``sha1``. :keyword serializer: Serializer used to encode messages after - they have been signed. See :setting:`CELERY_TASK_SERIALIZER` for + they have been signed. See :setting:`task_serializer` for the serializers supported. Default is ``json``. @@ -559,8 +584,8 @@ def autodiscover_tasks(self, packages=None, """ if force: return self._autodiscover_tasks(packages, related_name) - signals.import_modules.connect(promise( - self._autodiscover_tasks, (packages, related_name), + signals.import_modules.connect(starpromise( + self._autodiscover_tasks, packages, related_name, ), weak=False, sender=self) def _autodiscover_tasks(self, packages, related_name, **kwargs): @@ -603,9 +628,9 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, producer = producer or publisher # XXX compat router = router or amqp.router conf = self.conf - if conf.CELERY_ALWAYS_EAGER: # pragma: no cover + if conf.task_always_eager: # pragma: no cover warnings.warn(AlwaysEagerIgnored( - 'CELERY_ALWAYS_EAGER has no effect on send_task', + 'task_always_eager has no effect on send_task', ), stacklevel=2) options = router.route(options, route_name or name, args, kwargs) @@ -614,7 +639,7 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, expires, retries, chord, maybe_list(link), maybe_list(link_error), reply_to or self.oid, time_limit, soft_time_limit, - self.conf.CELERY_SEND_TASK_SENT_EVENT, + self.conf.task_send_sent_event, root_id, parent_id, shadow, ) @@ -646,8 +671,8 @@ def connection(self, hostname=None, userid=None, password=None, :keyword password: Password to authenticate with :keyword virtual_host: Virtual host to use (domain). :keyword port: Port to connect to. - :keyword ssl: Defaults to the :setting:`BROKER_USE_SSL` setting. - :keyword transport: defaults to the :setting:`BROKER_TRANSPORT` + :keyword ssl: Defaults to the :setting:`broker_use_ssl` setting. + :keyword transport: defaults to the :setting:`broker_transport` setting. :returns :class:`kombu.Connection`: @@ -655,23 +680,23 @@ def connection(self, hostname=None, userid=None, password=None, """ conf = self.conf return self.amqp.Connection( - hostname or conf.BROKER_URL, - userid or conf.BROKER_USER, - password or conf.BROKER_PASSWORD, - virtual_host or conf.BROKER_VHOST, - port or conf.BROKER_PORT, - transport=transport or conf.BROKER_TRANSPORT, - ssl=self.either('BROKER_USE_SSL', ssl), + hostname or conf.broker_url, + userid or conf.broker_user, + password or conf.broker_password, + virtual_host or conf.broker_vhost, + port or conf.broker_port, + transport=transport or conf.broker_transport, + ssl=self.either('broker_use_ssl', ssl), heartbeat=heartbeat, - login_method=login_method or conf.BROKER_LOGIN_METHOD, + login_method=login_method or conf.broker_login_method, failover_strategy=( - failover_strategy or conf.BROKER_FAILOVER_STRATEGY + failover_strategy or conf.broker_failover_strategy ), transport_options=dict( - conf.BROKER_TRANSPORT_OPTIONS, **transport_options or {} + conf.broker_transport_options, **transport_options or {} ), connect_timeout=self.either( - 'BROKER_CONNECTION_TIMEOUT', connect_timeout + 'broker_connection_timeout', connect_timeout ), ) broker_connection = connection @@ -712,24 +737,24 @@ def prepare_config(self, c): def now(self): """Return the current time and date as a :class:`~datetime.datetime` object.""" - return self.loader.now(utc=self.conf.CELERY_ENABLE_UTC) + return self.loader.now(utc=self.conf.enable_utc) def mail_admins(self, subject, body, fail_silently=False): - """Sends an email to the admins in the :setting:`ADMINS` setting.""" + """Sends an email to the admins in the :setting:`admins` setting.""" conf = self.conf - if conf.ADMINS: - to = [admin_email for _, admin_email in conf.ADMINS] + if conf.admins: + to = [admin_email for _, admin_email in conf.admins] return self.loader.mail_admins( subject, body, fail_silently, to=to, - sender=conf.SERVER_EMAIL, - host=conf.EMAIL_HOST, - port=conf.EMAIL_PORT, - user=conf.EMAIL_HOST_USER, - password=conf.EMAIL_HOST_PASSWORD, - timeout=conf.EMAIL_TIMEOUT, - use_ssl=conf.EMAIL_USE_SSL, - use_tls=conf.EMAIL_USE_TLS, - charset=conf.EMAIL_CHARSET, + sender=conf.server_email, + host=conf.email_host, + port=conf.email_port, + user=conf.email_host_user, + password=conf.email_host_password, + timeout=conf.email_timeout, + use_ssl=conf.email_use_ssl, + use_tls=conf.email_use_tls, + charset=conf.email_charset, ) def select_queues(self, queues=None): @@ -741,7 +766,9 @@ def select_queues(self, queues=None): def either(self, default_key, *values): """Fallback to the value of a configuration key if none of the `*values` are true.""" - return first(None, values) or self.conf.get(default_key) + return first(None, [ + first(None, values), starpromise(self.conf.get, default_key), + ]) def bugreport(self): """Return a string with information useful for the Celery core @@ -751,7 +778,7 @@ def bugreport(self): def _get_backend(self): from celery.backends import get_backend_by_url backend, url = get_backend_by_url( - self.backend_cls or self.conf.CELERY_RESULT_BACKEND, + self.backend_cls or self.conf.result_backend, self.loader) return backend(app=self, url=url) @@ -763,27 +790,32 @@ def _load_config(self): self.on_configure() if self._config_source: self.loader.config_from_object(self._config_source) - defaults = dict(deepcopy(DEFAULTS), **self._preconf) + self.configured = True - s = self._conf = Settings( - {}, [self.prepare_config(self.loader.conf), defaults], + settings = detect_settings( + self.prepare_config(self.loader.conf), self._preconf, + ignore_keys=self._preconf_set_by_auto, prefix=self.namespace, ) + if self._conf is not None: + # replace in place, as someone may have referenced app.conf, + # done some changes, accessed a key, and then try to make more + # changes to the reference and not the finalized value. + self._conf.swap_with(settings) + else: + self._conf = settings + # load lazy config dict initializers. pending_def = self._pending_defaults while pending_def: - s.add_defaults(maybe_evaluate(pending_def.popleft()())) + self._conf.add_defaults(maybe_evaluate(pending_def.popleft()())) # load lazy periodic tasks pending_beat = self._pending_periodic_tasks while pending_beat: self._add_periodic_task(*pending_beat.popleft()) - # Settings.__setitem__ method, set Settings.change - if self._preconf: - for key, value in items(self._preconf): - setattr(s, key, value) - self.on_after_configure.send(sender=self, source=s) - return s + self.on_after_configure.send(sender=self, source=self._conf) + return self._conf def _after_fork(self, obj_): self._maybe_close_pool() @@ -830,7 +862,7 @@ def _sig_to_periodic_task_entry(self, schedule, sig, } def _add_periodic_task(self, key, entry): - self._conf.CELERYBEAT_SCHEDULE[key] = entry + self._conf.beat_schedule[key] = entry def create_task_cls(self): """Creates a base task class using default configuration @@ -893,7 +925,8 @@ def __reduce_keys__(self): when unpickling.""" return { 'main': self.main, - 'changes': self._conf.changes if self._conf else self._preconf, + 'changes': + self._conf.changes if self.configured else self._preconf, 'loader': self.loader_cls, 'backend': self.backend_cls, 'amqp': self.amqp_cls, @@ -903,11 +936,12 @@ def __reduce_keys__(self): 'fixups': self.fixups, 'config_source': self._config_source, 'task_cls': self.task_cls, + 'namespace': self.namespace, } def __reduce_args__(self): """Deprecated method, please use :meth:`__reduce_keys__` instead.""" - return (self.main, self._conf.changes if self._conf else {}, + return (self.main, self._conf.changes if self.configured else {}, self.loader_cls, self.backend_cls, self.amqp_cls, self.events_cls, self.log_cls, self.control_cls, False, self._config_source) @@ -938,7 +972,7 @@ def Task(self): @cached_property def annotations(self): - return prepare_annotations(self.conf.CELERY_ANNOTATIONS) + return prepare_annotations(self.conf.task_annotations) @cached_property def AsyncResult(self): @@ -981,7 +1015,7 @@ def pool(self): """ if self._pool is None: _ensure_after_fork() - limit = self.conf.BROKER_POOL_LIMIT + limit = self.conf.broker_pool_limit self._pool = self.connection().Pool(limit=limit) return self._pool @@ -1009,9 +1043,13 @@ def backend(self): def conf(self): """Current configuration.""" if self._conf is None: - self._load_config() + self._conf = self._load_config() return self._conf + def _get_from_conf_and_finalize(self, key): + conf = self._conf = self._load_config() + return conf[key] + @conf.setter def conf(self, d): # noqa self._conf = d @@ -1056,14 +1094,14 @@ def timezone(self): """Current timezone for this app. This is a cached property taking the time zone from the - :setting:`CELERY_TIMEZONE` setting. + :setting:`timezone` setting. """ from celery.utils.timeutils import timezone conf = self.conf - tz = conf.CELERY_TIMEZONE + tz = conf.timezone if not tz: - return (timezone.get_timezone('UTC') if conf.CELERY_ENABLE_UTC + return (timezone.get_timezone('UTC') if conf.enable_utc else timezone.local) - return timezone.get_timezone(conf.CELERY_TIMEZONE) + return timezone.get_timezone(conf.timezone) App = Celery # compat diff --git a/celery/app/builtins.py b/celery/app/builtins.py index d1d341af2..cfe6cc884 100644 --- a/celery/app/builtins.py +++ b/celery/app/builtins.py @@ -54,7 +54,7 @@ def add_unlock_chord_task(app): from celery.exceptions import ChordError from celery.result import allow_join_result, result_from_tuple - default_propagate = app.conf.CELERY_CHORD_PROPAGATES + default_propagate = app.conf.chord_propagates @app.task(name='celery.chord_unlock', max_retries=None, shared=False, default_retry_delay=1, ignore_result=True, lazy=False, bind=True) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index e64716269..0730a551f 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -13,7 +13,7 @@ from collections import deque, namedtuple from datetime import timedelta -from celery.five import items +from celery.five import items, keys, values from celery.utils import strtobool from celery.utils.functional import memoize @@ -39,13 +39,29 @@ DEFAULT_TASK_LOG_FMT = """[%(asctime)s: %(levelname)s/%(processName)s] \ %(task_name)s[%(task_id)s]: %(message)s""" +OLD_NS = {'celery_{0}'} +OLD_NS_BEAT = {'celerybeat_{0}'} +OLD_NS_WORKER = {'celeryd_{0}'} + searchresult = namedtuple('searchresult', ('namespace', 'key', 'type')) +def Namespace(__old__=None, **options): + if __old__ is not None: + for opt in values(options): + opt.old = opt.old | __old__ + return options + + +def old_ns(ns): + return {'{0}_{{0}}'.format(ns)} + + class Option(object): alt = None deprecate_by = None remove_by = None + old = set() typemap = dict(string=str, int=int, float=float, any=lambda v: v, bool=strtobool, dict=dict, tuple=tuple) @@ -62,166 +78,260 @@ def __repr__(self): return '{0} default->{1!r}>'.format(self.type, self.default) -NAMESPACES = { - 'BROKER': { - 'URL': Option(None, type='string'), - 'CONNECTION_TIMEOUT': Option(4, type='float'), - 'CONNECTION_RETRY': Option(True, type='bool'), - 'CONNECTION_MAX_RETRIES': Option(100, type='int'), - 'FAILOVER_STRATEGY': Option(None, type='string'), - 'HEARTBEAT': Option(None, type='int'), - 'HEARTBEAT_CHECKRATE': Option(3.0, type='int'), - 'LOGIN_METHOD': Option(None, type='string'), - 'POOL_LIMIT': Option(10, type='int'), - 'USE_SSL': Option(False, type='bool'), - 'TRANSPORT': Option(type='string'), - 'TRANSPORT_OPTIONS': Option({}, type='dict'), - 'HOST': Option(type='string'), - 'PORT': Option(type='int'), - 'USER': Option(type='string'), - 'PASSWORD': Option(type='string'), - 'VHOST': Option(type='string'), - }, - 'CASSANDRA': { - 'COLUMN_FAMILY': Option(type='string'), - 'DETAILED_MODE': Option(False, type='bool'), - 'KEYSPACE': Option(type='string'), - 'READ_CONSISTENCY': Option(type='string'), - 'SERVERS': Option(type='list'), - 'PORT': Option(type="string"), - 'ENTRY_TTL': Option(type="float"), - 'WRITE_CONSISTENCY': Option(type='string'), - }, - 'CELERY': { - 'ACCEPT_CONTENT': Option(DEFAULT_ACCEPT_CONTENT, type='list'), - 'ACKS_LATE': Option(False, type='bool'), - 'ALWAYS_EAGER': Option(False, type='bool'), - 'ANNOTATIONS': Option(type='any'), - 'BROADCAST_QUEUE': Option('celeryctl'), - 'BROADCAST_EXCHANGE': Option('celeryctl'), - 'BROADCAST_EXCHANGE_TYPE': Option('fanout'), - 'CACHE_BACKEND': Option(), - 'CACHE_BACKEND_OPTIONS': Option({}, type='dict'), - 'CHORD_PROPAGATES': Option(True, type='bool'), - 'COUCHBASE_BACKEND_SETTINGS': Option(None, type='dict'), - 'CREATE_MISSING_QUEUES': Option(True, type='bool'), - 'DEFAULT_RATE_LIMIT': Option(type='string'), - 'DISABLE_RATE_LIMITS': Option(False, type='bool'), - 'DEFAULT_ROUTING_KEY': Option('celery'), - 'DEFAULT_QUEUE': Option('celery'), - 'DEFAULT_EXCHANGE': Option('celery'), - 'DEFAULT_EXCHANGE_TYPE': Option('direct'), - 'DEFAULT_DELIVERY_MODE': Option(2, type='string'), - 'EAGER_PROPAGATES_EXCEPTIONS': Option(False, type='bool'), - 'ENABLE_UTC': Option(True, type='bool'), - 'ENABLE_REMOTE_CONTROL': Option(True, type='bool'), - 'EVENT_SERIALIZER': Option('json'), - 'EVENT_QUEUE_EXPIRES': Option(60.0, type='float'), - 'EVENT_QUEUE_TTL': Option(5.0, type='float'), - 'IMPORTS': Option((), type='tuple'), - 'INCLUDE': Option((), type='tuple'), - 'IGNORE_RESULT': Option(False, type='bool'), - 'MAX_CACHED_RESULTS': Option(100, type='int'), - 'MESSAGE_COMPRESSION': Option(type='string'), - 'MONGODB_BACKEND_SETTINGS': Option(type='dict'), - 'REDIS_HOST': Option(type='string'), - 'REDIS_PORT': Option(type='int'), - 'REDIS_DB': Option(type='int'), - 'REDIS_PASSWORD': Option(type='string'), - 'REDIS_MAX_CONNECTIONS': Option(type='int'), - 'REJECT_ON_WORKER_LOST': Option(type='bool'), - 'RESULT_BACKEND': Option(type='string'), - 'RESULT_DB_SHORT_LIVED_SESSIONS': Option(False, type='bool'), - 'RESULT_DB_TABLENAMES': Option(type='dict'), - 'RESULT_DBURI': Option(), - 'RESULT_ENGINE_OPTIONS': Option(type='dict'), - 'RESULT_EXCHANGE': Option('celeryresults'), - 'RESULT_EXCHANGE_TYPE': Option('direct'), - 'RESULT_SERIALIZER': Option('json'), - 'RESULT_PERSISTENT': Option(None, type='bool'), - 'RIAK_BACKEND_SETTINGS': Option(type='dict'), - 'ROUTES': Option(type='any'), - 'SEND_EVENTS': Option(False, type='bool'), - 'SEND_TASK_ERROR_EMAILS': Option(False, type='bool'), - 'SEND_TASK_SENT_EVENT': Option(False, type='bool'), - 'STORE_ERRORS_EVEN_IF_IGNORED': Option(False, type='bool'), - 'TASK_PROTOCOL': Option(1, type='int'), - 'TASK_PUBLISH_RETRY': Option(True, type='bool'), - 'TASK_PUBLISH_RETRY_POLICY': Option({ - 'max_retries': 3, - 'interval_start': 0, - 'interval_max': 1, - 'interval_step': 0.2}, type='dict'), - 'TASK_RESULT_EXPIRES': Option(timedelta(days=1), type='float'), - 'TASK_SERIALIZER': Option('json'), - 'TIMEZONE': Option(type='string'), - 'TRACK_STARTED': Option(False, type='bool'), - 'REDIRECT_STDOUTS': Option(True, type='bool'), - 'REDIRECT_STDOUTS_LEVEL': Option('WARNING'), - 'QUEUES': Option(type='dict'), - 'QUEUE_HA_POLICY': Option(None, type='string'), - 'QUEUE_MAX_PRIORITY': Option(None, type='int'), - 'SECURITY_KEY': Option(type='string'), - 'SECURITY_CERTIFICATE': Option(type='string'), - 'SECURITY_CERT_STORE': Option(type='string'), - 'WORKER_DIRECT': Option(False, type='bool'), - }, - 'CELERYD': { - 'AGENT': Option(None, type='string'), - 'AUTOSCALER': Option('celery.worker.autoscale:Autoscaler'), - 'AUTORELOADER': Option('celery.worker.autoreload:Autoreloader'), - 'CONCURRENCY': Option(0, type='int'), - 'TIMER': Option(type='string'), - 'TIMER_PRECISION': Option(1.0, type='float'), - 'FORCE_EXECV': Option(False, type='bool'), - 'HIJACK_ROOT_LOGGER': Option(True, type='bool'), - 'CONSUMER': Option('celery.worker.consumer:Consumer', type='string'), - 'LOG_FORMAT': Option(DEFAULT_PROCESS_LOG_FMT), - 'LOG_COLOR': Option(type='bool'), - 'MAX_TASKS_PER_CHILD': Option(type='int'), - 'MAX_MEMORY_PER_CHILD': Option(type='int'), - 'POOL': Option(DEFAULT_POOL), - 'POOL_PUTLOCKS': Option(True, type='bool'), - 'POOL_RESTARTS': Option(False, type='bool'), - 'PREFETCH_MULTIPLIER': Option(4, type='int'), - 'STATE_DB': Option(), - 'TASK_LOG_FORMAT': Option(DEFAULT_TASK_LOG_FMT), - 'TASK_SOFT_TIME_LIMIT': Option(type='float'), - 'TASK_TIME_LIMIT': Option(type='float'), - 'WORKER_LOST_WAIT': Option(10.0, type='float') - }, - 'CELERYBEAT': { - 'SCHEDULE': Option({}, type='dict'), - 'SCHEDULER': Option('celery.beat:PersistentScheduler'), - 'SCHEDULE_FILENAME': Option('celerybeat-schedule'), - 'SYNC_EVERY': Option(0, type='int'), - 'MAX_LOOP_INTERVAL': Option(0, type='float'), - }, - 'EMAIL': { - 'HOST': Option('localhost'), - 'PORT': Option(25, type='int'), - 'HOST_USER': Option(), - 'HOST_PASSWORD': Option(), - 'TIMEOUT': Option(2, type='float'), - 'USE_SSL': Option(False, type='bool'), - 'USE_TLS': Option(False, type='bool'), - 'CHARSET': Option('us-ascii'), - }, - 'SERVER_EMAIL': Option('celery@localhost'), - 'ADMINS': Option((), type='tuple'), -} +NAMESPACES = Namespace( + accept_content=Option(DEFAULT_ACCEPT_CONTENT, type='list', old=OLD_NS), + admins=Option((), type='tuple'), + enable_utc=Option(True, type='bool'), + imports=Option((), type='tuple', old=OLD_NS), + include=Option((), type='tuple', old=OLD_NS), + server_email=Option('celery@localhost'), + timezone=Option(type='string', old=OLD_NS), + beat=Namespace( + __old__=OLD_NS_BEAT, + + max_loop_interval=Option(0, type='float'), + schedule=Option({}, type='dict'), + scheduler=Option('celery.beat:PersistentScheduler'), + schedule_filename=Option('celerybeat-schedule'), + sync_every=Option(0, type='int'), + ), + broker=Namespace( + url=Option(None, type='string'), + transport=Option(type='string'), + transport_options=Option({}, type='dict'), + connection_timeout=Option(4, type='float'), + connection_retry=Option(True, type='bool'), + connection_max_retries=Option(100, type='int'), + failover_strategy=Option(None, type='string'), + heartbeat=Option(None, type='int'), + heartbeat_checkrate=Option(3.0, type='int'), + login_method=Option(None, type='string'), + pool_limit=Option(10, type='int'), + use_ssl=Option(False, type='bool'), + + host=Option(type='string'), + port=Option(type='int'), + user=Option(type='string'), + password=Option(type='string'), + vhost=Option(type='string'), + ), + cache=Namespace( + __old__=old_ns('celery_cache'), + + backend=Option(), + backend_options=Option({}, type='dict'), + ), + cassandra=Namespace( + column_family=Option(type='string'), + entry_ttl=Option(type="float"), + keyspace=Option(type='string'), + port=Option(type="string"), + read_consistency=Option(type='string'), + servers=Option(type='list'), + write_consistency=Option(type='string'), + ), + chord=Namespace( + __old__=old_ns('celery_chord'), + + propagates=Option(True, type='bool'), + ), + couchbase=Namespace( + __old__=old_ns('celery_couchbase'), + + backend_settings=Option(None, type='dict'), + ), + email=Namespace( + charset=Option('us-ascii'), + host=Option('localhost'), + host_user=Option(), + host_password=Option(), + port=Option(25, type='int'), + timeout=Option(2, type='float'), + use_ssl=Option(False, type='bool'), + use_tls=Option(False, type='bool'), + ), + mongodb=Namespace( + __old__=old_ns('celery_mongodb'), + + backend_settings=Option(type='dict'), + ), + event=Namespace( + __old__=old_ns('celery_event'), + queue_expires=Option(60.0, type='float'), + queue_ttl=Option(5.0, type='float'), + serializer=Option('json'), + ), + redis=Namespace( + __old__=old_ns('celery_redis'), -def flatten(d, ns=''): - stack = deque([(ns, d)]) + db=Option(type='int'), + host=Option(type='string'), + max_connections=Option(type='int'), + password=Option(type='string'), + port=Option(type='int'), + ), + result=Namespace( + __old__=old_ns('celery_result'), + + backend=Option(type='string'), + cache_max=Option( + 100, + type='int', old={'celery_max_cached_results'}, + ), + compression=Option(type='str'), + exchange=Option('celeryresults'), + exchange_type=Option('direct'), + expires=Option( + timedelta(days=1), + type='float', old={'celery_task_result_expires'}, + ), + persistent=Option(None, type='bool'), + serializer=Option('json'), + ), + riak=Namespace( + __old__=old_ns('celery_riak'), + + backend_settings=Option(type='dict'), + ), + security=Namespace( + __old__=old_ns('celery_security'), + + certificate=Option(type='string'), + cert_store=Option(type='string'), + key=Option(type='string'), + ), + sqlalchemy=Namespace( + dburi=Option(old={'celery_result_dburi'}), + engine_options=Option( + type='dict', old={'celery_result_engine_options'}, + ), + short_lived_sessions=Option( + False, type='bool', old={'celery_result_db_short_lived_sessions'}, + ), + table_names=Option(type='dict', old={'celery_result_db_tablenames'}), + ), + task=Namespace( + __old__=OLD_NS, + acks_late=Option(False, type='bool'), + always_eager=Option(False, type='bool'), + annotations=Option(type='any'), + compression=Option(type='string', old={'celery_message_compression'}), + create_missing_queues=Option(True, type='bool'), + default_delivery_mode=Option(2, type='string'), + default_exchange=Option('celery'), + default_exchange_type=Option('direct'), + default_queue=Option('celery'), + default_rate_limit=Option(type='string'), + default_routing_key=Option('celery'), + eager_propagates_exceptions=Option(False, type='bool'), + ignore_result=Option(False, type='bool'), + protocol=Option(1, type='int', old={'celery_task_protocol'}), + publish_retry=Option( + True, type='bool', old={'celery_task_publish_retry'}, + ), + publish_retry_policy=Option( + {'max_retries': 3, + 'interval_start': 0, + 'interval_max': 1, + 'interval_step': 0.2}, + type='dict', old={'celery_task_publish_retry_policy'}, + ), + queues=Option(type='dict'), + queue_ha_policy=Option(None, type='string'), + queue_max_priority=Option(None, type='int'), + reject_on_worker_lost=Option(type='bool'), + routes=Option(type='any'), + send_error_emails=Option( + False, type='bool', old={'celery_send_task_error_emails'}, + ), + send_sent_event=Option( + False, type='bool', old={'celery_send_task_sent_event'}, + ), + serializer=Option('json', old={'celery_task_serializer'}), + soft_time_limit=Option( + type='float', old={'celeryd_task_soft_time_limit'}, + ), + time_limit=Option( + type='float', old={'celeryd_task_time_limit'}, + ), + store_errors_even_if_ignored=Option(False, type='bool'), + track_started=Option(False, type='bool'), + ), + worker=Namespace( + __old__=OLD_NS_WORKER, + agent=Option(None, type='string'), + autoscaler=Option('celery.worker.autoscale:Autoscaler'), + autoreloader=Option('celery.worker.autoreload:Autoreloader'), + concurrency=Option(0, type='int'), + consumer=Option('celery.worker.consumer:Consumer', type='string'), + direct=Option(False, type='bool', old={'celery_worker_direct'}), + disable_rate_limits=Option( + False, type='bool', old={'celery_disable_rate_limits'}, + ), + enable_remote_control=Option( + True, type='bool', old={'celery_enable_remote_control'}, + ), + force_execv=Option(False, type='bool'), + hijack_root_logger=Option(True, type='bool'), + log_color=Option(type='bool'), + log_format=Option(DEFAULT_PROCESS_LOG_FMT), + lost_wait=Option(10.0, type='float'), + max_memory_per_child=Option(type='int'), + max_tasks_per_child=Option(type='int'), + pool=Option(DEFAULT_POOL), + pool_putlocks=Option(True, type='bool'), + pool_restarts=Option(False, type='bool'), + prefetch_multiplier=Option(4, type='int'), + redirect_stdouts=Option( + True, type='bool', old={'celery_redirect_stdouts'}, + ), + redirect_stdouts_level=Option( + 'WARNING', old={'celery_redirect_stdouts_level'}, + ), + send_events=Option(False, type='bool'), + state_db=Option(), + task_log_format=Option(DEFAULT_TASK_LOG_FMT), + timer=Option(type='string'), + timer_precision=Option(1.0, type='float'), + ), +) + + +def _flatten_keys(ns, key, opt): + return [(ns + key, opt)] + + +def _to_compat(ns, key, opt): + if opt.old: + return [ + (oldkey.format(key).upper(), ns + key, opt) + for oldkey in opt.old + ] + return [((ns + key).upper(), ns + key, opt)] + + +def flatten(d, root='', keyfilter=_flatten_keys): + stack = deque([(root, d)]) while stack: - name, space = stack.popleft() - for key, value in items(space): - if isinstance(value, dict): - stack.append((name + key + '_', value)) + ns, options = stack.popleft() + for key, opt in items(options): + if isinstance(opt, dict): + stack.append((ns + key + '_', opt)) else: - yield name + key, value -DEFAULTS = {key: value.default for key, value in flatten(NAMESPACES)} + for ret in keyfilter(ns, key, opt): + yield ret +DEFAULTS = { + key: opt.default for key, opt in flatten(NAMESPACES) +} +__compat = list(flatten(NAMESPACES, keyfilter=_to_compat)) +_OLD_DEFAULTS = {old_key: opt.default for old_key, _, opt in __compat} +_TO_OLD_KEY = {new_key: old_key for old_key, new_key, _ in __compat} +_TO_NEW_KEY = {old_key: new_key for old_key, new_key, _ in __compat} +__compat = None + +SETTING_KEYS = set(keys(DEFAULTS)) +_OLD_SETTING_KEYS = set(keys(_TO_NEW_KEY)) def find_deprecated_settings(source): @@ -238,20 +348,20 @@ def find_deprecated_settings(source): @memoize(maxsize=None) def find(name, namespace='celery'): # - Try specified namespace first. - namespace = namespace.upper() + namespace = namespace.lower() try: return searchresult( - namespace, name.upper(), NAMESPACES[namespace][name.upper()], + namespace, name.lower(), NAMESPACES[namespace][name.lower()], ) except KeyError: # - Try all the other namespaces. - for ns, keys in items(NAMESPACES): - if ns.upper() == name.upper(): - return searchresult(None, ns, keys) - elif isinstance(keys, dict): + for ns, opts in items(NAMESPACES): + if ns.lower() == name.lower(): + return searchresult(None, ns, opts) + elif isinstance(opts, dict): try: - return searchresult(ns, name.upper(), keys[name.upper()]) + return searchresult(ns, name.lower(), opts[name.lower()]) except KeyError: pass # - See if name is a qualname last. - return searchresult(None, name.upper(), DEFAULTS[name.upper()]) + return searchresult(None, name.lower(), DEFAULTS[name.lower()]) diff --git a/celery/app/log.py b/celery/app/log.py index 3f6261b6a..4c8fb030e 100644 --- a/celery/app/log.py +++ b/celery/app/log.py @@ -59,9 +59,9 @@ class Logging(object): def __init__(self, app): self.app = app self.loglevel = mlevel(logging.WARN) - self.format = self.app.conf.CELERYD_LOG_FORMAT - self.task_format = self.app.conf.CELERYD_TASK_LOG_FORMAT - self.colorize = self.app.conf.CELERYD_LOG_COLOR + self.format = self.app.conf.worker_log_format + self.task_format = self.app.conf.worker_task_log_format + self.colorize = self.app.conf.worker_log_color def setup(self, loglevel=None, logfile=None, redirect_stdouts=False, redirect_level='WARNING', colorize=None, hostname=None): @@ -105,7 +105,7 @@ def setup_logging_subsystem(self, loglevel=None, logfile=None, format=None, if not receivers: root = logging.getLogger() - if self.app.conf.CELERYD_HIJACK_ROOT_LOGGER: + if self.app.conf.worker_hijack_root_logger: root.handlers = [] get_logger('celery').handlers = [] get_logger('celery.task').handlers = [] diff --git a/celery/app/routes.py b/celery/app/routes.py index 0fa384103..06ab34abc 100644 --- a/celery/app/routes.py +++ b/celery/app/routes.py @@ -4,7 +4,7 @@ ~~~~~~~~~~~~~ Contains utilities for working with task routers, - (:setting:`CELERY_ROUTES`). + (:setting:`task_routes`). """ from __future__ import absolute_import @@ -52,7 +52,7 @@ def route(self, options, task, args=(), kwargs={}): return lpmerge(self.expand_destination(route), options) if 'queue' not in options: options = lpmerge(self.expand_destination( - self.app.conf.CELERY_DEFAULT_QUEUE), options) + self.app.conf.task_default_queue), options) return options def expand_destination(self, route): @@ -72,7 +72,7 @@ def expand_destination(self, route): route['queue'] = self.queues[queue] except KeyError: raise QueueNotFound( - 'Queue {0!r} missing from CELERY_QUEUES'.format(queue)) + 'Queue {0!r} missing from task_queues'.format(queue)) return route def lookup_route(self, task, args=None, kwargs=None): @@ -80,7 +80,7 @@ def lookup_route(self, task, args=None, kwargs=None): def prepare(routes): - """Expands the :setting:`CELERY_ROUTES` setting.""" + """Expands the :setting:`task_routes` setting.""" def expand_route(route): if isinstance(route, dict): diff --git a/celery/app/task.py b/celery/app/task.py index 76c4d1f2f..1d1baa4c9 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -160,7 +160,7 @@ class Task(object): rate_limit = None #: If enabled the worker will not store task state and return values - #: for this task. Defaults to the :setting:`CELERY_IGNORE_RESULT` + #: for this task. Defaults to the :setting:`task_ignore_result` #: setting. ignore_result = None @@ -173,7 +173,7 @@ class Task(object): #: configured to ignore results. store_errors_even_if_ignored = None - #: If enabled an email will be sent to :setting:`ADMINS` whenever a task + #: If enabled an email will be sent to :setting:`admins` whenever a task #: of this type fails. send_error_emails = None @@ -182,11 +182,11 @@ class Task(object): serializer = None #: Hard time limit. - #: Defaults to the :setting:`CELERYD_TASK_TIME_LIMIT` setting. + #: Defaults to the :setting:`task_time_limit` setting. time_limit = None #: Soft time limit. - #: Defaults to the :setting:`CELERYD_TASK_SOFT_TIME_LIMIT` setting. + #: Defaults to the :setting:`task_soft_time_limit` setting. soft_time_limit = None #: The result store backend used for this task. @@ -205,7 +205,7 @@ class Task(object): #: running. #: #: The application default can be overridden using the - #: :setting:`CELERY_TRACK_STARTED` setting. + #: :setting:`task_track_started` setting. track_started = None #: When enabled messages for this task will be acknowledged **after** @@ -217,7 +217,7 @@ class Task(object): #: applications). #: #: The application default can be overridden with the - #: :setting:`CELERY_ACKS_LATE` setting. + #: :setting:`task_acks_late` setting. acks_late = None #: Even if :attr:`acks_late` is enabled, the worker will @@ -255,15 +255,14 @@ class Task(object): __bound__ = False from_config = ( - ('send_error_emails', 'CELERY_SEND_TASK_ERROR_EMAILS'), - ('serializer', 'CELERY_TASK_SERIALIZER'), - ('rate_limit', 'CELERY_DEFAULT_RATE_LIMIT'), - ('track_started', 'CELERY_TRACK_STARTED'), - ('acks_late', 'CELERY_ACKS_LATE'), - ('reject_on_worker_lost', 'CELERY_REJECT_ON_WORKER_LOST'), - ('ignore_result', 'CELERY_IGNORE_RESULT'), - ('store_errors_even_if_ignored', - 'CELERY_STORE_ERRORS_EVEN_IF_IGNORED'), + ('send_error_emails', 'task_send_error_emails'), + ('serializer', 'task_serializer'), + ('rate_limit', 'task_default_rate_limit'), + ('track_started', 'task_track_started'), + ('acks_late', 'task_acks_late'), + ('reject_on_worker_lost', 'task_reject_on_worker_lost'), + ('ignore_result', 'task_ignore_result'), + ('store_errors_even_if_ignored', 'task_store_errors_even_if_ignored'), ) #: ignored @@ -409,12 +408,12 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, :keyword retry: If enabled sending of the task message will be retried in the event of connection loss or failure. Default - is taken from the :setting:`CELERY_TASK_PUBLISH_RETRY` + is taken from the :setting:`task_publish_retry` setting. Note that you need to handle the producer/connection manually for this to work. :keyword retry_policy: Override the retry policy used. See the - :setting:`CELERY_TASK_PUBLISH_RETRY_POLICY` + :setting:`task_publish_retry_policy` setting. :keyword routing_key: Custom routing key used to route the task to a @@ -423,8 +422,8 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, routing keys to topic exchanges. :keyword queue: The queue to route the task to. This must be a key - present in :setting:`CELERY_QUEUES`, or - :setting:`CELERY_CREATE_MISSING_QUEUES` must be + present in :setting:`task_queues`, or + :setting:`task_create_missing_queues` must be enabled. See :ref:`guide-routing` for more information. @@ -446,7 +445,7 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, to use. Can be one of ``zlib``, ``bzip2``, or any custom compression methods registered with :func:`kombu.compression.register`. Defaults to - the :setting:`CELERY_MESSAGE_COMPRESSION` + the :setting:`task_compression` setting. :keyword link: A single, or a list of tasks to apply if the task exits successfully. @@ -467,14 +466,14 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, task (a :class:`dict`) :rtype :class:`celery.result.AsyncResult`: if - :setting:`CELERY_ALWAYS_EAGER` is not set, otherwise + :setting:`task_always_eager` is not set, otherwise :class:`celery.result.EagerResult`: Also supports all keyword arguments supported by :meth:`kombu.Producer.publish`. .. note:: - If the :setting:`CELERY_ALWAYS_EAGER` setting is set, it will + If the :setting:`task_always_eager` setting is set, it will be replaced by a local :func:`apply` call instead. """ @@ -486,7 +485,7 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, check_arguments(*(args or ()), **(kwargs or {})) app = self._get_app() - if app.conf.CELERY_ALWAYS_EAGER: + if app.conf.task_always_eager: return self.apply(args, kwargs, task_id=task_id or uuid(), link=link, link_error=link_error, **options) # add 'self' if this is a "task_method". @@ -670,7 +669,7 @@ def apply(self, args=None, kwargs=None, :param args: positional arguments passed on to the task. :param kwargs: keyword arguments passed on to the task. :keyword throw: Re-raise task exceptions. Defaults to - the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS` + the :setting:`task_eager_propagates_exceptions` setting. :rtype :class:`celery.result.EagerResult`: @@ -687,7 +686,7 @@ def apply(self, args=None, kwargs=None, kwargs = kwargs or {} task_id = options.get('task_id') or uuid() retries = options.get('retries', 0) - throw = app.either('CELERY_EAGER_PROPAGATES_EXCEPTIONS', + throw = app.either('task_eager_propagates_exceptions', options.pop('throw', None)) # Make sure we get the task instance, not class. diff --git a/celery/app/trace.py b/celery/app/trace.py index 97860f817..ffd63fa50 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -464,7 +464,7 @@ def _trace_task_ret(name, uuid, request, body, content_type, app = app or current_app._get_current_object() embed = None if content_type: - accept = prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT) + accept = prepare_accept_content(app.conf.accept_content) args, kwargs, embed = loads( body, content_type, content_encoding, accept=accept, ) @@ -539,7 +539,7 @@ def setup_worker_optimizations(app, hostname=None): # set fast shortcut to task registry _localized[:] = [ app._tasks, - prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT), + prepare_accept_content(app.conf.accept_content), hostname, ] diff --git a/celery/app/utils.py b/celery/app/utils.py index 32ad7c24d..9a308cb0c 100644 --- a/celery/app/utils.py +++ b/celery/app/utils.py @@ -12,18 +12,23 @@ import platform as _platform import re -from collections import Mapping +from collections import Mapping, namedtuple +from copy import deepcopy from types import ModuleType from kombu.utils.url import maybe_sanitize_url from celery.datastructures import ConfigurationView -from celery.five import items, string_t, values +from celery.exceptions import ImproperlyConfigured +from celery.five import items, keys, string_t, values from celery.platforms import pyimplementation from celery.utils.text import pretty from celery.utils.imports import import_from_cwd, symbol_by_name, qualname -from .defaults import find +from .defaults import ( + _TO_NEW_KEY, _TO_OLD_KEY, _OLD_DEFAULTS, _OLD_SETTING_KEYS, + DEFAULTS, SETTING_KEYS, find, +) __all__ = ['Settings', 'appstr', 'bugreport', 'filter_hidden_settings', 'find_app'] @@ -44,6 +49,28 @@ re.IGNORECASE, ) +E_MIX_OLD_INTO_NEW = """ + +Cannot mix new and old setting keys, please rename the +following settings to the new format: + +{renames} + +""" + +E_MIX_NEW_INTO_OLD = """ + +Cannot mix new setting names with old setting names, please +rename the following settings to use the old format: + +{renames} + +Or change all of the settings to use the new format :) + +""" + +FMT_REPLACE_SETTING = '{replace:<36} -> {with_}' + def appstr(app): """String used in __repr__ etc, to id app instances.""" @@ -60,28 +87,14 @@ class Settings(ConfigurationView): """ @property - def CELERY_RESULT_BACKEND(self): - return self.first('CELERY_RESULT_BACKEND', 'CELERY_BACKEND') - - @property - def BROKER_TRANSPORT(self): - return self.first('BROKER_TRANSPORT', - 'BROKER_BACKEND', 'CARROT_BACKEND') - - @property - def BROKER_BACKEND(self): - """Deprecated compat alias to :attr:`BROKER_TRANSPORT`.""" - return self.BROKER_TRANSPORT - - @property - def BROKER_URL(self): + def broker_url(self): return (os.environ.get('CELERY_BROKER_URL') or - self.first('BROKER_URL', 'BROKER_HOST')) + self.first('broker_url', 'broker_host')) @property - def CELERY_TIMEZONE(self): + def timezone(self): # this way we also support django's time zone. - return self.first('CELERY_TIMEZONE', 'TIME_ZONE') + return self.first('timezone', 'time_zone') def without_defaults(self): """Return the current configuration, but without defaults.""" @@ -91,18 +104,18 @@ def without_defaults(self): def value_set_for(self, key): return key in self.without_defaults() - def find_option(self, name, namespace='celery'): + def find_option(self, name, namespace=''): """Search for option by name. Will return ``(namespace, key, type)`` tuple, e.g.:: >>> from proj.celery import app >>> app.conf.find_option('disable_rate_limits') - ('CELERY', 'DISABLE_RATE_LIMITS', + ('worker', 'prefetch_multiplier', bool default->False>)) :param name: Name of option, cannot be partial. - :keyword namespace: Preferred namespace (``CELERY`` by default). + :keyword namespace: Preferred namespace (``None`` by default). """ return find(name, namespace) @@ -117,7 +130,7 @@ def get_by_parts(self, *parts): Example:: >>> from proj.celery import app - >>> app.conf.get_by_parts('CELERY', 'DISABLE_RATE_LIMITS') + >>> app.conf.get_by_parts('worker', 'disable_rate_limits') False """ @@ -139,6 +152,72 @@ def humanize(self, with_defaults=False, censored=True): for key, value in items(self.table(with_defaults, censored))) +def _new_key_to_old(key, convert=_TO_OLD_KEY.get): + return convert(key, key) + + +def _old_key_to_new(key, convert=_TO_NEW_KEY.get): + return convert(key, key) + + +_settings_info_t = namedtuple('settings_info_t', ( + 'defaults', 'convert', 'key_t', 'mix_error', +)) + +_settings_info = _settings_info_t( + DEFAULTS, _TO_NEW_KEY, _old_key_to_new, E_MIX_OLD_INTO_NEW, +) +_old_settings_info = _settings_info_t( + _OLD_DEFAULTS, _TO_OLD_KEY, _new_key_to_old, E_MIX_NEW_INTO_OLD, +) + + +def detect_settings(conf, preconf={}, ignore_keys=set(), prefix=None, + all_keys=SETTING_KEYS, old_keys=_OLD_SETTING_KEYS): + source = conf + if conf is None: + source, conf = preconf, {} + have = set(keys(source)) - ignore_keys + is_in_new = have.intersection(all_keys) + is_in_old = have.intersection(old_keys) + + if is_in_new: + # have new setting names + info, left = _settings_info, is_in_old + if is_in_old and len(is_in_old) > len(is_in_new): + # Majority of the settings are old. + info, left = _old_settings_info, is_in_new + elif is_in_old: + print('IS IN OLD: %r' % (is_in_old, )) + # have old setting names, or a majority of the names are old. + info, left = _old_settings_info, is_in_new + if is_in_new and len(is_in_new) > len(is_in_old): + # Majority of the settings are new + info, left = _settings_info, is_in_old + else: + # no settings, just use new format. + info, left = _settings_info, is_in_old + + if prefix: + # always use new format if prefix is used. + info, left = _settings_info, set() + + # only raise error for keys that the user did not provide two keys + # for (e.g. both ``result_expires`` and ``CELERY_TASK_RESULT_EXPIRES``). + really_left = {key for key in left if info.convert[key] not in have} + if really_left: + # user is mixing old/new, or new/old settings, give renaming + # suggestions. + raise ImproperlyConfigured(info.mix_error.format(renames='\n'.join( + FMT_REPLACE_SETTING.format(replace=key, with_=info.convert[key]) + for key in sorted(really_left) + ))) + + preconf = {info.convert.get(k, k): v for k, v in items(preconf)} + defaults = dict(deepcopy(info.defaults), **preconf) + return Settings(preconf, [conf, defaults], info.key_t, prefix=prefix) + + class AppPickler(object): """Old application pickler/unpickler (< 3.1).""" @@ -185,10 +264,10 @@ def maybe_censor(key, value, mask='*' * 8): if isinstance(key, string_t): if HIDDEN_SETTINGS.search(key): return mask - elif 'BROKER_URL' in key.upper(): + elif 'broker_url' in key.lower(): from kombu import Connection return Connection(value).as_uri(mask=mask) - elif key.upper() in ('CELERY_RESULT_BACKEND', 'CELERY_BACKEND'): + elif 'backend' in key.lower(): return maybe_sanitize_url(value, mask=mask) return value @@ -220,7 +299,7 @@ def bugreport(app): py_v=_platform.python_version(), driver_v=driver_v, transport=transport, - results=app.conf.CELERY_RESULT_BACKEND or 'disabled', + results=app.conf.result_backend or 'disabled', human_settings=app.conf.humanize(), loader=qualname(app.loader.__class__), ) diff --git a/celery/apps/beat.py b/celery/apps/beat.py index 727d7d4f4..24b6828d8 100644 --- a/celery/apps/beat.py +++ b/celery/apps/beat.py @@ -48,15 +48,16 @@ def __init__(self, max_interval=None, app=None, redirect_stdouts_level=None, **kwargs): """Starts the beat task scheduler.""" self.app = app = app or self.app + either = self.app.either self.loglevel = loglevel self.logfile = logfile - self.schedule = self._getopt('schedule_filename', schedule) - self.scheduler_cls = self._getopt('scheduler', scheduler_cls) - self.redirect_stdouts = self._getopt( - 'redirect_stdouts', redirect_stdouts, + self.schedule = either('beat_schedule_filename', schedule) + self.scheduler_cls = either('beat_scheduler', scheduler_cls) + self.redirect_stdouts = either( + 'worker_redirect_stdouts', redirect_stdouts, ) - self.redirect_stdouts_level = self._getopt( - 'redirect_stdouts_level', redirect_stdouts_level, + self.redirect_stdouts_level = either( + 'worker_redirect_stdouts_level', redirect_stdouts_level, ) self.max_interval = max_interval @@ -71,11 +72,6 @@ def __init__(self, max_interval=None, app=None, if not isinstance(self.loglevel, numbers.Integral): self.loglevel = LOG_LEVELS[self.loglevel.upper()] - def _getopt(self, key, value): - if value is not None: - return value - return self.app.conf.find_value_for_key(key, namespace='celerybeat') - def run(self): print(str(self.colored.cyan( 'celery beat v{0} is starting.'.format(VERSION_BANNER)))) diff --git a/celery/apps/worker.py b/celery/apps/worker.py index cfb302795..0cdf0fdb8 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -57,7 +57,7 @@ warning and to be sure that everything will continue working when you upgrade to Celery 4.0:: - CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml'] + accept_content = ['pickle', 'json', 'msgpack', 'yaml'] You must only enable the serializers that you will actually use. @@ -120,16 +120,16 @@ def on_before_init(self, **kwargs): sender=self.hostname, instance=self, conf=self.app.conf, options=kwargs, ) - check_privileges(self.app.conf.CELERY_ACCEPT_CONTENT) + check_privileges(self.app.conf.accept_content) def on_after_init(self, purge=False, no_color=None, redirect_stdouts=None, redirect_stdouts_level=None, **kwargs): - self.redirect_stdouts = self._getopt( - 'redirect_stdouts', redirect_stdouts, + self.redirect_stdouts = self.app.either( + 'worker_redirect_stdouts', redirect_stdouts, ) - self.redirect_stdouts_level = self._getopt( - 'redirect_stdouts_level', redirect_stdouts_level, + self.redirect_stdouts_level = self.app.either( + 'worker_redirect_stdouts_level', redirect_stdouts_level, ) super(Worker, self).setup_defaults(**kwargs) self.purge = purge @@ -158,7 +158,7 @@ def on_start(self): sender=self.hostname, instance=self, conf=self.app.conf, ) - if not self.app.conf.value_set_for('CELERY_ACCEPT_CONTENT'): + if not self.app.conf.value_set_for('accept_content'): warnings.warn(CDeprecationWarning(W_PICKLE_DEPRECATED)) if self.purge: @@ -229,7 +229,7 @@ def startup_info(self): version=VERSION_BANNER, conninfo=self.app.connection().as_uri(), results=maybe_sanitize_url( - self.app.conf.CELERY_RESULT_BACKEND or 'disabled', + self.app.conf.result_backend or 'disabled', ), concurrency=concurrency, platform=safe_str(_platform.platform()), @@ -281,7 +281,6 @@ def set_process_status(self, info): def _shutdown_handler(worker, sig='TERM', how='Warm', exc=WorkerShutdown, callback=None, exitcode=EX_OK): - def _handle_request(*args): with in_sighandler(): from celery.worker import state diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index 4871e0623..16cc9dd7b 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -73,12 +73,12 @@ def __init__(self, app, connection=None, exchange=None, exchange_type=None, self._connection = connection self.persistent = self.prepare_persistent(persistent) self.delivery_mode = 2 if self.persistent else 1 - exchange = exchange or conf.CELERY_RESULT_EXCHANGE - exchange_type = exchange_type or conf.CELERY_RESULT_EXCHANGE_TYPE + exchange = exchange or conf.result_exchange + exchange_type = exchange_type or conf.result_exchange_type self.exchange = self._create_exchange( exchange, exchange_type, self.delivery_mode, ) - self.serializer = serializer or conf.CELERY_RESULT_SERIALIZER + self.serializer = serializer or conf.result_serializer self.auto_delete = auto_delete self.queue_arguments = dictfilter({ 'x-expires': maybe_s_to_ms(self.expires), diff --git a/celery/backends/base.py b/celery/backends/base.py index e03432f30..05cd82a9f 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -96,16 +96,16 @@ def __init__(self, app, expires=None, expires_type=None, **kwargs): self.app = app conf = self.app.conf - self.serializer = serializer or conf.CELERY_RESULT_SERIALIZER + self.serializer = serializer or conf.result_serializer (self.content_type, self.content_encoding, self.encoder) = serializer_registry._encoders[self.serializer] - cmax = max_cached_results or conf.CELERY_MAX_CACHED_RESULTS + cmax = max_cached_results or conf.result_cache_max self._cache = _nulldict() if cmax == -1 else LRUCache(limit=cmax) self.expires = self.prepare_expires(expires, expires_type) self.accept = prepare_accept_content( - conf.CELERY_ACCEPT_CONTENT if accept is None else accept, + conf.accept_content if accept is None else accept, ) def mark_as_started(self, task_id, **meta): @@ -242,7 +242,7 @@ def wait_for(self, task_id, def prepare_expires(self, value, type=None): if value is None: - value = self.app.conf.CELERY_TASK_RESULT_EXPIRES + value = self.app.conf.result_expires if isinstance(value, timedelta): value = value.total_seconds() if value is not None and type: @@ -252,7 +252,7 @@ def prepare_expires(self, value, type=None): def prepare_persistent(self, enabled=None): if enabled is not None: return enabled - p = self.app.conf.CELERY_RESULT_PERSISTENT + p = self.app.conf.result_persistent return self.persistent if p is None else p def encode_result(self, result, status): @@ -558,7 +558,7 @@ def on_chord_part_return(self, request, state, result, propagate=None): return app = self.app if propagate is None: - propagate = app.conf.CELERY_CHORD_PROPAGATES + propagate = app.conf.chord_propagates gid = request.group if not gid: return diff --git a/celery/backends/cache.py b/celery/backends/cache.py index b9480fb31..8736d6765 100644 --- a/celery/backends/cache.py +++ b/celery/backends/cache.py @@ -101,10 +101,10 @@ def __init__(self, app, expires=None, backend=None, options={}, url=None, **kwargs): super(CacheBackend, self).__init__(app, **kwargs) - self.options = dict(self.app.conf.CELERY_CACHE_BACKEND_OPTIONS, + self.options = dict(self.app.conf.cache_backend_options, **options) - self.backend = url or backend or self.app.conf.CELERY_CACHE_BACKEND + self.backend = url or backend or self.app.conf.cache_backend if self.backend: self.backend, _, servers = self.backend.partition('://') self.servers = servers.rstrip('/').split(';') diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index 07c5880eb..991c73d69 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -59,7 +59,7 @@ def __init__(self, servers=None, keyspace=None, column_family=None, """Initialize Cassandra backend. Raises :class:`celery.exceptions.ImproperlyConfigured` if - the :setting:`CASSANDRA_SERVERS` setting is not set. + the :setting:`cassandra_servers` setting is not set. """ super(CassandraBackend, self).__init__(**kwargs) @@ -71,21 +71,21 @@ def __init__(self, servers=None, keyspace=None, column_family=None, conf = self.app.conf self.servers = (servers or - conf.get('CASSANDRA_SERVERS') or + conf.get('cassandra_servers') or self.servers) self.keyspace = (keyspace or - conf.get('CASSANDRA_KEYSPACE') or + conf.get('cassandra_keyspace') or self.keyspace) self.column_family = (column_family or - conf.get('CASSANDRA_COLUMN_FAMILY') or + conf.get('cassandra_column_family') or self.column_family) - self.cassandra_options = dict(conf.get('CASSANDRA_OPTIONS') or {}, + self.cassandra_options = dict(conf.get('cassandra_options') or {}, **cassandra_options or {}) self.detailed_mode = (detailed_mode or - conf.get('CASSANDRA_DETAILED_MODE') or + conf.get('cassandra_detailed_mode') or self.detailed_mode) - read_cons = conf.get('CASSANDRA_READ_CONSISTENCY') or 'LOCAL_QUORUM' - write_cons = conf.get('CASSANDRA_WRITE_CONSISTENCY') or 'LOCAL_QUORUM' + read_cons = conf.get('cassandra_read_consistency') or 'LOCAL_QUORUM' + write_cons = conf.get('cassandra_write_consistency') or 'LOCAL_QUORUM' try: self.read_consistency = getattr(pycassa.ConsistencyLevel, read_cons) diff --git a/celery/backends/couchbase.py b/celery/backends/couchbase.py index 793a69d88..1cf9a7b59 100644 --- a/celery/backends/couchbase.py +++ b/celery/backends/couchbase.py @@ -63,7 +63,7 @@ def __init__(self, url=None, *args, **kwargs): _, uhost, uport, uname, upass, ubucket, _ = _parse_url(url) ubucket = ubucket.strip('/') if ubucket else None - config = self.app.conf.get('CELERY_COUCHBASE_BACKEND_SETTINGS', None) + config = self.app.conf.get('couchbase_backend_settings', None) if config is not None: if not isinstance(config, dict): raise ImproperlyConfigured( diff --git a/celery/backends/database/__init__.py b/celery/backends/database/__init__.py index 86bb4189c..508f3413f 100644 --- a/celery/backends/database/__init__.py +++ b/celery/backends/database/__init__.py @@ -80,23 +80,23 @@ def __init__(self, dburi=None, engine_options=None, url=None, **kwargs): expires_type=maybe_timedelta, **kwargs ) conf = self.app.conf - self.dburi = url or dburi or conf.CELERY_RESULT_DBURI + self.dburi = url or dburi or conf.sqlalchemy_dburi self.engine_options = dict( engine_options or {}, - **conf.CELERY_RESULT_ENGINE_OPTIONS or {}) + **conf.sqlalchemy_engine_options or {}) self.short_lived_sessions = kwargs.get( 'short_lived_sessions', - conf.CELERY_RESULT_DB_SHORT_LIVED_SESSIONS, + conf.sqlalchemy_short_lived_sessions, ) - tablenames = conf.CELERY_RESULT_DB_TABLENAMES or {} + tablenames = conf.sqlalchemy_table_names or {} Task.__table__.name = tablenames.get('task', 'celery_taskmeta') TaskSet.__table__.name = tablenames.get('group', 'celery_tasksetmeta') if not self.dburi: raise ImproperlyConfigured( - 'Missing connection string! Do you have ' - 'CELERY_RESULT_DBURI set to a real value?') + 'Missing connection string! Do you have the' + ' sqlalchemy_dburi setting set to a real value?') def ResultSession(self, session_manager=SessionManager()): return session_manager.session_factory( diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 5a57ffccc..bd1075ba7 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -98,7 +98,7 @@ def __init__(self, app=None, url=None, **kwargs): self.options.update(uri_data['options']) # update conf with specific settings - config = self.app.conf.get('CELERY_MONGODB_BACKEND_SETTINGS') + config = self.app.conf.get('mongodb_backend_settings') if config is not None: if not isinstance(config, dict): raise ImproperlyConfigured( diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index 02610c887..48079e02f 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -83,7 +83,7 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, """Initialize Cassandra backend. Raises :class:`celery.exceptions.ImproperlyConfigured` if - the :setting:`CASSANDRA_SERVERS` setting is not set. + the :setting:`cassandra_servers` setting is not set. """ super(CassandraBackend, self).__init__(**kwargs) @@ -93,24 +93,24 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, conf = self.app.conf self.servers = (servers or - conf.get('CASSANDRA_SERVERS', None)) + conf.get('cassandra_servers', None)) self.port = (port or - conf.get('CASSANDRA_PORT', None)) + conf.get('cassandra_port', None)) self.keyspace = (keyspace or - conf.get('CASSANDRA_KEYSPACE', None)) + conf.get('cassandra_keyspace', None)) self.table = (table or - conf.get('CASSANDRA_TABLE', None)) + conf.get('cassandra_table', None)) if not self.servers or not self.keyspace or not self.table: raise ImproperlyConfigured('Cassandra backend not configured.') - expires = (entry_ttl or conf.get('CASSANDRA_ENTRY_TTL', None)) + expires = (entry_ttl or conf.get('cassandra_entry_ttl', None)) self.cqlexpires = (Q_EXPIRES.format(expires) if expires is not None else '') - read_cons = conf.get('CASSANDRA_READ_CONSISTENCY') or 'LOCAL_QUORUM' - write_cons = conf.get('CASSANDRA_WRITE_CONSISTENCY') or 'LOCAL_QUORUM' + read_cons = conf.get('cassandra_read_consistency') or 'LOCAL_QUORUM' + write_cons = conf.get('cassandra_write_consistency') or 'LOCAL_QUORUM' self.read_consistency = getattr( cassandra.ConsistencyLevel, read_cons, diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 8afc33aaf..bf9d0e770 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -60,32 +60,27 @@ def __init__(self, host=None, port=None, db=None, password=None, max_connections=None, url=None, connection_pool=None, new_join=False, **kwargs): super(RedisBackend, self).__init__(expires_type=int, **kwargs) - conf = self.app.conf + _get = self.app.conf.get if self.redis is None: raise ImproperlyConfigured(REDIS_MISSING) - # For compatibility with the old REDIS_* configuration keys. - def _get(key): - for prefix in 'CELERY_REDIS_{0}', 'REDIS_{0}': - try: - return conf[prefix.format(key)] - except KeyError: - pass if host and '://' in host: url = host host = None self.max_connections = ( - max_connections or _get('MAX_CONNECTIONS') or self.max_connections + max_connections or + _get('redis_max_connections') or + self.max_connections ) self._ConnectionPool = connection_pool self.connparams = { - 'host': _get('HOST') or 'localhost', - 'port': _get('PORT') or 6379, - 'db': _get('DB') or 0, - 'password': _get('PASSWORD'), - 'socket_timeout': _get('SOCKET_TIMEOUT'), + 'host': _get('redis_host') or 'localhost', + 'port': _get('redis_port') or 6379, + 'db': _get('redis_db') or 0, + 'password': _get('redis_password'), + 'socket_timeout': _get('redis_socket_timeout'), 'max_connections': self.max_connections, } if url: diff --git a/celery/backends/riak.py b/celery/backends/riak.py index 5e4565ede..005be46b9 100644 --- a/celery/backends/riak.py +++ b/celery/backends/riak.py @@ -85,7 +85,7 @@ def __init__(self, host=None, port=None, bucket_name=None, protocol=None, if ubucket: ubucket = ubucket.strip('/') - config = self.app.conf.get('CELERY_RIAK_BACKEND_SETTINGS', None) + config = self.app.conf.get('riak_backend_settings', None) if config is not None: if not isinstance(config, dict): raise ImproperlyConfigured( diff --git a/celery/beat.py b/celery/beat.py index 9dbd4386f..2c63f12e0 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -186,19 +186,19 @@ def __init__(self, app, schedule=None, max_interval=None, self.app = app self.data = maybe_evaluate({} if schedule is None else schedule) self.max_interval = (max_interval or - app.conf.CELERYBEAT_MAX_LOOP_INTERVAL or + app.conf.beat_max_loop_interval or self.max_interval) self.Producer = Producer or app.amqp.Producer self._heap = None self.sync_every_tasks = ( - app.conf.CELERYBEAT_SYNC_EVERY if sync_every_tasks is None + app.conf.beat_sync_every if sync_every_tasks is None else sync_every_tasks) if not lazy: self.setup_schedule() def install_default_entries(self, data): entries = {} - if self.app.conf.CELERY_TASK_RESULT_EXPIRES and \ + if self.app.conf.result_expires and \ not self.app.backend.supports_autoexpire: if 'celery.backend_cleanup' not in data: entries['celery.backend_cleanup'] = { @@ -363,7 +363,7 @@ def _error_handler(exc, interval): 'Trying again in %s seconds...', exc, interval) return self.connection.ensure_connection( - _error_handler, self.app.conf.BROKER_CONNECTION_MAX_RETRIES + _error_handler, self.app.conf.broker_connection_max_retries ) def get_schedule(self): @@ -438,12 +438,12 @@ def setup_schedule(self): self._store.clear() # remove schedule at 3.0.9 upgrade break - tz = self.app.conf.CELERY_TIMEZONE + tz = self.app.conf.timezone stored_tz = self._store.get('tz') if stored_tz is not None and stored_tz != tz: warning('Reset: Timezone changed from %r to %r', stored_tz, tz) self._store.clear() # Timezone changed, reset db! - utc = self.app.conf.CELERY_ENABLE_UTC + utc = self.app.conf.enable_utc stored_utc = self._store.get('utc_enabled') if stored_utc is not None and stored_utc != utc: choices = {True: 'enabled', False: 'disabled'} @@ -451,7 +451,7 @@ def setup_schedule(self): choices[stored_utc], choices[utc]) self._store.clear() # UTC setting changed, reset db! entries = self._store.setdefault('entries', {}) - self.merge_inplace(self.app.conf.CELERYBEAT_SCHEDULE) + self.merge_inplace(self.app.conf.beat_schedule) self.install_default_entries(self.schedule) self._store.update(__version__=__version__, tz=tz, utc_enabled=utc) self.sync() @@ -485,10 +485,10 @@ def __init__(self, app, max_interval=None, schedule_filename=None, scheduler_cls=None): self.app = app self.max_interval = (max_interval or - app.conf.CELERYBEAT_MAX_LOOP_INTERVAL) + app.conf.beat_max_loop_interval) self.scheduler_cls = scheduler_cls or self.scheduler_cls self.schedule_filename = ( - schedule_filename or app.conf.CELERYBEAT_SCHEDULE_FILENAME) + schedule_filename or app.conf.beat_schedule_filename) self._is_shutdown = Event() self._is_stopped = Event() diff --git a/celery/bin/base.py b/celery/bin/base.py index 7c029d0f9..d39dee309 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -219,7 +219,7 @@ class Command(object): enable_config_from_cmdline = False #: Default configuration namespace. - namespace = 'celery' + namespace = None #: Text to print at end of --help epilog = None diff --git a/celery/bin/beat.py b/celery/bin/beat.py index c8041217b..f203b3b47 100644 --- a/celery/bin/beat.py +++ b/celery/bin/beat.py @@ -84,7 +84,7 @@ def get_options(self): return ( (Option('--detach', action='store_true'), Option('-s', '--schedule', - default=c.CELERYBEAT_SCHEDULE_FILENAME), + default=c.beat_schedule_filename), Option('--max-interval', type='float'), Option('-S', '--scheduler', dest='scheduler_cls'), Option('-l', '--loglevel', default='WARN')) + diff --git a/celery/bin/celery.py b/celery/bin/celery.py index 54a9f5bb8..3df1966c6 100644 --- a/celery/bin/celery.py +++ b/celery/bin/celery.py @@ -660,7 +660,6 @@ def run(self, *args, **kwargs): class CeleryCommand(Command): - namespace = 'celery' ext_fmt = '{self.namespace}.commands' commands = { 'amqp': amqp, diff --git a/celery/bin/graph.py b/celery/bin/graph.py index d8aa31187..5216ab0ab 100644 --- a/celery/bin/graph.py +++ b/celery/bin/graph.py @@ -156,7 +156,7 @@ def maybe_abbr(l, name, max=Wmax): threads.append(reply['pool']['max-concurrency']) wlen = len(workers) - backend = args.get('backend', self.app.conf.CELERY_RESULT_BACKEND) + backend = args.get('backend', self.app.conf.result_backend) threads_for = {} workers = maybe_abbr(workers, 'Worker') if Wmax and wlen > Wmax: diff --git a/celery/bin/worker.py b/celery/bin/worker.py index b3492cb0c..b1648c98d 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -173,7 +173,7 @@ class worker(Command): celery worker --autoscale=10,0 """ doc = __MODULE_DOC__ # parse help from this too - namespace = 'celeryd' + namespace = 'worker' enable_config_from_cmdline = True supports_args = False @@ -200,7 +200,7 @@ def run(self, hostname=None, pool_cls=None, app=None, uid=None, gid=None, # Pools like eventlet/gevent needs to patch libs as early # as possible. pool_cls = (concurrency.get_implementation(pool_cls) or - self.app.conf.CELERYD_POOL) + self.app.conf.worker_pool) if self.app.IS_WINDOWS and kwargs.get('beat'): self.die('-B option does not work on Windows. ' 'Please run celery beat as a separate service.') @@ -231,29 +231,29 @@ def get_options(self): conf = self.app.conf return ( Option('-c', '--concurrency', - default=conf.CELERYD_CONCURRENCY, type='int'), - Option('-P', '--pool', default=conf.CELERYD_POOL, dest='pool_cls'), + default=conf.worker_concurrency, type='int'), + Option('-P', '--pool', default=conf.worker_pool, dest='pool_cls'), Option('--purge', '--discard', default=False, action='store_true'), Option('-l', '--loglevel', default='WARN'), Option('-n', '--hostname'), Option('-B', '--beat', action='store_true'), Option('-s', '--schedule', dest='schedule_filename', - default=conf.CELERYBEAT_SCHEDULE_FILENAME), + default=conf.beat_schedule_filename), Option('--scheduler', dest='scheduler_cls'), Option('-S', '--statedb', - default=conf.CELERYD_STATE_DB, dest='state_db'), - Option('-E', '--events', default=conf.CELERY_SEND_EVENTS, + default=conf.worker_state_db, dest='state_db'), + Option('-E', '--events', default=conf.worker_send_events, action='store_true', dest='send_events'), Option('--time-limit', type='float', dest='task_time_limit', - default=conf.CELERYD_TASK_TIME_LIMIT), + default=conf.task_time_limit), Option('--soft-time-limit', dest='task_soft_time_limit', - default=conf.CELERYD_TASK_SOFT_TIME_LIMIT, type='float'), + default=conf.task_soft_time_limit, type='float'), Option('--maxtasksperchild', dest='max_tasks_per_child', - default=conf.CELERYD_MAX_TASKS_PER_CHILD, type='int'), + default=conf.worker_max_tasks_per_child, type='int'), Option('--prefetch-multiplier', dest='prefetch_multiplier', - default=conf.CELERYD_PREFETCH_MULTIPLIER, type='int'), + default=conf.worker_prefetch_multiplier, type='int'), Option('--maxmemperchild', dest='max_memory_per_child', - default=conf.CELERYD_MAX_MEMORY_PER_CHILD, type='int'), + default=conf.worker_max_memory_per_child, type='int'), Option('--queues', '-Q', default=[]), Option('--exclude-queues', '-X', default=[]), Option('--include', '-I', default=[]), diff --git a/celery/canvas.py b/celery/canvas.py index adb7aa465..2f9cb4483 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -393,7 +393,7 @@ def __call__(self, *args, **kwargs): def apply_async(self, args=(), kwargs={}, **options): # python is best at unpacking kwargs, so .run is here to do that. app = self.app - if app.conf.CELERY_ALWAYS_EAGER: + if app.conf.task_always_eager: return self.apply(args, kwargs, **options) return self.run(args, kwargs, app=app, **( dict(self.options, **options) if options else self.options)) @@ -688,7 +688,7 @@ def _freeze_gid(self, options): def apply_async(self, args=(), kwargs=None, add_to_parent=True, producer=None, **options): app = self.app - if app.conf.CELERY_ALWAYS_EAGER: + if app.conf.task_always_eager: return self.apply(args, kwargs, **options) if not self.tasks: return self.freeze() @@ -846,7 +846,7 @@ def apply_async(self, args=(), kwargs={}, task_id=None, app = self._get_app(body) tasks = (self.tasks.clone() if isinstance(self.tasks, group) else group(self.tasks)) - if app.conf.CELERY_ALWAYS_EAGER: + if app.conf.task_always_eager: return self.apply((), kwargs, body=body, task_id=task_id, **options) return self.run(tasks, body, args, task_id=task_id, **options) @@ -875,7 +875,7 @@ def run(self, header, body, partial_args, app=None, interval=None, countdown=1, max_retries=None, propagate=None, eager=False, task_id=None, **options): app = app or self._get_app(body) - propagate = (app.conf.CELERY_CHORD_PROPAGATES + propagate = (app.conf.chord_propagates if propagate is None else propagate) group_id = uuid() root_id = body.options.get('root_id') diff --git a/celery/contrib/batches.py b/celery/contrib/batches.py index a476387d1..6a0858b08 100644 --- a/celery/contrib/batches.py +++ b/celery/contrib/batches.py @@ -8,7 +8,7 @@ .. warning:: For this to work you have to set - :setting:`CELERYD_PREFETCH_MULTIPLIER` to zero, or some value where + :setting:`worker_prefetch_multiplier` to zero, or some value where the final multiplied value is higher than ``flush_every``. In the future we hope to add the ability to direct batching tasks diff --git a/celery/contrib/migrate.py b/celery/contrib/migrate.py index c829cdb5a..8919d9b9f 100644 --- a/celery/contrib/migrate.py +++ b/celery/contrib/migrate.py @@ -141,7 +141,7 @@ def move(predicate, connection=None, exchange=None, routing_key=None, :keyword connection: Custom connection to use. :keyword source: Optional list of source queues to use instead of the - default (which is the queues in :setting:`CELERY_QUEUES`). + default (which is the queues in :setting:`task_queues`). This list can also contain new :class:`~kombu.entity.Queue` instances. :keyword exchange: Default destination exchange. :keyword routing_key: Default destination routing key. diff --git a/celery/datastructures.py b/celery/datastructures.py index 84c393c9f..cc4330870 100644 --- a/celery/datastructures.py +++ b/celery/datastructures.py @@ -387,11 +387,8 @@ def get(self, key, default=None): return default def setdefault(self, key, default): - try: - return self[key] - except KeyError: + if key not in self: self[key] = default - return default def __getitem__(self, key): try: @@ -451,13 +448,27 @@ class ConfigurationView(AttributeDictMixin): :param defaults: List of dicts containing the default configuration. """ + key_t = None changes = None defaults = None _order = None - def __init__(self, changes, defaults): - self.__dict__.update(changes=changes, defaults=defaults, - _order=[changes] + defaults) + def __init__(self, changes, defaults=None, key_t=None, prefix=None): + defaults = [] if defaults is None else defaults + self.__dict__.update( + changes=changes, + defaults=defaults, + key_t=key_t, + _order=[changes] + defaults, + prefix=prefix, + ) + + def _key(self, key): + if self.prefix: + key = self.prefix + key + if self.prefix.isupper(): + key = key.upper() + return self.key_t(key) if self.key_t is not None else key def add_defaults(self, d): d = force_mapping(d) @@ -465,6 +476,7 @@ def add_defaults(self, d): self._order.insert(1, d) def __getitem__(self, key): + key = self._key(key) for d in self._order: try: return d[key] @@ -473,14 +485,14 @@ def __getitem__(self, key): raise KeyError(key) def __setitem__(self, key, value): - self.changes[key] = value + self.changes[self._key(key)] = value def first(self, *keys): - return first(None, (self.get(key) for key in keys)) + return first(None, (self.get(self._key(key)) for key in keys)) def get(self, key, default=None): try: - return self[key] + return self[self._key(key)] except KeyError: return default @@ -489,16 +501,15 @@ def clear(self): self.changes.clear() def setdefault(self, key, default): - try: - return self[key] - except KeyError: + key = self._key(key) + if key not in self: self[key] = default - return default def update(self, *args, **kwargs): return self.changes.update(*args, **kwargs) def __contains__(self, key): + key = self._key(key) return any(key in m for m in self._order) def __bool__(self): @@ -521,8 +532,19 @@ def _iter(self, op): # changes takes precedence. return chain(*[op(d) for d in reversed(self._order)]) + def swap_with(self, other): + changes = other.__dict__['changes'] + defaults = other.__dict__['defaults'] + self.__dict__.update( + changes=changes, + defaults=defaults, + key_t=other.__dict__['key_t'], + prefix=other.__dict__['prefix'], + _order=[changes] + defaults + ) + def _iterate_keys(self): - return uniq(self._iter(lambda d: d)) + return uniq(self._iter(lambda d: d.keys())) iterkeys = _iterate_keys def _iterate_items(self): diff --git a/celery/events/__init__.py b/celery/events/__init__.py index 800a615a5..6a79802cc 100644 --- a/celery/events/__init__.py +++ b/celery/events/__init__.py @@ -4,7 +4,7 @@ ~~~~~~~~~~~~~ Events is a stream of messages sent for certain actions occurring - in the worker (and clients if :setting:`CELERY_SEND_TASK_SENT_EVENT` + in the worker (and clients if :setting:`task_send_sent_event` is enabled), used for monitoring purposes. """ @@ -130,7 +130,7 @@ def __init__(self, connection=None, hostname=None, enabled=True, self.mutex = threading.Lock() self.producer = None self._outbound_buffer = deque() - self.serializer = serializer or self.app.conf.CELERY_EVENT_SERIALIZER + self.serializer = serializer or self.app.conf.event_serializer self.on_enabled = set() self.on_disabled = set() self.groups = set(groups or []) @@ -321,18 +321,18 @@ def __init__(self, channel, handlers=None, routing_key='#', self.adjust_clock = self.clock.adjust self.forward_clock = self.clock.forward if accept is None: - accept = {self.app.conf.CELERY_EVENT_SERIALIZER, 'json'} + accept = {self.app.conf.event_serializer, 'json'} self.accept = accept def _get_queue_arguments(self, ttl=None, expires=None): conf = self.app.conf return dictfilter({ 'x-message-ttl': maybe_s_to_ms( - ttl if ttl is not None else conf.CELERY_EVENT_QUEUE_TTL, + ttl if ttl is not None else conf.event_queue_ttl, ), 'x-expires': maybe_s_to_ms( expires if expires is not None - else conf.CELERY_EVENT_QUEUE_EXPIRES, + else conf.event_queue_expires, ), }) diff --git a/celery/events/cursesmon.py b/celery/events/cursesmon.py index 3ac164fa7..923ca8a2d 100644 --- a/celery/events/cursesmon.py +++ b/celery/events/cursesmon.py @@ -511,7 +511,7 @@ def on_connection_error(exc, interval): with app.connection() as conn: try: conn.ensure_connection(on_connection_error, - app.conf.BROKER_CONNECTION_MAX_RETRIES) + app.conf.broker_connection_max_retries) recv = app.events.Receiver(conn, handlers={'*': state.event}) display.resetscreen() display.init_screen() diff --git a/celery/exceptions.py b/celery/exceptions.py index 39e764918..fcd40d1be 100644 --- a/celery/exceptions.py +++ b/celery/exceptions.py @@ -120,7 +120,7 @@ class WorkerShutdown(SystemExit): class QueueNotFound(KeyError): - """Task routed to a queue not in CELERY_QUEUES.""" + """Task routed to a queue not in ``conf.queues``.""" class ImproperlyConfigured(ImportError): @@ -155,7 +155,7 @@ class NotConfigured(CeleryWarning): class AlwaysEagerIgnored(CeleryWarning): - """send_task ignores CELERY_ALWAYS_EAGER option""" + """send_task ignores :setting:`task_always_eager` option""" class InvalidTaskError(CeleryError): diff --git a/celery/loaders/base.py b/celery/loaders/base.py index b1a1f6366..39699689b 100644 --- a/celery/loaders/base.py +++ b/celery/loaders/base.py @@ -40,6 +40,8 @@ Did you mean '{suggest}'? """ +unconfigured = object() + class BaseLoader(object): """The base class for loaders. @@ -65,7 +67,7 @@ class BaseLoader(object): override_backends = {} worker_initialized = False - _conf = None + _conf = unconfigured def __init__(self, app, **kwargs): self.app = app @@ -117,8 +119,8 @@ def import_default_modules(self): return [ self.import_task_module(m) for m in ( tuple(self.builtin_modules) + - tuple(maybe_list(self.app.conf.CELERY_IMPORTS)) + - tuple(maybe_list(self.app.conf.CELERY_INCLUDE)) + tuple(maybe_list(self.app.conf.imports)) + + tuple(maybe_list(self.app.conf.include)) ) ] @@ -183,7 +185,7 @@ def cmdline_config_parser( 'list': 'json', 'dict': 'json'}): from celery.app.defaults import Option, NAMESPACES - namespace = namespace.upper() + namespace = namespace.lower() typemap = dict(Option.typemap, **extra_types) def getarg(arg): @@ -193,7 +195,7 @@ def getarg(arg): # ## find key/value # ns.key=value|ns_key=value (case insensitive) key, value = arg.split('=', 1) - key = key.upper().replace('.', '_') + key = key.lower().replace('.', '_') # ## find namespace. # .key=value|_key=value expands to default namespace. @@ -214,7 +216,7 @@ def getarg(arg): value = typemap[type_](value) else: try: - value = NAMESPACES[ns][key].to_python(value) + value = NAMESPACES[ns.lower()][key].to_python(value) except ValueError as exc: # display key name in error message. raise ValueError('{0!r}: {1}'.format(ns_key, exc)) @@ -244,7 +246,6 @@ def read_configuration(self, env='CELERY_CONFIG_MODULE'): if custom_config: usercfg = self._import_config_module(custom_config) return DictAttribute(usercfg) - return {} def autodiscover_tasks(self, packages, related_name='tasks'): self.task_modules.update( @@ -254,7 +255,7 @@ def autodiscover_tasks(self, packages, related_name='tasks'): @property def conf(self): """Loader configuration.""" - if self._conf is None: + if self._conf is unconfigured: self._conf = self.read_configuration() return self._conf diff --git a/celery/schedules.py b/celery/schedules.py index 2c7ce96ea..6b03e59d1 100644 --- a/celery/schedules.py +++ b/celery/schedules.py @@ -111,7 +111,7 @@ def is_due(self, last_run_at): The next time to check is used to save energy/cpu cycles, it does not need to be accurate but will influence the precision of your schedule. You must also keep in mind - the value of :setting:`CELERYBEAT_MAX_LOOP_INTERVAL`, + the value of :setting:`beat_max_loop_interval`, which decides the maximum number of seconds the scheduler can sleep between re-checking the periodic task intervals. So if you have a task that changes schedule at runtime then your next_run_at @@ -172,7 +172,7 @@ def tz(self): @cached_property def utc_enabled(self): - return self.app.conf.CELERY_ENABLE_UTC + return self.app.conf.enable_utc def to_local(self, dt): if not self.utc_enabled: diff --git a/celery/security/__init__.py b/celery/security/__init__.py index 352d400cf..8366ad7f3 100644 --- a/celery/security/__init__.py +++ b/celery/security/__init__.py @@ -25,9 +25,9 @@ SETTING_MISSING = """\ Sorry, but you have to configure the - * CELERY_SECURITY_KEY - * CELERY_SECURITY_CERTIFICATE, and the - * CELERY_SECURITY_CERT_STORE + * security_key + * security_certificate, and the + * security_cert_storE configuration settings to use the auth serializer. Please see the configuration reference for more information. @@ -46,7 +46,7 @@ def setup_security(allowed_serializers=None, key=None, cert=None, store=None, _disable_insecure_serializers(allowed_serializers) conf = app.conf - if conf.CELERY_TASK_SERIALIZER != 'auth': + if conf.task_serializer != 'auth': return try: @@ -54,9 +54,9 @@ def setup_security(allowed_serializers=None, key=None, cert=None, store=None, except ImportError: raise ImproperlyConfigured(SSL_NOT_INSTALLED) - key = key or conf.CELERY_SECURITY_KEY - cert = cert or conf.CELERY_SECURITY_CERTIFICATE - store = store or conf.CELERY_SECURITY_CERT_STORE + key = key or conf.security_key + cert = cert or conf.security_certificate + store = store or conf.security_cert_store if not (key and cert and store): raise ImproperlyConfigured(SETTING_MISSING) diff --git a/celery/states.py b/celery/states.py index 054b448db..592c08b5f 100644 --- a/celery/states.py +++ b/celery/states.py @@ -128,7 +128,7 @@ def __le__(self, other): PENDING = 'PENDING' #: Task was received by a worker. RECEIVED = 'RECEIVED' -#: Task was started by a worker (:setting:`CELERY_TRACK_STARTED`). +#: Task was started by a worker (:setting:`task_track_started`). STARTED = 'STARTED' #: Task succeeded SUCCESS = 'SUCCESS' diff --git a/celery/task/base.py b/celery/task/base.py index 31a45544c..b248f428a 100644 --- a/celery/task/base.py +++ b/celery/task/base.py @@ -148,8 +148,8 @@ class Task(BaseTask): disable_error_emails = False from_config = BaseTask.from_config + ( - ('exchange_type', 'CELERY_DEFAULT_EXCHANGE_TYPE'), - ('delivery_mode', 'CELERY_DEFAULT_DELIVERY_MODE'), + ('exchange_type', 'task_default_exchange_type'), + ('delivery_mode', 'task_default_delivery_mode'), ) # In old Celery the @task decorator didn't exist, so one would create @@ -244,7 +244,7 @@ def get_consumer(self, connection=None, queues=None, **kwargs): class PeriodicTask(Task): """A periodic task is a task that adds itself to the - :setting:`CELERYBEAT_SCHEDULE` setting.""" + :setting:`beat_schedule` setting.""" abstract = True ignore_result = True relative = False @@ -260,7 +260,7 @@ def __init__(self): @classmethod def on_bound(cls, app): - app.conf.CELERYBEAT_SCHEDULE[cls.name] = { + app.conf.beat_schedule[cls.name] = { 'task': cls.name, 'schedule': cls.run_every, 'args': (), @@ -276,5 +276,5 @@ def task(*args, **kwargs): def periodic_task(*args, **options): - """Deprecated decorator, please use :setting:`CELERYBEAT_SCHEDULE`.""" + """Deprecated decorator, please use :setting:`beat_schedule`.""" return task(**dict({'base': PeriodicTask}, **options)) diff --git a/celery/task/sets.py b/celery/task/sets.py index 7d4355f62..2ea0012c3 100644 --- a/celery/task/sets.py +++ b/celery/task/sets.py @@ -53,7 +53,7 @@ def apply_async(self, connection=None, publisher=None, taskset_id=None): """Apply TaskSet.""" app = self.app - if app.conf.CELERY_ALWAYS_EAGER: + if app.conf.task_always_eager: return self.apply(taskset_id=taskset_id) with app.connection_or_acquire(connection) as conn: diff --git a/celery/tests/app/test_amqp.py b/celery/tests/app/test_amqp.py index 254c594cd..883e8603a 100644 --- a/celery/tests/app/test_amqp.py +++ b/celery/tests/app/test_amqp.py @@ -11,7 +11,7 @@ class test_TaskConsumer(AppCase): def test_accept_content(self): with self.app.pool.acquire(block=True) as conn: - self.app.conf.CELERY_ACCEPT_CONTENT = ['application/json'] + self.app.conf.accept_content = ['application/json'] self.assertEqual( self.app.amqp.TaskConsumer(conn).accept, {'application/json'}, @@ -25,7 +25,7 @@ def test_accept_content(self): class test_ProducerPool(AppCase): def test_setup_nolimit(self): - self.app.conf.BROKER_POOL_LIMIT = None + self.app.conf.broker_pool_limit = None try: delattr(self.app, '_pool') except AttributeError: @@ -43,7 +43,7 @@ def test_setup_nolimit(self): r2 = pool.acquire() def test_setup(self): - self.app.conf.BROKER_POOL_LIMIT = 2 + self.app.conf.broker_pool_limit = 2 try: delattr(self.app, '_pool') except AttributeError: diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index de0d1f034..b04a3f1a3 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -15,8 +15,8 @@ from celery.app import base as _appbase from celery.app import defaults from celery.exceptions import ImproperlyConfigured -from celery.five import items -from celery.loaders.base import BaseLoader +from celery.five import items, keys +from celery.loaders.base import BaseLoader, unconfigured from celery.platforms import pyimplementation from celery.utils.serialization import pickle @@ -225,7 +225,73 @@ def lazy_list(): @with_environ('CELERY_BROKER_URL', '') def test_with_broker(self): with self.Celery(broker='foo://baribaz') as app: - self.assertEqual(app.conf.BROKER_URL, 'foo://baribaz') + self.assertEqual(app.conf.broker_url, 'foo://baribaz') + + def test_pending_configuration__setattr(self): + with self.Celery(broker='foo://bar') as app: + app.conf.task_default_delivery_mode = 44 + app.conf.worker_agent = 'foo:Bar' + self.assertFalse(app.configured) + self.assertEqual(app.conf.worker_agent, 'foo:Bar') + self.assertEqual(app.conf.broker_url, 'foo://bar') + self.assertEqual(app._preconf['worker_agent'], 'foo:Bar') + + self.assertTrue(app.configured) + reapp = pickle.loads(pickle.dumps(app)) + self.assertEqual(reapp._preconf['worker_agent'], 'foo:Bar') + self.assertFalse(reapp.configured) + self.assertEqual(reapp.conf.worker_agent, 'foo:Bar') + self.assertTrue(reapp.configured) + self.assertEqual(reapp.conf.broker_url, 'foo://bar') + self.assertEqual(reapp._preconf['worker_agent'], 'foo:Bar') + + def test_pending_configuration__update(self): + with self.Celery(broker='foo://bar') as app: + app.conf.update( + task_default_delivery_mode=44, + worker_agent='foo:Bar', + ) + self.assertFalse(app.configured) + self.assertEqual(app.conf.worker_agent, 'foo:Bar') + self.assertEqual(app.conf.broker_url, 'foo://bar') + self.assertEqual(app._preconf['worker_agent'], 'foo:Bar') + + def test_pending_configuration__compat_settings(self): + with self.Celery(broker='foo://bar', backend='foo') as app: + app.conf.update( + CELERY_ALWAYS_EAGER=4, + CELERY_DEFAULT_DELIVERY_MODE=63, + CELERYD_AGENT='foo:Barz', + ) + self.assertEqual(app.conf.task_always_eager, 4) + self.assertEqual(app.conf.task_default_delivery_mode, 63) + self.assertEqual(app.conf.worker_agent, 'foo:Barz') + self.assertEqual(app.conf.broker_url, 'foo://bar') + self.assertEqual(app.conf.result_backend, 'foo') + + def test_pending_configuration__setdefault(self): + with self.Celery(broker='foo://bar') as app: + app.conf.setdefault('worker_agent', 'foo:Bar') + self.assertFalse(app.configured) + + def test_pending_configuration__iter(self): + with self.Celery(broker='foo://bar') as app: + app.conf.worker_agent = 'foo:Bar' + self.assertFalse(app.configured) + self.assertTrue(list(keys(app.conf))) + self.assertFalse(app.configured) + self.assertIn('worker_agent', app.conf) + self.assertFalse(app.configured) + self.assertTrue(dict(app.conf)) + self.assertTrue(app.configured) + + def test_pending_configuration__raises_ImproperlyConfigured(self): + with self.Celery() as app: + app.conf.worker_agent = 'foo://bar' + app.conf.task_default_delivery_mode = 44 + app.conf.CELERY_ALWAYS_EAGER = True + with self.assertRaises(ImproperlyConfigured): + app.finalize() def test_repr(self): self.assertTrue(repr(self.app)) @@ -236,7 +302,7 @@ def test_custom_task_registry(self): def test_include_argument(self): with self.Celery(include=('foo', 'bar.foo')) as app: - self.assertEqual(app.conf.CELERY_IMPORTS, ('foo', 'bar.foo')) + self.assertEqual(app.conf.include, ('foo', 'bar.foo')) def test_set_as_current(self): current = _state._tls.current_app @@ -324,7 +390,7 @@ def _inner(*args, **kwargs): return fun(*args, **kwargs) return _inner - self.app.conf.CELERY_ANNOTATIONS = { + self.app.conf.task_annotations = { adX.name: {'@__call__': deco} } adX.bind(self.app) @@ -416,7 +482,7 @@ def assert_config2(self): def test_config_from_object__lazy(self): conf = ObjectConfig2() self.app.config_from_object(conf) - self.assertFalse(self.app.loader._conf) + self.assertIs(self.app.loader._conf, unconfigured) self.assertIs(self.app._config_source, conf) self.assert_config2() @@ -427,46 +493,110 @@ def test_config_from_object__force(self): self.assert_config2() + def test_config_from_object__compat(self): + + class Config(object): + CELERY_ALWAYS_EAGER = 44 + CELERY_DEFAULT_DELIVERY_MODE = 30 + CELERY_TASK_PUBLISH_RETRY = False + + self.app.config_from_object(Config) + self.assertEqual(self.app.conf.task_always_eager, 44) + self.assertEqual(self.app.conf.CELERY_ALWAYS_EAGER, 44) + self.assertFalse(self.app.conf.task_publish_retry) + self.assertEqual(self.app.conf.task_default_routing_key, 'celery') + + def test_config_from_object__supports_old_names(self): + + class Config(object): + task_always_eager = 44 + task_default_delivery_mode = 301 + + self.app.config_from_object(Config()) + self.assertEqual(self.app.conf.CELERY_ALWAYS_EAGER, 44) + self.assertEqual(self.app.conf.task_always_eager, 44) + self.assertEqual(self.app.conf.CELERY_DEFAULT_DELIVERY_MODE, 301) + self.assertEqual(self.app.conf.task_default_delivery_mode, 301) + self.assertEqual(self.app.conf.task_default_routing_key, 'testcelery') + + def test_config_from_object__namespace_uppercase(self): + + class Config(object): + CELERY_TASK_ALWAYS_EAGER = 44 + CELERY_TASK_DEFAULT_DELIVERY_MODE = 301 + + self.app.config_from_object(Config(), namespace='CELERY_') + self.assertEqual(self.app.conf.task_always_eager, 44) + + def test_config_from_object__namespace_lowercase(self): + + class Config(object): + celery_task_always_eager = 44 + celery_task_default_delivery_mode = 301 + + self.app.config_from_object(Config(), namespace='celery_') + self.assertEqual(self.app.conf.task_always_eager, 44) + + def test_config_from_object__mixing_new_and_old(self): + + class Config(object): + task_always_eager = 44 + worker_agent = 'foo:Agent' + worker_consumer = 'foo:Consumer' + beat_schedule = '/foo/schedule' + CELERY_DEFAULT_DELIVERY_MODE = 301 + + with self.assertRaises(ImproperlyConfigured) as exc: + self.app.config_from_object(Config(), force=True) + self.assertTrue( + exc.args[0].startswith('CELERY_DEFAULT_DELIVERY_MODE')) + self.assertIn('task_default_delivery_mode', exc.args[0]) + + def test_config_from_object__mixing_old_and_new(self): + + class Config(object): + CELERY_ALWAYS_EAGER = 44 + CELERYD_AGENT = 'foo:Agent' + CELERYD_CONSUMER = 'foo:Consumer' + CELERYBEAT_SCHEDULE = '/foo/schedule' + task_default_delivery_mode = 301 + + with self.assertRaises(ImproperlyConfigured) as exc: + self.app.config_from_object(Config(), force=True) + self.assertTrue( + exc.args[0].startswith('task_default_delivery_mode')) + self.assertIn('CELERY_DEFAULT_DELIVERY_MODE', exc.args[0]) + def test_config_from_cmdline(self): - cmdline = ['.always_eager=no', - '.result_backend=/dev/null', - 'celeryd.prefetch_multiplier=368', + cmdline = ['task_always_eager=no', + 'result_backend=/dev/null', + 'worker_prefetch_multiplier=368', '.foobarstring=(string)300', '.foobarint=(int)300', - '.result_engine_options=(dict){"foo": "bar"}'] - self.app.config_from_cmdline(cmdline, namespace='celery') - self.assertFalse(self.app.conf.CELERY_ALWAYS_EAGER) - self.assertEqual(self.app.conf.CELERY_RESULT_BACKEND, '/dev/null') - self.assertEqual(self.app.conf.CELERYD_PREFETCH_MULTIPLIER, 368) - self.assertEqual(self.app.conf.CELERY_FOOBARSTRING, '300') - self.assertEqual(self.app.conf.CELERY_FOOBARINT, 300) - self.assertDictEqual(self.app.conf.CELERY_RESULT_ENGINE_OPTIONS, + 'sqlalchemy_engine_options=(dict){"foo": "bar"}'] + self.app.config_from_cmdline(cmdline, namespace='worker') + self.assertFalse(self.app.conf.task_always_eager) + self.assertEqual(self.app.conf.result_backend, '/dev/null') + self.assertEqual(self.app.conf.worker_prefetch_multiplier, 368) + self.assertEqual(self.app.conf.worker_foobarstring, '300') + self.assertEqual(self.app.conf.worker_foobarint, 300) + self.assertDictEqual(self.app.conf.sqlalchemy_engine_options, {'foo': 'bar'}) - def test_compat_setting_CELERY_BACKEND(self): - self.app._preconf = {} - self.app.conf.defaults[0]['CELERY_RESULT_BACKEND'] = None - self.app.config_from_object(Object(CELERY_BACKEND='set_by_us')) - self.assertEqual(self.app.conf.CELERY_RESULT_BACKEND, 'set_by_us') - - def test_setting_BROKER_TRANSPORT_OPTIONS(self): + def test_setting__broker_transport_options(self): _args = {'foo': 'bar', 'spam': 'baz'} self.app.config_from_object(Object()) - self.assertEqual(self.app.conf.BROKER_TRANSPORT_OPTIONS, {}) + self.assertEqual(self.app.conf.broker_transport_options, {}) - self.app.config_from_object(Object(BROKER_TRANSPORT_OPTIONS=_args)) - self.assertEqual(self.app.conf.BROKER_TRANSPORT_OPTIONS, _args) + self.app.config_from_object(Object(broker_transport_options=_args)) + self.assertEqual(self.app.conf.broker_transport_options, _args) def test_Windows_log_color_disabled(self): self.app.IS_WINDOWS = True self.assertFalse(self.app.log.supports_color(True)) - def test_compat_setting_CARROT_BACKEND(self): - self.app.config_from_object(Object(CARROT_BACKEND='set_by_us')) - self.assertEqual(self.app.conf.BROKER_TRANSPORT, 'set_by_us') - def test_WorkController(self): x = self.app.WorkController self.assertIs(x.app, self.app) @@ -537,9 +667,9 @@ def mail_admins(*args, **kwargs): return args, kwargs self.app.loader = Loader(app=self.app) - self.app.conf.ADMINS = None + self.app.conf.admins = None self.assertFalse(self.app.mail_admins('Subject', 'Body')) - self.app.conf.ADMINS = [('George Costanza', 'george@vandelay.com')] + self.app.conf.admins = [('George Costanza', 'george@vandelay.com')] self.assertTrue(self.app.mail_admins('Subject', 'Body')) def test_amqp_get_broker_info(self): @@ -550,8 +680,8 @@ def test_amqp_get_broker_info(self): 'virtual_host': '/'}, self.app.connection('pyamqp://').info(), ) - self.app.conf.BROKER_PORT = 1978 - self.app.conf.BROKER_VHOST = 'foo' + self.app.conf.broker_port = 1978 + self.app.conf.broker_vhost = 'foo' self.assertDictContainsSubset( {'port': 1978, 'virtual_host': 'foo'}, self.app.connection('pyamqp://:1978/foo').info(), @@ -563,14 +693,14 @@ def test_amqp_get_broker_info(self): def test_amqp_failover_strategy_selection(self): # Test passing in a string and make sure the string # gets there untouched - self.app.conf.BROKER_FAILOVER_STRATEGY = 'foo-bar' + self.app.conf.broker_failover_strategy = 'foo-bar' self.assertEqual( self.app.connection('amqp:////value').failover_strategy, 'foo-bar', ) # Try passing in None - self.app.conf.BROKER_FAILOVER_STRATEGY = None + self.app.conf.broker_failover_strategy = None self.assertEqual( self.app.connection('amqp:////value').failover_strategy, itertools.cycle, @@ -580,16 +710,12 @@ def test_amqp_failover_strategy_selection(self): def my_failover_strategy(it): yield True - self.app.conf.BROKER_FAILOVER_STRATEGY = my_failover_strategy + self.app.conf.broker_failover_strategy = my_failover_strategy self.assertEqual( self.app.connection('amqp:////value').failover_strategy, my_failover_strategy, ) - def test_BROKER_BACKEND_alias(self): - self.assertEqual(self.app.conf.BROKER_BACKEND, - self.app.conf.BROKER_TRANSPORT) - def test_after_fork(self): p = self.app._pool = Mock() self.app._after_fork(self.app) diff --git a/celery/tests/app/test_beat.py b/celery/tests/app/test_beat.py index 0718e2a77..da4638c8a 100644 --- a/celery/tests/app/test_beat.py +++ b/celery/tests/app/test_beat.py @@ -182,7 +182,7 @@ def not_sync(): self.assertFalse(s._do_sync.called) def test_should_sync_increments_sync_every_counter(self): - self.app.conf.CELERYBEAT_SYNC_EVERY = 2 + self.app.conf.beat_sync_every = 2 @self.app.task(shared=False) def not_sync(): @@ -198,10 +198,10 @@ def not_sync(): s.apply_async(s.Entry(task=not_sync.name, app=self.app)) s._do_sync.assert_called_with() - self.app.conf.CELERYBEAT_SYNC_EVERY = 0 + self.app.conf.beat_sync_every = 0 def test_sync_task_counter_resets_on_do_sync(self): - self.app.conf.CELERYBEAT_SYNC_EVERY = 1 + self.app.conf.beat_sync_every = 1 @self.app.task(shared=False) def not_sync(): @@ -214,7 +214,7 @@ def not_sync(): s.apply_async(s.Entry(task=not_sync.name, app=self.app)) self.assertEqual(s._tasks_since_sync, 0) - self.app.conf.CELERYBEAT_SYNC_EVERY = 0 + self.app.conf.beat_sync_every = 0 @patch('celery.app.base.Celery.send_task') def test_send_task(self, send_task): @@ -249,20 +249,20 @@ def test_ensure_connection_error_handler(self, ensure): callback(KeyError(), 5) def test_install_default_entries(self): - self.app.conf.CELERY_TASK_RESULT_EXPIRES = None - self.app.conf.CELERYBEAT_SCHEDULE = {} + self.app.conf.result_expires = None + self.app.conf.beat_schedule = {} s = mScheduler(app=self.app) s.install_default_entries({}) self.assertNotIn('celery.backend_cleanup', s.data) self.app.backend.supports_autoexpire = False - self.app.conf.CELERY_TASK_RESULT_EXPIRES = 30 + self.app.conf.result_expires = 30 s = mScheduler(app=self.app) s.install_default_entries({}) self.assertIn('celery.backend_cleanup', s.data) self.app.backend.supports_autoexpire = True - self.app.conf.CELERY_TASK_RESULT_EXPIRES = 31 + self.app.conf.result_expires = 31 s = mScheduler(app=self.app) s.install_default_entries({}) self.assertNotIn('celery.backend_cleanup', s.data) diff --git a/celery/tests/app/test_builtins.py b/celery/tests/app/test_builtins.py index bb70a8e1f..02f8a2b5c 100644 --- a/celery/tests/app/test_builtins.py +++ b/celery/tests/app/test_builtins.py @@ -83,7 +83,7 @@ def setup(self): def test_apply_async_eager(self): self.task.apply = Mock() - self.app.conf.CELERY_ALWAYS_EAGER = True + self.app.conf.task_always_eager = True self.task.apply_async((1, 2, 3, 4, 5)) self.assertTrue(self.task.apply.called) @@ -208,7 +208,7 @@ def test_forward_options(self): self.assertEqual(resbody.options['chord'], 'some_chord_id') def test_apply_eager(self): - self.app.conf.CELERY_ALWAYS_EAGER = True + self.app.conf.task_always_eager = True x = chord([self.add.s(i, i) for i in range(10)], body=self.xsum.s()) r = x.apply_async() self.assertEqual(r.get(), 90) diff --git a/celery/tests/app/test_defaults.py b/celery/tests/app/test_defaults.py index 61dd4ba33..9cef9b15d 100644 --- a/celery/tests/app/test_defaults.py +++ b/celery/tests/app/test_defaults.py @@ -4,11 +4,13 @@ from importlib import import_module -from celery.app.defaults import NAMESPACES - -from celery.tests.case import ( - AppCase, pypy_version, sys_platform, +from celery.app.defaults import ( + _OLD_DEFAULTS, _OLD_SETTING_KEYS, _TO_NEW_KEY, _TO_OLD_KEY, + DEFAULTS, NAMESPACES, SETTING_KEYS ) +from celery.five import values + +from celery.tests.case import AppCase, pypy_version, sys_platform class test_defaults(AppCase): @@ -21,7 +23,7 @@ def teardown(self): sys.modules['celery.app.defaults'] = self._prev def test_option_repr(self): - self.assertTrue(repr(NAMESPACES['BROKER']['URL'])) + self.assertTrue(repr(NAMESPACES['broker']['url'])) def test_any(self): val = object() @@ -37,6 +39,21 @@ def test_default_pool_pypy_15(self): with pypy_version((1, 5, 0)): self.assertEqual(self.defaults.DEFAULT_POOL, 'prefork') + def test_compat_indices(self): + self.assertFalse(any(key.isupper() for key in DEFAULTS)) + self.assertFalse(any(key.islower() for key in _OLD_DEFAULTS)) + self.assertFalse(any(key.isupper() for key in _TO_OLD_KEY)) + self.assertFalse(any(key.islower() for key in _TO_NEW_KEY)) + self.assertFalse(any(key.isupper() for key in SETTING_KEYS)) + self.assertFalse(any(key.islower() for key in _OLD_SETTING_KEYS)) + self.assertFalse(any(value.isupper() for value in values(_TO_NEW_KEY))) + self.assertFalse(any(value.islower() for value in values(_TO_OLD_KEY))) + + for key in _TO_NEW_KEY: + self.assertIn(key, _OLD_SETTING_KEYS) + for key in _TO_OLD_KEY: + self.assertIn(key, SETTING_KEYS) + def test_default_pool_jython(self): with sys_platform('java 1.6.51'): self.assertEqual(self.defaults.DEFAULT_POOL, 'threads') @@ -46,7 +63,7 @@ def test_find(self): self.assertEqual(find('server_email')[2].default, 'celery@localhost') self.assertEqual(find('default_queue')[2].default, 'celery') - self.assertEqual(find('celery_default_exchange')[2], 'celery') + self.assertEqual(find('task_default_exchange')[2], 'celery') @property def defaults(self): diff --git a/celery/tests/app/test_loaders.py b/celery/tests/app/test_loaders.py index cb3d3c337..99812fb8c 100644 --- a/celery/tests/app/test_loaders.py +++ b/celery/tests/app/test_loaders.py @@ -22,7 +22,7 @@ class DummyLoader(base.BaseLoader): def read_configuration(self): - return {'foo': 'bar', 'CELERY_IMPORTS': ('os', 'sys')} + return {'foo': 'bar', 'imports': ('os', 'sys')} class test_loaders(AppCase): @@ -65,10 +65,9 @@ def test_now(self): self.assertTrue(self.loader.now(utc=False)) def test_read_configuration_no_env(self): - self.assertDictEqual( + self.assertIsNone( base.BaseLoader(app=self.app).read_configuration( 'FOO_X_S_WE_WQ_Q_WE'), - {}, ) def test_autodiscovery(self): @@ -101,7 +100,7 @@ def test_conf_property(self): def test_import_default_modules(self): def modnames(l): return [m.__name__ for m in l] - self.app.conf.CELERY_IMPORTS = ('os', 'sys') + self.app.conf.imports = ('os', 'sys') self.assertEqual( sorted(modnames(self.loader.import_default_modules())), sorted(modnames([os, sys])), @@ -183,7 +182,7 @@ class ConfigModule(ModuleType): configname = os.environ.get('CELERY_CONFIG_MODULE') or 'celeryconfig' celeryconfig = ConfigModule(configname) - celeryconfig.CELERY_IMPORTS = ('os', 'sys') + celeryconfig.imports = ('os', 'sys') prevconfig = sys.modules.get(configname) sys.modules[configname] = celeryconfig @@ -191,9 +190,9 @@ class ConfigModule(ModuleType): l = default.Loader(app=self.app) l.find_module = Mock(name='find_module') settings = l.read_configuration(fail_silently=False) - self.assertTupleEqual(settings.CELERY_IMPORTS, ('os', 'sys')) + self.assertTupleEqual(settings.imports, ('os', 'sys')) settings = l.read_configuration(fail_silently=False) - self.assertTupleEqual(settings.CELERY_IMPORTS, ('os', 'sys')) + self.assertTupleEqual(settings.imports, ('os', 'sys')) l.on_worker_init() finally: if prevconfig: @@ -239,7 +238,7 @@ def setup(self): self.loader = AppLoader(app=self.app) def test_on_worker_init(self): - self.app.conf.CELERY_IMPORTS = ('subprocess',) + self.app.conf.imports = ('subprocess',) sys.modules.pop('subprocess', None) self.loader.init_worker() self.assertIn('subprocess', sys.modules) diff --git a/celery/tests/app/test_log.py b/celery/tests/app/test_log.py index fffffa7b2..2920d97a2 100644 --- a/celery/tests/app/test_log.py +++ b/celery/tests/app/test_log.py @@ -190,7 +190,7 @@ def test_setup_logging_subsystem_misc(self): def test_setup_logging_subsystem_misc2(self): with restore_logging(): - self.app.conf.CELERYD_HIJACK_ROOT_LOGGER = True + self.app.conf.worker_hijack_root_logger = True self.app.log.setup_logging_subsystem() def test_get_default_logger(self): diff --git a/celery/tests/app/test_routes.py b/celery/tests/app/test_routes.py index bbc53b4d3..7eed424f2 100644 --- a/celery/tests/app/test_routes.py +++ b/celery/tests/app/test_routes.py @@ -21,7 +21,7 @@ def expand(answer): def set_queues(app, **queues): - app.conf.CELERY_QUEUES = queues + app.conf.task_queues = queues app.amqp.queues = app.amqp.Queues(queues) @@ -39,9 +39,9 @@ def setup(self): 'routing_key': 'b.b.#', } self.d_queue = { - 'exchange': self.app.conf.CELERY_DEFAULT_EXCHANGE, - 'exchange_type': self.app.conf.CELERY_DEFAULT_EXCHANGE_TYPE, - 'routing_key': self.app.conf.CELERY_DEFAULT_ROUTING_KEY, + 'exchange': self.app.conf.task_default_exchange, + 'exchange_type': self.app.conf.task_default_exchange_type, + 'routing_key': self.app.conf.task_default_routing_key, } @self.app.task(shared=False) @@ -74,7 +74,7 @@ def test_route_for_task(self): def test_expand_route_not_found(self): expand = E(self.app, self.app.amqp.Queues( - self.app.conf.CELERY_QUEUES, False)) + self.app.conf.task_queues, False)) route = routes.MapRoute({'a': {'queue': 'x'}}) with self.assertRaises(QueueNotFound): expand(route.route_for_task('a')) @@ -124,7 +124,7 @@ def test_expand_destination_string(self): def test_lookup_paths_traversed(self): set_queues( self.app, foo=self.a_queue, bar=self.b_queue, - **{self.app.conf.CELERY_DEFAULT_QUEUE: self.d_queue} + **{self.app.conf.task_default_queue: self.d_queue} ) R = routes.prepare(( {'celery.xaza': {'queue': 'bar'}}, @@ -135,7 +135,7 @@ def test_lookup_paths_traversed(self): args=[1, 2], kwargs={})['queue'].name, 'foo') self.assertEqual( router.route({}, 'celery.poza')['queue'].name, - self.app.conf.CELERY_DEFAULT_QUEUE, + self.app.conf.task_default_queue, ) diff --git a/celery/tests/backends/test_amqp.py b/celery/tests/backends/test_amqp.py index 15b7ba82e..78edddf79 100644 --- a/celery/tests/backends/test_amqp.py +++ b/celery/tests/backends/test_amqp.py @@ -375,7 +375,7 @@ def se(*args, **kwargs): def test_no_expires(self): b = self.create_backend(expires=None) app = self.app - app.conf.CELERY_TASK_RESULT_EXPIRES = None + app.conf.result_expires = None b = self.create_backend(expires=None) with self.assertRaises(KeyError): b.queue_arguments['x-expires'] diff --git a/celery/tests/backends/test_base.py b/celery/tests/backends/test_base.py index 0728ae890..60f7a800d 100644 --- a/celery/tests/backends/test_base.py +++ b/celery/tests/backends/test_base.py @@ -343,7 +343,7 @@ def test_chord_part_return_propagate_default(self): self.assertFalse(self.b.expire.called) deps.delete.assert_called_with() deps.join_native.assert_called_with( - propagate=self.b.app.conf.CELERY_CHORD_PROPAGATES, + propagate=self.b.app.conf.chord_propagates, timeout=3.0, ) diff --git a/celery/tests/backends/test_cache.py b/celery/tests/backends/test_cache.py index 4121df84d..e5e2fce74 100644 --- a/celery/tests/backends/test_cache.py +++ b/celery/tests/backends/test_cache.py @@ -31,12 +31,12 @@ def __init__(self, data): class test_CacheBackend(AppCase): def setup(self): - self.app.conf.CELERY_RESULT_SERIALIZER = 'pickle' + self.app.conf.result_serializer = 'pickle' self.tb = CacheBackend(backend='memory://', app=self.app) self.tid = uuid() def test_no_backend(self): - self.app.conf.CELERY_CACHE_BACKEND = None + self.app.conf.cache_backend = None with self.assertRaises(ImproperlyConfigured): CacheBackend(backend=None, app=self.app) diff --git a/celery/tests/backends/test_cassandra.py b/celery/tests/backends/test_cassandra.py index 1a43be9ef..bfcbf3c87 100644 --- a/celery/tests/backends/test_cassandra.py +++ b/celery/tests/backends/test_cassandra.py @@ -48,9 +48,9 @@ class test_CassandraBackend(AppCase): def setup(self): self.app.conf.update( - CASSANDRA_SERVERS=['example.com'], - CASSANDRA_KEYSPACE='keyspace', - CASSANDRA_COLUMN_FAMILY='columns', + cassandra_servers=['example.com'], + cassandra_keyspace='keyspace', + cassandra_column_family='columns', ) def test_init_no_pycassa(self): @@ -71,8 +71,8 @@ def test_init_with_and_without_LOCAL_QUROM(self): cons = mod.pycassa.ConsistencyLevel = Object() cons.LOCAL_QUORUM = 'foo' - self.app.conf.CASSANDRA_READ_CONSISTENCY = 'LOCAL_FOO' - self.app.conf.CASSANDRA_WRITE_CONSISTENCY = 'LOCAL_FOO' + self.app.conf.cassandra_read_consistency = 'LOCAL_FOO' + self.app.conf.cassandra_write_consistency = 'LOCAL_FOO' mod.CassandraBackend(app=self.app) cons.LOCAL_FOO = 'bar' @@ -80,7 +80,7 @@ def test_init_with_and_without_LOCAL_QUROM(self): # no servers raises ImproperlyConfigured with self.assertRaises(ImproperlyConfigured): - self.app.conf.CASSANDRA_SERVERS = None + self.app.conf.cassandra_servers = None mod.CassandraBackend( app=self.app, keyspace='b', column_family='c', ) diff --git a/celery/tests/backends/test_couchbase.py b/celery/tests/backends/test_couchbase.py index 94f72f5c4..8879ff430 100644 --- a/celery/tests/backends/test_couchbase.py +++ b/celery/tests/backends/test_couchbase.py @@ -47,13 +47,13 @@ def test_init_no_couchbase(self): def test_init_no_settings(self): """Test init no settings.""" - self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = [] + self.app.conf.couchbase_backend_settings = [] with self.assertRaises(ImproperlyConfigured): CouchBaseBackend(app=self.app) def test_init_settings_is_None(self): """Test init settings is None.""" - self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = None + self.app.conf.couchbase_backend_settings = None CouchBaseBackend(app=self.app) def test_get_connection_connection_exists(self): @@ -75,7 +75,7 @@ def test_get(self): TODO Should test on key not exists """ - self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = {} + self.app.conf.couchbase_backend_settings = {} x = CouchBaseBackend(app=self.app) x._connection = Mock() mocked_get = x._connection.get = Mock() @@ -91,7 +91,7 @@ def test_set(self): CouchBaseBackend.set should return None and take two params db conn to couchbase is mocked. """ - self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = None + self.app.conf.couchbase_backend_settings = None x = CouchBaseBackend(app=self.app) x._connection = MagicMock() x._connection.set = MagicMock() @@ -107,7 +107,7 @@ def test_delete(self): TODO Should test on key not exists. """ - self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = {} + self.app.conf.couchbase_backend_settings = {} x = CouchBaseBackend(app=self.app) x._connection = Mock() mocked_delete = x._connection.delete = Mock() @@ -120,9 +120,9 @@ def test_config_params(self): """ Test config params are correct. - celery.conf.CELERY_COUCHBASE_BACKEND_SETTINGS is properly set. + app.conf.couchbase_backend_settings is properly set. """ - self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = { + self.app.conf.couchbase_backend_settings = { 'bucket': 'mycoolbucket', 'host': ['here.host.com', 'there.host.com'], 'username': 'johndoe', diff --git a/celery/tests/backends/test_database.py b/celery/tests/backends/test_database.py index 4e3cabfeb..c7d5f8fbe 100644 --- a/celery/tests/backends/test_database.py +++ b/celery/tests/backends/test_database.py @@ -39,7 +39,7 @@ def setup(self): if DatabaseBackend is None: raise SkipTest('sqlalchemy not installed') self.uri = 'sqlite:///test.db' - self.app.conf.CELERY_RESULT_SERIALIZER = 'pickle' + self.app.conf.result_serializer = 'pickle' def test_retry_helper(self): from celery.backends.database import DatabaseError @@ -56,7 +56,7 @@ def raises(): self.assertEqual(calls[0], 5) def test_missing_dburi_raises_ImproperlyConfigured(self): - self.app.conf.CELERY_RESULT_DBURI = None + self.app.conf.sqlalchemy_dburi = None with self.assertRaises(ImproperlyConfigured): DatabaseBackend(app=self.app) diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index 1ade2e8f5..923316b1e 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -59,16 +59,16 @@ def test_init_no_mongodb(self): module.pymongo = prev def test_init_no_settings(self): - self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = [] + self.app.conf.mongodb_backend_settings = [] with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_settings_is_None(self): - self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None + self.app.conf.mongodb_backend_settings = None MongoBackend(app=self.app) def test_init_with_settings(self): - self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None + self.app.conf.mongodb_backend_settings = None # empty settings mb = MongoBackend(app=self.app) @@ -103,7 +103,7 @@ def test_init_with_settings(self): self.assertEqual(mb.database_name, 'celerydatabase') # same uri, change some parameters in backend settings - self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = { + self.app.conf.mongodb_backend_settings = { 'replicaset': 'rs1', 'user': 'backenduser', 'database': 'another_db', diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index bc0188f18..61b5fdfb6 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -20,9 +20,9 @@ class test_CassandraBackend(AppCase): def setup(self): self.app.conf.update( - CASSANDRA_SERVERS=['example.com'], - CASSANDRA_KEYSPACE='celery', - CASSANDRA_TABLE='task_results', + cassandra_servers=['example.com'], + cassandra_keyspace='celery', + cassandra_table='task_results', ) def test_init_no_cassandra(self): @@ -44,8 +44,8 @@ def test_init_with_and_without_LOCAL_QUROM(self): cons = mod.cassandra.ConsistencyLevel = Object() cons.LOCAL_QUORUM = 'foo' - self.app.conf.CASSANDRA_READ_CONSISTENCY = 'LOCAL_FOO' - self.app.conf.CASSANDRA_WRITE_CONSISTENCY = 'LOCAL_FOO' + self.app.conf.cassandra_read_consistency = 'LOCAL_FOO' + self.app.conf.cassandra_write_consistency = 'LOCAL_FOO' mod.CassandraBackend(app=self.app) cons.LOCAL_FOO = 'bar' @@ -53,7 +53,7 @@ def test_init_with_and_without_LOCAL_QUROM(self): # no servers raises ImproperlyConfigured with self.assertRaises(ImproperlyConfigured): - self.app.conf.CASSANDRA_SERVERS = None + self.app.conf.cassandra_servers = None mod.CassandraBackend( app=self.app, keyspace='b', column_family='c', ) diff --git a/celery/tests/backends/test_redis.py b/celery/tests/backends/test_redis.py index ac54bb75f..878caa542 100644 --- a/celery/tests/backends/test_redis.py +++ b/celery/tests/backends/test_redis.py @@ -180,15 +180,15 @@ def test_compat_propertie(self): def test_conf_raises_KeyError(self): self.app.conf = AttributeDict({ - 'CELERY_RESULT_SERIALIZER': 'json', - 'CELERY_MAX_CACHED_RESULTS': 1, - 'CELERY_ACCEPT_CONTENT': ['json'], - 'CELERY_TASK_RESULT_EXPIRES': None, + 'result_serializer': 'json', + 'result_cache_max': 1, + 'result_expires': None, + 'accept_content': ['json'], }) self.Backend(app=self.app, new_join=True) def test_expires_defaults_to_config(self): - self.app.conf.CELERY_TASK_RESULT_EXPIRES = 10 + self.app.conf.result_expires = 10 b = self.Backend(expires=None, app=self.app, new_join=True) self.assertEqual(b.expires, 10) @@ -216,7 +216,7 @@ def test_expires_is_None(self): b = self.Backend(expires=None, app=self.app, new_join=True) self.assertEqual( b.expires, - self.app.conf.CELERY_TASK_RESULT_EXPIRES.total_seconds(), + self.app.conf.result_expires.total_seconds(), ) def test_expires_is_timedelta(self): diff --git a/celery/tests/backends/test_riak.py b/celery/tests/backends/test_riak.py index b3323e35c..e5781a910 100644 --- a/celery/tests/backends/test_riak.py +++ b/celery/tests/backends/test_riak.py @@ -18,7 +18,7 @@ class test_RiakBackend(AppCase): def setup(self): if riak is None: raise SkipTest('riak is not installed.') - self.app.conf.CELERY_RESULT_BACKEND = 'riak://' + self.app.conf.result_backend = 'riak://' @property def backend(self): @@ -37,7 +37,7 @@ def test_init_no_riak(self): def test_init_no_settings(self): """Test init no settings.""" - self.app.conf.CELERY_RIAK_BACKEND_SETTINGS = [] + self.app.conf.riak_backend_settings = [] with self.assertRaises(ImproperlyConfigured): RiakBackend(app=self.app) @@ -45,7 +45,7 @@ def test_init_settings_is_None(self): """ Test init settings is None """ - self.app.conf.CELERY_RIAK_BACKEND_SETTINGS = None + self.app.conf.riak_backend_settings = None self.assertTrue(self.app.backend) def test_get_client_client_exists(self): @@ -67,7 +67,7 @@ def test_get(self): db conn to riak is mocked TODO Should test on key not exists """ - self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = {} + self.app.conf.couchbase_backend_settings = {} self.backend._client = Mock(name='_client') self.backend._bucket = Mock(name='_bucket') mocked_get = self.backend._bucket.get = Mock(name='bucket.get') @@ -84,7 +84,7 @@ def test_set(self): db conn to couchbase is mocked. """ - self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = None + self.app.conf.couchbase_backend_settings = None self.backend._client = MagicMock() self.backend._bucket = MagicMock() self.backend._bucket.set = MagicMock() @@ -100,7 +100,7 @@ def test_delete(self): TODO Should test on key not exists """ - self.app.conf.CELERY_COUCHBASE_BACKEND_SETTINGS = {} + self.app.conf.couchbase_backend_settings = {} self.backend._client = Mock(name='_client') self.backend._bucket = Mock(name='_bucket') @@ -112,11 +112,11 @@ def test_delete(self): def test_config_params(self): """ - test celery.conf.CELERY_RIAK_BACKEND_SETTINGS - celery.conf.CELERY_RIAK_BACKEND_SETTINGS + test celery.conf.riak_backend_settingS + celery.conf.riak_backend_settingS is properly set """ - self.app.conf.CELERY_RIAK_BACKEND_SETTINGS = { + self.app.conf.riak_backend_settings = { 'bucket': 'mycoolbucket', 'host': 'there.host.com', 'port': '1234', @@ -139,17 +139,17 @@ def test_backend_params_by_url(self): """ test get backend params by url """ - self.app.conf.CELERY_RESULT_BACKEND = 'riak://myhost:123/mycoolbucket' + self.app.conf.result_backend = 'riak://myhost:123/mycoolbucket' self.assertEqual(self.backend.bucket_name, 'mycoolbucket') self.assertEqual(self.backend.host, 'myhost') self.assertEqual(self.backend.port, 123) def test_non_ASCII_bucket_raises(self): - """test celery.conf.CELERY_RIAK_BACKEND_SETTINGS and - celery.conf.CELERY_RIAK_BACKEND_SETTINGS + """test app.conf.riak_backend_settings and + app.conf.riak_backend_settings is properly set """ - self.app.conf.CELERY_RIAK_BACKEND_SETTINGS = { + self.app.conf.riak_backend_settings = { 'bucket': 'héhé', 'host': 'there.host.com', 'port': '1234', diff --git a/celery/tests/bin/test_base.py b/celery/tests/bin/test_base.py index 36de997cb..1f4387105 100644 --- a/celery/tests/bin/test_base.py +++ b/celery/tests/bin/test_base.py @@ -281,14 +281,14 @@ def test_say_chat_no_body(self): def test_with_cmdline_config(self): cmd = MockCommand(app=self.app) cmd.enable_config_from_cmdline = True - cmd.namespace = 'celeryd' + cmd.namespace = 'worker' rest = cmd.setup_app_from_commandline(argv=[ '--loglevel=INFO', '--', 'broker.url=amqp://broker.example.com', '.prefetch_multiplier=100']) - self.assertEqual(cmd.app.conf.BROKER_URL, + self.assertEqual(cmd.app.conf.broker_url, 'amqp://broker.example.com') - self.assertEqual(cmd.app.conf.CELERYD_PREFETCH_MULTIPLIER, 100) + self.assertEqual(cmd.app.conf.worker_prefetch_multiplier, 100) self.assertListEqual(rest, ['--loglevel=INFO']) def test_find_app(self): diff --git a/celery/tests/bin/test_worker.py b/celery/tests/bin/test_worker.py index ea60da462..746ab8eb9 100644 --- a/celery/tests/bin/test_worker.py +++ b/celery/tests/bin/test_worker.py @@ -237,12 +237,12 @@ def test_init_queues(self): self.assertIn('celery', app.amqp.queues) self.assertNotIn('celery', app.amqp.queues.consume_from) - c.CELERY_CREATE_MISSING_QUEUES = False + c.task_create_missing_queues = False del(app.amqp.queues) with self.assertRaises(ImproperlyConfigured): self.Worker(app=self.app).setup_queues(['image']) del(app.amqp.queues) - c.CELERY_CREATE_MISSING_QUEUES = True + c.task_create_missing_queues = True worker = self.Worker(app=self.app) worker.setup_queues(['image']) self.assertIn('image', app.amqp.queues.consume_from) @@ -283,7 +283,7 @@ def test_warns_if_running_as_privileged_user(self, _exit): with patch('os.getuid') as getuid: getuid.return_value = 0 - self.app.conf.CELERY_ACCEPT_CONTENT = ['pickle'] + self.app.conf.accept_content = ['pickle'] worker = self.Worker(app=self.app) worker.on_start() _exit.assert_called_with(1) @@ -297,7 +297,7 @@ def test_warns_if_running_as_privileged_user(self, _exit): worker.on_start() finally: platforms.C_FORCE_ROOT = False - self.app.conf.CELERY_ACCEPT_CONTENT = ['json'] + self.app.conf.accept_content = ['json'] with self.assertWarnsRegex( RuntimeWarning, r'absolutely not recommended'): diff --git a/celery/tests/case.py b/celery/tests/case.py index 6446fd98c..0901c97b4 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -85,21 +85,21 @@ CELERY_TEST_CONFIG = { #: Don't want log output when running suite. - 'CELERYD_HIJACK_ROOT_LOGGER': False, - 'CELERY_SEND_TASK_ERROR_EMAILS': False, - 'CELERY_DEFAULT_QUEUE': 'testcelery', - 'CELERY_DEFAULT_EXCHANGE': 'testcelery', - 'CELERY_DEFAULT_ROUTING_KEY': 'testcelery', - 'CELERY_QUEUES': ( + 'worker_hijack_root_logger': False, + 'worker_log_color': False, + 'task_send_error_emails': False, + 'task_default_queue': 'testcelery', + 'task_default_exchange': 'testcelery', + 'task_default_routing_key': 'testcelery', + 'task_queues': ( Queue('testcelery', routing_key='testcelery'), ), - 'CELERY_ACCEPT_CONTENT': ('json', 'pickle'), - 'CELERY_ENABLE_UTC': True, - 'CELERY_TIMEZONE': 'UTC', - 'CELERYD_LOG_COLOR': False, + 'accept_content': ('json', 'pickle'), + 'enable_utc': True, + 'timezone': 'UTC', # Mongo results tests (only executed if installed and running) - 'CELERY_MONGODB_BACKEND_SETTINGS': { + 'mongodb_backend_settings': { 'host': os.environ.get('MONGO_HOST') or 'localhost', 'port': os.environ.get('MONGO_PORT') or 27017, 'database': os.environ.get('MONGO_DB') or 'celery_unittests', diff --git a/celery/tests/compat_modules/test_http.py b/celery/tests/compat_modules/test_http.py index c3a23b613..1c4edf0e4 100644 --- a/celery/tests/compat_modules/test_http.py +++ b/celery/tests/compat_modules/test_http.py @@ -142,7 +142,7 @@ def test_dispatch_POST(self): class test_URL(AppCase): def test_URL_get_async(self): - self.app.conf.CELERY_ALWAYS_EAGER = True + self.app.conf.task_always_eager = True with mock_urlopen(success_response(100)): d = http.URL( 'http://example.com/mul', app=self.app, @@ -150,7 +150,7 @@ def test_URL_get_async(self): self.assertEqual(d.get(), 100) def test_URL_post_async(self): - self.app.conf.CELERY_ALWAYS_EAGER = True + self.app.conf.task_always_eager = True with mock_urlopen(success_response(100)): d = http.URL( 'http://example.com/mul', app=self.app, diff --git a/celery/tests/compat_modules/test_sets.py b/celery/tests/compat_modules/test_sets.py index 710adae76..4869716cb 100644 --- a/celery/tests/compat_modules/test_sets.py +++ b/celery/tests/compat_modules/test_sets.py @@ -170,10 +170,10 @@ def apply(self, *args, **kwargs): [self.MockTask.subtask((i, i)) for i in (2, 4, 8)], app=self.app, ) - app.conf.CELERY_ALWAYS_EAGER = True + app.conf.task_always_eager = True ts.apply_async() self.assertEqual(ts.applied, 1) - app.conf.CELERY_ALWAYS_EAGER = False + app.conf.task_always_eager = False with patch('celery.task.sets.get_current_worker_task') as gwt: parent = gwt.return_value = Mock() diff --git a/celery/tests/events/test_events.py b/celery/tests/events/test_events.py index 0c78a4f4d..1e16f93ef 100644 --- a/celery/tests/events/test_events.py +++ b/celery/tests/events/test_events.py @@ -125,7 +125,7 @@ def test_enabled_disable(self): self.assertTrue(dispatcher.enabled) self.assertTrue(dispatcher.producer.channel) self.assertEqual(dispatcher.producer.serializer, - self.app.conf.CELERY_EVENT_SERIALIZER) + self.app.conf.event_serializer) created_channel = dispatcher.producer.channel dispatcher.disable() diff --git a/celery/tests/security/test_security.py b/celery/tests/security/test_security.py index 9cc49e5f6..134efc9bb 100644 --- a/celery/tests/security/test_security.py +++ b/celery/tests/security/test_security.py @@ -57,7 +57,7 @@ def test_setup_security(self): disabled = registry._disabled_content_types self.assertEqual(0, len(disabled)) - self.app.conf.CELERY_TASK_SERIALIZER = 'json' + self.app.conf.task_serializer = 'json' self.app.setup_security() self.assertIn('application/x-python-serialize', disabled) disabled.clear() @@ -75,7 +75,7 @@ def effect(*args): finally: calls[0] += 1 - self.app.conf.CELERY_TASK_SERIALIZER = 'auth' + self.app.conf.task_serializer = 'auth' with mock_open(side_effect=effect): with patch('celery.security.registry') as registry: store = Mock() @@ -85,7 +85,7 @@ def effect(*args): registry._set_default_serializer.assert_called_with('auth') def test_security_conf(self): - self.app.conf.CELERY_TASK_SERIALIZER = 'auth' + self.app.conf.task_serializer = 'auth' with self.assertRaises(ImproperlyConfigured): self.app.setup_security() diff --git a/celery/tests/tasks/test_canvas.py b/celery/tests/tasks/test_canvas.py index 365f11a64..287241d2d 100644 --- a/celery/tests/tasks/test_canvas.py +++ b/celery/tests/tasks/test_canvas.py @@ -198,7 +198,7 @@ def test_chunks(self): x() gr.apply_async.assert_called_with((), {}, route_name=self.add.name) - self.app.conf.CELERY_ALWAYS_EAGER = True + self.app.conf.task_always_eager = True chunks.apply_chunks(app=self.app, **x['kwargs']) @@ -216,7 +216,7 @@ def test_reverse(self): self.assertIsInstance(signature(dict(x)), chain) def test_always_eager(self): - self.app.conf.CELERY_ALWAYS_EAGER = True + self.app.conf.task_always_eager = True self.assertEqual(~(self.add.s(4, 4) | self.add.s(8)), 16) def test_apply(self): diff --git a/celery/tests/tasks/test_chord.py b/celery/tests/tasks/test_chord.py index a7cc1d859..e458213a6 100644 --- a/celery/tests/tasks/test_chord.py +++ b/celery/tests/tasks/test_chord.py @@ -194,18 +194,18 @@ def addX(x, y): def sumX(n): return sum(n) - self.app.conf.CELERY_ALWAYS_EAGER = True + self.app.conf.task_always_eager = True x = chord(addX.s(i, i) for i in range(10)) body = sumX.s() result = x(body) self.assertEqual(result.get(), sum(i + i for i in range(10))) def test_apply(self): - self.app.conf.CELERY_ALWAYS_EAGER = False + self.app.conf.task_always_eager = False from celery import chord m = Mock() - m.app.conf.CELERY_ALWAYS_EAGER = False + m.app.conf.task_always_eager = False m.AsyncResult = AsyncResult prev, chord.run = chord.run, m try: diff --git a/celery/tests/tasks/test_result.py b/celery/tests/tasks/test_result.py index a92b22448..590b0f494 100644 --- a/celery/tests/tasks/test_result.py +++ b/celery/tests/tasks/test_result.py @@ -44,7 +44,7 @@ def make_mock_group(app, size=10): class test_AsyncResult(AppCase): def setup(self): - self.app.conf.CELERY_RESULT_SERIALIZER = 'pickle' + self.app.conf.result_serializer = 'pickle' self.task1 = mock_task('task1', states.SUCCESS, 'the') self.task2 = mock_task('task2', states.SUCCESS, 'quick') self.task3 = mock_task('task3', states.FAILURE, KeyError('brown')) @@ -618,7 +618,7 @@ def test_result(self): class test_failed_AsyncResult(test_GroupResult): def setup(self): - self.app.conf.CELERY_RESULT_SERIALIZER = 'pickle' + self.app.conf.result_serializer = 'pickle' self.size = 11 subtasks = make_mock_group(self.app, 10) failed = mock_task('ts11', states.FAILURE, KeyError('Baz')) diff --git a/celery/tests/tasks/test_tasks.py b/celery/tests/tasks/test_tasks.py index 522bb6f8e..38ca84cba 100644 --- a/celery/tests/tasks/test_tasks.py +++ b/celery/tests/tasks/test_tasks.py @@ -462,8 +462,8 @@ def test_apply_throw(self): with self.assertRaises(KeyError): self.raising.apply(throw=True) - def test_apply_with_CELERY_EAGER_PROPAGATES_EXCEPTIONS(self): - self.app.conf.CELERY_EAGER_PROPAGATES_EXCEPTIONS = True + def test_apply_with_task_eager_propagates_exceptions(self): + self.app.conf.task_eager_propagates_exceptions = True with self.assertRaises(KeyError): self.raising.apply() diff --git a/celery/tests/utils/test_datastructures.py b/celery/tests/utils/test_datastructures.py index bb148c653..be81c364b 100644 --- a/celery/tests/utils/test_datastructures.py +++ b/celery/tests/utils/test_datastructures.py @@ -44,8 +44,10 @@ def test_get_set_keys_values_items(self): def test_setdefault(self): x = DictAttribute(Object()) - self.assertEqual(x.setdefault('foo', 'NEW'), 'NEW') - self.assertEqual(x.setdefault('foo', 'XYZ'), 'NEW') + x.setdefault('foo', 'NEW') + self.assertEqual(x['foo'], 'NEW') + x.setdefault('foo', 'XYZ') + self.assertEqual(x['foo'], 'NEW') def test_contains(self): x = DictAttribute(Object()) @@ -71,8 +73,10 @@ def setUp(self): 'both': 1}]) def test_setdefault(self): - self.assertEqual(self.view.setdefault('both', 36), 2) - self.assertEqual(self.view.setdefault('new', 36), 36) + self.view.setdefault('both', 36) + self.assertEqual(self.view['both'], 2) + self.view.setdefault('new', 36) + self.assertEqual(self.view['new'], 36) def test_get(self): self.assertEqual(self.view.get('both'), 2) diff --git a/celery/tests/worker/test_consumer.py b/celery/tests/worker/test_consumer.py index 59ee8edc6..88daff4ac 100644 --- a/celery/tests/worker/test_consumer.py +++ b/celery/tests/worker/test_consumer.py @@ -34,10 +34,11 @@ def get_consumer(self, no_hub=False, **kwargs): hub=None if no_hub else Mock(), **kwargs ) - consumer.blueprint = Mock() - consumer._restart_state = Mock() + consumer.blueprint = Mock(name='blueprint') + consumer._restart_state = Mock(name='_restart_state') consumer.connection = _amqp_connection() consumer.connection_errors = (socket.error, OSError,) + consumer.conninfo = consumer.connection return consumer def test_taskbuckets_defaultdict(self): @@ -56,16 +57,16 @@ def test_dump_body_buffer(self): def test_sets_heartbeat(self): c = self.get_consumer(amqheartbeat=10) self.assertEqual(c.amqheartbeat, 10) - self.app.conf.BROKER_HEARTBEAT = 20 + self.app.conf.broker_heartbeat = 20 c = self.get_consumer(amqheartbeat=None) self.assertEqual(c.amqheartbeat, 20) def test_gevent_bug_disables_connection_timeout(self): with patch('celery.worker.consumer._detect_environment') as de: de.return_value = 'gevent' - self.app.conf.BROKER_CONNECTION_TIMEOUT = 33.33 + self.app.conf.broker_connection_timeout = 33.33 self.get_consumer() - self.assertIsNone(self.app.conf.BROKER_CONNECTION_TIMEOUT) + self.assertIsNone(self.app.conf.broker_connection_timeout) def test_limit_task(self): c = self.get_consumer() @@ -116,7 +117,7 @@ def se(*args, **kwargs): sleep.assert_called_with(1) def test_no_retry_raises_error(self): - self.app.conf.BROKER_CONNECTION_RETRY = False + self.app.conf.broker_connection_retry = False c = self.get_consumer() c.blueprint.start.side_effect = socket.error() with self.assertRaises(socket.error): @@ -280,8 +281,8 @@ def test_start(self): def _amqp_connection(): - connection = ContextMock() - connection.return_value = ContextMock() + connection = ContextMock(name='Connection') + connection.return_value = ContextMock(name='connection') connection.return_value.transport.driver_type = 'amqp' return connection diff --git a/celery/tests/worker/test_control.py b/celery/tests/worker/test_control.py index b9df3fefe..d2cd234af 100644 --- a/celery/tests/worker/test_control.py +++ b/celery/tests/worker/test_control.py @@ -517,7 +517,7 @@ def test_pool_restart(self): with self.assertRaises(ValueError): panel.handle('pool_restart', {'reloader': _reload}) - self.app.conf.CELERYD_POOL_RESTARTS = True + self.app.conf.worker_pool_restarts = True panel.handle('pool_restart', {'reloader': _reload}) self.assertTrue(consumer.controller.pool.restart.called) consumer.reset_rate_limits.assert_called_with() @@ -538,7 +538,7 @@ def test_pool_restart_import_modules(self): _import = consumer.controller.app.loader.import_from_cwd = Mock() _reload = Mock() - self.app.conf.CELERYD_POOL_RESTARTS = True + self.app.conf.worker_pool_restarts = True panel.handle('pool_restart', {'modules': ['foo', 'bar'], 'reloader': _reload}) @@ -563,7 +563,7 @@ def test_pool_restart_reload_modules(self): _import = panel.app.loader.import_from_cwd = Mock() _reload = Mock() - self.app.conf.CELERYD_POOL_RESTARTS = True + self.app.conf.worker_pool_restarts = True with patch.dict(sys.modules, {'foo': None}): panel.handle('pool_restart', {'modules': ['foo'], 'reload': False, diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index e05e5121e..72ab9c7ce 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -210,7 +210,7 @@ def send(self, event, **fields): class test_Request(AppCase): def setup(self): - self.app.conf.CELERY_RESULT_SERIALIZER = 'pickle' + self.app.conf.result_serializer = 'pickle' @self.app.task(shared=False) def add(x, y, **kw_): diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index 794d10791..1eca31def 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -587,12 +587,12 @@ def pp(*args, **kwargs): pp('+ BLUEPRINT START 1') l.blueprint.start(l) pp('- BLUEPRINT START 1') - p = l.app.conf.BROKER_CONNECTION_RETRY - l.app.conf.BROKER_CONNECTION_RETRY = False + p = l.app.conf.broker_connection_retry + l.app.conf.broker_connection_retry = False pp('+ BLUEPRINT START 2') l.blueprint.start(l) pp('- BLUEPRINT START 2') - l.app.conf.BROKER_CONNECTION_RETRY = p + l.app.conf.broker_connection_retry = p pp('+ BLUEPRINT RESTART') l.blueprint.restart(l) pp('- BLUEPRINT RESTART') @@ -825,7 +825,7 @@ def test_on_consumer_ready(self): self.worker.on_consumer_ready(Mock()) def test_setup_queues_worker_direct(self): - self.app.conf.CELERY_WORKER_DIRECT = True + self.app.conf.worker_direct = True self.app.amqp.__dict__['queues'] = Mock() self.worker.setup_queues({}) self.app.amqp.queues.select_add.assert_called_with( diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 578b31a47..fbb4fc468 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -16,6 +16,7 @@ from inspect import getargspec, isfunction from itertools import islice +from amqp import promise from kombu.utils import cached_property from kombu.utils.functional import lazy, maybe_evaluate, is_list, maybe_list @@ -210,6 +211,13 @@ def noop(*args, **kwargs): pass +def evaluate_promises(it): + for value in it: + if isinstance(value, promise): + value = value() + yield value + + def first(predicate, it): """Return the first element in `iterable` that `predicate` Gives a :const:`True` value for. @@ -218,7 +226,8 @@ def first(predicate, it): """ return next( - (v for v in it if (predicate(v) if predicate else v is not None)), + (v for v in evaluate_promises(it) if ( + predicate(v) if predicate is not None else v is not None)), None, ) diff --git a/celery/worker/__init__.py b/celery/worker/__init__.py index c006c5280..1ff4cb107 100644 --- a/celery/worker/__init__.py +++ b/celery/worker/__init__.py @@ -46,15 +46,15 @@ SELECT_UNKNOWN_QUEUE = """\ Trying to select queue subset of {0!r}, but queue {1} is not -defined in the CELERY_QUEUES setting. +defined in the `task_queues` setting. If you want to automatically declare unknown queues you can -enable the CELERY_CREATE_MISSING_QUEUES setting. +enable the `task_create_missing_queues` setting. """ DESELECT_UNKNOWN_QUEUE = """\ Trying to deselect queue subset of {0!r}, but queue {1} is not -defined in the CELERY_QUEUES setting. +defined in the `task_queues` setting. """ @@ -180,20 +180,20 @@ def setup_queues(self, include, exclude=None): except KeyError as exc: raise ImproperlyConfigured( DESELECT_UNKNOWN_QUEUE.format(exclude, exc)) - if self.app.conf.CELERY_WORKER_DIRECT: + if self.app.conf.worker_direct: self.app.amqp.queues.select_add(worker_direct(self.hostname)) def setup_includes(self, includes): # Update celery_include to have all known task modules, so that we # ensure all task modules are imported in case an execv happens. - prev = tuple(self.app.conf.CELERY_INCLUDE) + prev = tuple(self.app.conf.include) if includes: prev += tuple(includes) [self.app.loader.import_task_module(m) for m in includes] self.include = includes task_modules = {task.__class__.__module__ for task in values(self.app.tasks)} - self.app.conf.CELERY_INCLUDE = tuple(set(prev) | task_modules) + self.app.conf.include = tuple(set(prev) | task_modules) def prepare_args(self, **kwargs): return kwargs @@ -353,49 +353,42 @@ def setup_defaults(self, concurrency=None, loglevel='WARN', logfile=None, max_tasks_per_child=None, prefetch_multiplier=None, disable_rate_limits=None, worker_lost_wait=None, max_memory_per_child=None, **_kw): + either = self.app.either self.loglevel = loglevel self.logfile = logfile - self.concurrency = self._getopt('concurrency', concurrency) - self.send_events = self._getopt('send_events', send_events) - self.pool_cls = self._getopt('pool', pool_cls) - self.consumer_cls = self._getopt('consumer', consumer_cls) - self.timer_cls = self._getopt('timer', timer_cls) - self.timer_precision = self._getopt('timer_precision', timer_precision) - self.autoscaler_cls = self._getopt('autoscaler', autoscaler_cls) - self.autoreloader_cls = self._getopt('autoreloader', autoreloader_cls) - self.pool_putlocks = self._getopt('pool_putlocks', pool_putlocks) - self.pool_restarts = self._getopt('pool_restarts', pool_restarts) - self.force_execv = self._getopt('force_execv', force_execv) - self.state_db = self._getopt('state_db', state_db) - self.schedule_filename = self._getopt( - 'schedule_filename', schedule_filename, - ) - self.scheduler_cls = self._getopt( - 'celerybeat_scheduler', scheduler_cls, + + self.concurrency = either('worker_concurrency', concurrency) + self.send_events = either('worker_send_events', send_events) + self.pool_cls = either('worker_pool', pool_cls) + self.consumer_cls = either('worker_consumer', consumer_cls) + self.timer_cls = either('worker_timer', timer_cls) + self.timer_precision = either( + 'worker_timer_precision', timer_precision, ) - self.task_time_limit = self._getopt( - 'task_time_limit', task_time_limit, + self.autoscaler_cls = either('worker_autoscaler', autoscaler_cls) + self.autoreloader_cls = either('worker_autoreloader', autoreloader_cls) + self.pool_putlocks = either('worker_pool_putlocks', pool_putlocks) + self.pool_restarts = either('worker_pool_restarts', pool_restarts) + self.force_execv = either('worker_force_execv', force_execv) + self.state_db = either('worker_state_db', state_db) + self.schedule_filename = either( + 'beat_schedule_filename', schedule_filename, ) - self.task_soft_time_limit = self._getopt( + self.scheduler_cls = either('beat_scheduler', scheduler_cls) + self.task_time_limit = either('task_time_limit', task_time_limit) + self.task_soft_time_limit = either( 'task_soft_time_limit', task_soft_time_limit, ) - self.max_tasks_per_child = self._getopt( - 'max_tasks_per_child', max_tasks_per_child, + self.max_tasks_per_child = either( + 'worker_max_tasks_per_child', max_tasks_per_child, ) - self.max_memory_per_child = self._getopt( - 'max_memory_per_child', max_memory_per_child, + self.max_memory_per_child = either( + 'worker_max_memory_per_child', max_memory_per_child, ) - self.prefetch_multiplier = int(self._getopt( - 'prefetch_multiplier', prefetch_multiplier, + self.prefetch_multiplier = int(either( + 'worker_prefetch_multiplier', prefetch_multiplier, )) - self.disable_rate_limits = self._getopt( - 'disable_rate_limits', disable_rate_limits, + self.disable_rate_limits = either( + 'worker_disable_rate_limits', disable_rate_limits, ) - self.worker_lost_wait = self._getopt( - 'worker_lost_wait', worker_lost_wait, - ) - - def _getopt(self, key, value): - if value is not None: - return value - return self.app.conf.find_value_for_key(key, namespace='celeryd') + self.worker_lost_wait = either('worker_lost_wait', worker_lost_wait) diff --git a/celery/worker/components.py b/celery/worker/components.py index 2c09156ff..200173d74 100644 --- a/celery/worker/components.py +++ b/celery/worker/components.py @@ -31,7 +31,7 @@ """ W_POOL_SETTING = """ -The CELERYD_POOL setting should not be used to select the eventlet/gevent +The worker_pool setting should not be used to select the eventlet/gevent pools, instead you *must use the -P* argument so that patches are applied as early as possible. """ @@ -138,7 +138,7 @@ def terminate(self, w): w.pool.terminate() def create(self, w, semaphore=None, max_restarts=None): - if w.app.conf.CELERYD_POOL in ('eventlet', 'gevent'): + if w.app.conf.worker_pool in ('eventlet', 'gevent'): warnings.warn(UserWarning(W_POOL_SETTING)) threaded = not w.use_eventloop or IS_WINDOWS procs = w.min_concurrency diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index a5bb52013..20d392288 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -185,7 +185,7 @@ def __init__(self, on_task_request, self._limit_order = 0 self.on_task_request = on_task_request self.on_task_message = set() - self.amqheartbeat_rate = self.app.conf.BROKER_HEARTBEAT_CHECKRATE + self.amqheartbeat_rate = self.app.conf.broker_heartbeat_checkrate self.disable_rate_limits = disable_rate_limits self.initial_prefetch_count = initial_prefetch_count self.prefetch_multiplier = prefetch_multiplier @@ -199,7 +199,7 @@ def __init__(self, on_task_request, if self.hub: self.amqheartbeat = amqheartbeat if self.amqheartbeat is None: - self.amqheartbeat = self.app.conf.BROKER_HEARTBEAT + self.amqheartbeat = self.app.conf.broker_heartbeat else: self.amqheartbeat = 0 @@ -210,7 +210,7 @@ def __init__(self, on_task_request, # there's a gevent bug that causes timeouts to not be reset, # so if the connection timeout is exceeded once, it can NEVER # connect again. - self.app.conf.BROKER_CONNECTION_TIMEOUT = None + self.app.conf.broker_connection_timeout = None self.steps = [] self.blueprint = self.Blueprint( @@ -279,7 +279,7 @@ def start(self): except self.connection_errors as exc: # If we're not retrying connections, no need to catch # connection errors - if not self.app.conf.BROKER_CONNECTION_RETRY: + if not self.app.conf.broker_connection_retry: raise if isinstance(exc, OSError) and exc.errno == errno.EMFILE: raise # Too many open files @@ -354,7 +354,7 @@ def connect(self): """Establish the broker connection. Will retry establishing the connection if the - :setting:`BROKER_CONNECTION_RETRY` setting is enabled + :setting:`broker_connection_retry` setting is enabled """ conn = self.app.connection(heartbeat=self.amqheartbeat) @@ -369,13 +369,13 @@ def _error_handler(exc, interval, next_step=CONNECTION_RETRY_STEP): # remember that the connection is lazy, it won't establish # until needed. - if not self.app.conf.BROKER_CONNECTION_RETRY: + if not self.app.conf.broker_connection_retry: # retry disabled, just call connect directly. conn.connect() return conn conn = conn.ensure_connection( - _error_handler, self.app.conf.BROKER_CONNECTION_MAX_RETRIES, + _error_handler, self.app.conf.broker_connection_max_retries, callback=maybe_shutdown, ) if self.hub: @@ -395,7 +395,7 @@ def add_task_queue(self, queue, exchange=None, exchange_type=None, cset = self.task_consumer queues = self.app.amqp.queues # Must use in' here, as __missing__ will automatically - # create queues when CELERY_CREATE_MISSING_QUEUES is enabled. + # create queues when :setting:`task_create_missing_queues` is enabled. # (Issue #1079) if queue in queues: q = queues[queue] @@ -667,7 +667,7 @@ class Agent(bootsteps.StartStopStep): requires = (Connection,) def __init__(self, c, **kwargs): - self.agent_cls = self.enabled = c.app.conf.CELERYD_AGENT + self.agent_cls = self.enabled = c.app.conf.worker_agent def create(self, c): agent = c.agent = self.instantiate(self.agent_cls, c.connection) @@ -685,7 +685,7 @@ def __init__(self, c, **kwargs): self.shutdown = self.box.shutdown def include_if(self, c): - return (c.app.conf.CELERY_ENABLE_REMOTE_CONTROL and + return (c.app.conf.worker_enable_remote_control and c.conninfo.supports_exchange_type('fanout')) diff --git a/celery/worker/control.py b/celery/worker/control.py index 3b2953da5..36f066b03 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -321,7 +321,7 @@ def pool_shrink(state, n=1, **kwargs): @Panel.register def pool_restart(state, modules=None, reload=False, reloader=None, **kwargs): - if state.app.conf.CELERYD_POOL_RESTARTS: + if state.app.conf.worker_pool_restarts: state.consumer.controller.reload(modules, reload, reloader=reloader) return {'ok': 'reload started'} else: diff --git a/celery/worker/request.py b/celery/worker/request.py index c47ae81d5..73cbc86cd 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -420,7 +420,7 @@ def __repr__(self): @property def tzlocal(self): if self._tzlocal is None: - self._tzlocal = self.app.conf.CELERY_TIMEZONE + self._tzlocal = self.app.conf.timezone return self._tzlocal @property diff --git a/docs/configuration.rst b/docs/configuration.rst index 8373b2ecd..1f76da414 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -24,29 +24,53 @@ It should contain all you need to run a basic Celery set-up. .. code-block:: python ## Broker settings. - BROKER_URL = 'amqp://guest:guest@localhost:5672//' + broker_url = 'amqp://guest:guest@localhost:5672//' # List of modules to import when celery starts. - CELERY_IMPORTS = ('myapp.tasks',) + imports = ('myapp.tasks',) ## Using the database to store task state and results. - CELERY_RESULT_BACKEND = 'db+sqlite:///results.db' - - CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} + result_backend = 'db+sqlite:///results.db' + task_annotations = {'tasks.add': {'rate_limit': '10/s'}} Configuration Directives ======================== .. _conf-datetime: +General settings +---------------- + +.. setting:: accept_content + +accept_content +~~~~~~~~~~~~~~ + +A whitelist of content-types/serializers to allow. + +If a message is received that is not in this list then +the message will be discarded with an error. + +By default any content type is enabled (including pickle and yaml) +so make sure untrusted parties do not have access to your broker. +See :ref:`guide-security` for more. + +Example:: + + # using serializer name + accept_content = ['json'] + + # or the actual content-type (MIME) + accept_content = ['application/json'] + Time and date settings ---------------------- -.. setting:: CELERY_ENABLE_UTC +.. setting:: enable_utc -CELERY_ENABLE_UTC -~~~~~~~~~~~~~~~~~ +enable_utc +~~~~~~~~~~ .. versionadded:: 2.5 @@ -59,52 +83,48 @@ upgraded. Enabled by default since version 3.0. -.. setting:: CELERY_TIMEZONE +.. setting:: timezone -CELERY_TIMEZONE -~~~~~~~~~~~~~~~ +timezone +~~~~~~~~ Configure Celery to use a custom time zone. The timezone value can be any time zone supported by the `pytz`_ library. If not set the UTC timezone is used. For backwards compatibility -there is also a :setting:`CELERY_ENABLE_UTC` setting, and this is set +there is also a :setting:`enable_utc` setting, and this is set to false the system local timezone is used instead. .. _`pytz`: http://pypi.python.org/pypi/pytz/ - - .. _conf-tasks: Task settings ------------- -.. setting:: CELERY_ANNOTATIONS +.. setting:: task_annotations -CELERY_ANNOTATIONS -~~~~~~~~~~~~~~~~~~ +task_annotations +~~~~~~~~~~~~~~~~ This setting can be used to rewrite any task attribute from the configuration. The setting can be a dict, or a list of annotation objects that filter for tasks and return a map of attributes to change. - This will change the ``rate_limit`` attribute for the ``tasks.add`` task: .. code-block:: python - CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} + task_annotations = {'tasks.add': {'rate_limit': '10/s'}} or change the same for all tasks: .. code-block:: python - CELERY_ANNOTATIONS = {'*': {'rate_limit': '10/s'}} - + task_annotations = {'*': {'rate_limit': '10/s'}} You can change methods too, for example the ``on_failure`` handler: @@ -113,8 +133,7 @@ You can change methods too, for example the ``on_failure`` handler: def my_on_failure(self, exc, task_id, args, kwargs, einfo): print('Oh no! Task failed: {0!r}'.format(exc)) - CELERY_ANNOTATIONS = {'*': {'on_failure': my_on_failure}} - + task_annotations = {'*': {'on_failure': my_on_failure}} If you need more flexibility then you can use objects instead of a dict to choose which tasks to annotate: @@ -127,63 +146,209 @@ instead of a dict to choose which tasks to annotate: if task.name.startswith('tasks.'): return {'rate_limit': '10/s'} - CELERY_ANNOTATIONS = (MyAnnotate(), {…}) + task_annotations = (MyAnnotate(), {…}) +.. setting:: task_compression +task_compression +~~~~~~~~~~~~~~~~ -.. _conf-concurrency: +Default compression used for task messages. +Can be ``gzip``, ``bzip2`` (if available), or any custom +compression schemes registered in the Kombu compression registry. -Concurrency settings --------------------- +The default is to send uncompressed messages. -.. setting:: CELERYD_CONCURRENCY +.. setting:: task_protocol -CELERYD_CONCURRENCY -~~~~~~~~~~~~~~~~~~~ +task_protocol +~~~~~~~~~~~~~ -The number of concurrent worker processes/threads/green threads executing -tasks. +Default task message protocol version. +Supports protocols: 1 and 2 (default is 1 for backwards compatibility). -If you're doing mostly I/O you can have more processes, -but if mostly CPU-bound, try to keep it close to the -number of CPUs on your machine. If not set, the number of CPUs/cores -on the host will be used. +.. setting:: task_serializer -Defaults to the number of available CPUs. +task_serializer +~~~~~~~~~~~~~~~ -.. setting:: CELERYD_PREFETCH_MULTIPLIER +A string identifying the default serialization method to use. Can be +`pickle` (default), `json`, `yaml`, `msgpack` or any custom serialization +methods that have been registered with :mod:`kombu.serialization.registry`. -CELERYD_PREFETCH_MULTIPLIER -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. seealso:: -How many messages to prefetch at a time multiplied by the number of -concurrent processes. The default is 4 (four messages for each -process). The default setting is usually a good choice, however -- if you -have very long running tasks waiting in the queue and you have to start the -workers, note that the first worker to start will receive four times the -number of messages initially. Thus the tasks may not be fairly distributed -to the workers. + :ref:`calling-serializers`. -To disable prefetching, set CELERYD_PREFETCH_MULTIPLIER to 1. Setting -CELERYD_PREFETCH_MULTIPLIER to 0 will allow the worker to keep consuming -as many messages as it wants. +.. setting:: task_publish_retry -For more on prefetching, read :ref:`optimizing-prefetch-limit` +task_publish_retry +~~~~~~~~~~~~~~~~~~ -.. note:: +.. versionadded:: 2.2 - Tasks with ETA/countdown are not affected by prefetch limits. +Decides if publishing task messages will be retried in the case +of connection loss or other connection errors. +See also :setting:`task_publish_retry_policy`. + +Enabled by default. + +.. setting:: task_publish_retry_policy + +task_publish_retry_policy +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 2.2 + +Defines the default policy when retrying publishing a task message in +the case of connection loss or other connection errors. + +See :ref:`calling-retry` for more information. +.. _conf-task-execution: + +Task execution settings +----------------------- + +.. setting:: task_always_eager + +task_always_eager +~~~~~~~~~~~~~~~~~ + +If this is :const:`True`, all tasks will be executed locally by blocking until +the task returns. ``apply_async()`` and ``Task.delay()`` will return +an :class:`~celery.result.EagerResult` instance, which emulates the API +and behavior of :class:`~celery.result.AsyncResult`, except the result +is already evaluated. + +That is, tasks will be executed locally instead of being sent to +the queue. + +.. setting:: task_eager_propagates_exceptions + +task_eager_propagates_exceptions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If this is :const:`True`, eagerly executed tasks (applied by `task.apply()`, +or when the :setting:`task_always_eager` setting is enabled), will +propagate exceptions. + +It's the same as always running ``apply()`` with ``throw=True``. + +.. setting:: task_ignore_result + +task_ignore_result +~~~~~~~~~~~~~~~~~~ + +Whether to store the task return values or not (tombstones). +If you still want to store errors, just not successful return values, +you can set :setting:`task_store_errors_even_if_ignored`. + +.. setting:: task_store_errors_even_if_ignored + +task_store_errors_even_if_ignored +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If set, the worker stores all task errors in the result store even if +:attr:`Task.ignore_result ` is on. + +.. setting:: task_track_started + +task_track_started +~~~~~~~~~~~~~~~~~~ + +If :const:`True` the task will report its status as "started" when the +task is executed by a worker. The default value is :const:`False` as +the normal behaviour is to not report that level of granularity. Tasks +are either pending, finished, or waiting to be retried. Having a "started" +state can be useful for when there are long running tasks and there is a +need to report which task is currently running. + +.. setting:: task_time_limit + +task_time_limit +~~~~~~~~~~~~~~~ + +Task hard time limit in seconds. The worker processing the task will +be killed and replaced with a new one when this is exceeded. + +.. setting:: task_soft_time_limit + +task_soft_time_limit +~~~~~~~~~~~~~~~~~~~~ + +Task soft time limit in seconds. + +The :exc:`~@SoftTimeLimitExceeded` exception will be +raised when this is exceeded. The task can catch this to +e.g. clean up before the hard time limit comes. + +Example: + +.. code-block:: python + + from celery.exceptions import SoftTimeLimitExceeded + + @app.task + def mytask(): + try: + return do_work() + except SoftTimeLimitExceeded: + cleanup_in_a_hurry() + +.. setting:: task_acks_late + +task_acks_late +~~~~~~~~~~~~~~ + +Late ack means the task messages will be acknowledged **after** the task +has been executed, not *just before*, which is the default behavior. + +.. seealso:: + + FAQ: :ref:`faq-acks_late-vs-retry`. + +.. setting:: task_reject_on_worker_lost + +task_reject_on_worker_lost +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Even if :setting:`task_acks_late` is enabled, the worker will +acknowledge tasks when the worker process executing them abrubtly +exits or is signalled (e.g. :sig:`KILL`/:sig:`INT`, etc). + +Setting this to true allows the message to be requeued instead, +so that the task will execute again by the same worker, or another +worker. + +.. warning:: + + Enabling this can cause message loops; make sure you know + what you're doing. + +.. setting:: task_default_rate_limit + +task_default_rate_limit +~~~~~~~~~~~~~~~~~~~~~~~ + +The global default rate limit for tasks. + +This value is used for tasks that does not have a custom rate limit +The default is no rate limit. + +.. seealso:: + + The setting:`worker_disable_rate_limits` setting can + disable all rate limits. .. _conf-result-backend: Task result backend settings ---------------------------- -.. setting:: CELERY_RESULT_BACKEND +.. setting:: result_backend -CELERY_RESULT_BACKEND -~~~~~~~~~~~~~~~~~~~~~ -:Deprecated aliases: ``CELERY_BACKEND`` +result_backend +~~~~~~~~~~~~~~ The backend used to store task results (tombstones). Disabled by default. @@ -247,16 +412,61 @@ Can be one of the following: .. _`CouchDB`: http://www.couchdb.com/ .. _`Couchbase`: http://www.couchbase.com/ +.. setting:: result_serializer -.. setting:: CELERY_RESULT_SERIALIZER - -CELERY_RESULT_SERIALIZER -~~~~~~~~~~~~~~~~~~~~~~~~ +result_serializer +~~~~~~~~~~~~~~~~~ Result serialization format. Default is ``pickle``. See :ref:`calling-serializers` for information about supported serialization formats. +.. setting:: result_compression + +result_compression +~~~~~~~~~~~~~~~~~~ + +Optional compression method used for task results. +Supports the same options as the :setting:`task_serializer` setting. + +Default is no compression. + +.. setting:: result_expires + +result_expires +~~~~~~~~~~~~~~ + +Time (in seconds, or a :class:`~datetime.timedelta` object) for when after +stored task tombstones will be deleted. + +A built-in periodic task will delete the results after this time +(``celery.backend_cleanup``), assuming that ``celery beat`` is +enabled. The task runs daily at 4am. + +A value of :const:`None` or 0 means results will never expire (depending +on backend specifications). + +Default is to expire after 1 day. + +.. note:: + + For the moment this only works with the amqp, database, cache, redis and MongoDB + backends. + + When using the database or MongoDB backends, `celery beat` must be + running for the results to be expired. + +.. setting:: result_cache_max + +result_cache_max +~~~~~~~~~~~~~~~~ + +Result backends caches ready results used by the client. + +This is the total number of results to cache before older results are evicted. +The default is 5000. 0 or None means no limit, and a value of :const:`-1` +will disable the cache. + .. _conf-database-result-backend: Database backend settings @@ -266,26 +476,26 @@ Database URL Examples ~~~~~~~~~~~~~~~~~~~~~ To use the database backend you have to configure the -:setting:`CELERY_RESULT_BACKEND` setting with a connection URL and the ``db+`` +:setting:`result_backend` setting with a connection URL and the ``db+`` prefix: .. code-block:: python - CELERY_RESULT_BACKEND = 'db+scheme://user:password@host:port/dbname' + result_backend = 'db+scheme://user:password@host:port/dbname' Examples:: # sqlite (filename) - CELERY_RESULT_BACKEND = 'db+sqlite:///results.sqlite' + result_backend = 'db+sqlite:///results.sqlite' # mysql - CELERY_RESULT_BACKEND = 'db+mysql://scott:tiger@localhost/foo' + result_backend = 'db+mysql://scott:tiger@localhost/foo' # postgresql - CELERY_RESULT_BACKEND = 'db+postgresql://scott:tiger@localhost/mydatabase' + result_backend = 'db+postgresql://scott:tiger@localhost/mydatabase' # oracle - CELERY_RESULT_BACKEND = 'db+oracle://scott:tiger@127.0.0.1:1521/sidname' + result_backend = 'db+oracle://scott:tiger@127.0.0.1:1521/sidname' .. code-block:: python @@ -299,31 +509,31 @@ strings (which is the part of the URI that comes after the ``db+`` prefix). .. _`Connection String`: http://www.sqlalchemy.org/docs/core/engines.html#database-urls -.. setting:: CELERY_RESULT_DBURI +.. setting:: sqlalchemy_dburi -CELERY_RESULT_DBURI -~~~~~~~~~~~~~~~~~~~ +sqlalchemy_dburi +~~~~~~~~~~~~~~~~ This setting is no longer used as it's now possible to specify -the database URL directly in the :setting:`CELERY_RESULT_BACKEND` setting. +the database URL directly in the :setting:`result_backend` setting. -.. setting:: CELERY_RESULT_ENGINE_OPTIONS +.. setting:: sqlalchemy_engine_options -CELERY_RESULT_ENGINE_OPTIONS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +sqlalchemy_engine_options +~~~~~~~~~~~~~~~~~~~~~~~~~ To specify additional SQLAlchemy database engine options you can use -the :setting:`CELERY_RESULT_ENGINE_OPTIONS` setting:: +the :setting:`sqlalchmey_engine_options` setting:: # echo enables verbose logging from SQLAlchemy. - CELERY_RESULT_ENGINE_OPTIONS = {'echo': True} + sqlalchemy_engine_options = {'echo': True} -.. setting:: CELERY_RESULT_DB_SHORT_LIVED_SESSIONS +.. setting:: sqlalchemy_short_lived_sessions -Short lived sessions -~~~~~~~~~~~~~~~~~~~~ +sqlalchemy_short_lived_sessions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - CELERY_RESULT_DB_SHORT_LIVED_SESSIONS = True + sqlalchemy_short_lived_sessions = True Short lived sessions are disabled by default. If enabled they can drastically reduce performance, especially on systems processing lots of tasks. This option is useful @@ -332,10 +542,10 @@ going stale through inactivity. For example, intermittent errors like `(OperationalError) (2006, 'MySQL server has gone away')` can be fixed by enabling short lived sessions. This option only affects the database backend. -Specifying Table Names -~~~~~~~~~~~~~~~~~~~~~~ +.. setting:: sqlalchemy_table_names -.. setting:: CELERY_RESULT_DB_TABLENAMES +sqlalchemy_table_names +~~~~~~~~~~~~~~~~~~~~~~ When SQLAlchemy is configured as the result backend, Celery automatically creates two tables to store result metadata for tasks. This setting allows @@ -344,7 +554,7 @@ you to customize the table names: .. code-block:: python # use custom table names for the database result backend. - CELERY_RESULT_DB_TABLENAMES = { + sqlalchemy_table_names = { 'task': 'myapp_taskmeta', 'group': 'myapp_groupmeta', } @@ -356,8 +566,10 @@ RPC backend settings .. _conf-amqp-result-backend: -CELERY_RESULT_PERSISTENT -~~~~~~~~~~~~~~~~~~~~~~~~ +.. setting:: result_persistent + +result_persistent +~~~~~~~~~~~~~~~~~ If set to :const:`True`, result messages will be persistent. This means the messages will not be lost after a broker restart. The default is for the @@ -368,9 +580,8 @@ Example configuration .. code-block:: python - CELERY_RESULT_BACKEND = 'rpc://' - CELERY_RESULT_PERSISTENT = False - + result_backend = 'rpc://' + result_persistent = False .. _conf-cache-result-backend: @@ -386,45 +597,47 @@ Using a single memcached server: .. code-block:: python - CELERY_RESULT_BACKEND = 'cache+memcached://127.0.0.1:11211/' + result_backend = 'cache+memcached://127.0.0.1:11211/' Using multiple memcached servers: .. code-block:: python - CELERY_RESULT_BACKEND = """ + result_backend = """ cache+memcached://172.19.26.240:11211;172.19.26.242:11211/ """.strip() -.. setting:: CELERY_CACHE_BACKEND_OPTIONS - The "memory" backend stores the cache in memory only: .. code-block:: python - CELERY_RESULT_BACKEND = 'cache' - CELERY_CACHE_BACKEND = 'memory' + result_backend = 'cache' + cache_backend = 'memory' -CELERY_CACHE_BACKEND_OPTIONS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. setting:: cache_backend_options -You can set pylibmc options using the :setting:`CELERY_CACHE_BACKEND_OPTIONS` +cache_backend_options +~~~~~~~~~~~~~~~~~~~~~ + +You can set pylibmc options using the :setting:`cache_backend_options` setting: .. code-block:: python - CELERY_CACHE_BACKEND_OPTIONS = {'binary': True, - 'behaviors': {'tcp_nodelay': True}} + cache_backend_options = { + 'binary': True, + 'behaviors': {'tcp_nodelay': True}, + } .. _`pylibmc`: http://sendapatch.se/projects/pylibmc/ -.. setting:: CELERY_CACHE_BACKEND +.. setting:: cache_backend -CELERY_CACHE_BACKEND -~~~~~~~~~~~~~~~~~~~~ +cache_backend +~~~~~~~~~~~~~ This setting is no longer used as it's now possible to specify -the cache backend directly in the :setting:`CELERY_RESULT_BACKEND` setting. +the cache backend directly in the :setting:`result_backend` setting. .. _conf-redis-result-backend: @@ -445,18 +658,18 @@ Configuring the backend URL $ pip install redis -This backend requires the :setting:`CELERY_RESULT_BACKEND` +This backend requires the :setting:`result_backend` setting to be set to a Redis URL:: - CELERY_RESULT_BACKEND = 'redis://:password@host:port/db' + result_backend = 'redis://:password@host:port/db' For example:: - CELERY_RESULT_BACKEND = 'redis://localhost/0' + result_backend = 'redis://localhost/0' which is the same as:: - CELERY_RESULT_BACKEND = 'redis://' + result_backend = 'redis://' The fields of the URL are defined as follows: @@ -477,10 +690,10 @@ The db can include an optional leading slash. Password used to connect to the database. -.. setting:: CELERY_REDIS_MAX_CONNECTIONS +.. setting:: redis_max_connections -CELERY_REDIS_MAX_CONNECTIONS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +redis_max_connections +~~~~~~~~~~~~~~~~~~~~~ Maximum number of connections available in the Redis connection pool used for sending and retrieving results. @@ -495,9 +708,9 @@ MongoDB backend settings The MongoDB backend requires the :mod:`pymongo` library: http://github.com/mongodb/mongo-python-driver/tree/master -.. setting:: CELERY_MONGODB_BACKEND_SETTINGS +.. setting:: mongodb_backend_settings -CELERY_MONGODB_BACKEND_SETTINGS +mongodb_backend_settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This is a dict supporting the following keys: @@ -529,8 +742,8 @@ Example configuration .. code-block:: python - CELERY_RESULT_BACKEND = 'mongodb://192.168.1.100:30000/' - CELERY_MONGODB_BACKEND_SETTINGS = { + result_backend = 'mongodb://192.168.1.100:30000/' + mongodb_backend_settings = { 'database': 'mydb', 'taskmeta_collection': 'my_taskmeta_collection', } @@ -553,60 +766,60 @@ new_cassandra backend settings This backend requires the following configuration directives to be set. -.. setting:: CASSANDRA_SERVERS +.. setting:: cassandra_servers -CASSANDRA_SERVERS +cassandra_servers ~~~~~~~~~~~~~~~~~ List of ``host`` Cassandra servers. e.g.:: - CASSANDRA_SERVERS = ['localhost'] + cassandra_servers = ['localhost'] -.. setting:: CASSANDRA_PORT +.. setting:: cassandra_port -CASSANDRA_PORT +cassandra_port ~~~~~~~~~~~~~~ Port to contact the Cassandra servers on. Default is 9042. -.. setting:: CASSANDRA_KEYSPACE +.. setting:: cassandra_keyspace -CASSANDRA_KEYSPACE +cassandra_keyspace ~~~~~~~~~~~~~~~~~~ The keyspace in which to store the results. e.g.:: - CASSANDRA_KEYSPACE = 'tasks_keyspace' + cassandra_keyspace = 'tasks_keyspace' -.. setting:: CASSANDRA_COLUMN_FAMILY +.. setting:: cassandra_column_family -CASSANDRA_TABLE +cassandra_column_family ~~~~~~~~~~~~~~~~~~~~~~~ The table (column family) in which to store the results. e.g.:: - CASSANDRA_TABLE = 'tasks' + cassandra_column_family = 'tasks' -.. setting:: CASSANDRA_READ_CONSISTENCY +.. setting:: cassandra_read_consistency -CASSANDRA_READ_CONSISTENCY +cassandra_read_consistency ~~~~~~~~~~~~~~~~~~~~~~~~~~ The read consistency used. Values can be ``ONE``, ``TWO``, ``THREE``, ``QUORUM``, ``ALL``, ``LOCAL_QUORUM``, ``EACH_QUORUM``, ``LOCAL_ONE``. -.. setting:: CASSANDRA_WRITE_CONSISTENCY +.. setting:: cassandra_write_consistency -CASSANDRA_WRITE_CONSISTENCY +cassandra_write_consistency ~~~~~~~~~~~~~~~~~~~~~~~~~~~ The write consistency used. Values can be ``ONE``, ``TWO``, ``THREE``, ``QUORUM``, ``ALL``, ``LOCAL_QUORUM``, ``EACH_QUORUM``, ``LOCAL_ONE``. -.. setting:: CASSANDRA_ENTRY_TTL +.. setting:: cassandra_entry_ttl -CASSANDRA_ENTRY_TTL +cassandra_entry_ttl ~~~~~~~~~~~~~~~~~~~ Time-to-live for status entries. They will expire and be removed after that many seconds @@ -617,110 +830,14 @@ Example configuration .. code-block:: python - CASSANDRA_SERVERS = ['localhost'] - CASSANDRA_KEYSPACE = 'celery' - CASSANDRA_COLUMN_FAMILY = 'task_results' - CASSANDRA_READ_CONSISTENCY = 'ONE' - CASSANDRA_WRITE_CONSISTENCY = 'ONE' - CASSANDRA_ENTRY_TTL = 86400 + cassandra_servers = ['localhost'] + cassandra_keyspace = 'celery' + cassandra_column_family = 'task_results' + cassandra_read_consistency = 'ONE' + cassandra_write_consistency = 'ONE' + cassandra_entry_ttl = 86400 -.. _conf-cassandra-result-backend: - -Cassandra backend settings --------------------------- - -.. note:: - - The Cassandra backend requires the :mod:`pycassa` library: - http://pypi.python.org/pypi/pycassa/ - - To install the pycassa package use `pip` or `easy_install`: - - .. code-block:: console - - $ pip install pycassa - -This backend requires the following configuration directives to be set. - -.. setting:: CASSANDRA_SERVERS - -CASSANDRA_SERVERS -~~~~~~~~~~~~~~~~~ - -List of ``host:port`` Cassandra servers. e.g.:: - - CASSANDRA_SERVERS = ['localhost:9160'] - -.. setting:: CASSANDRA_KEYSPACE - -CASSANDRA_KEYSPACE -~~~~~~~~~~~~~~~~~~ - -The keyspace in which to store the results. e.g.:: - - CASSANDRA_KEYSPACE = 'tasks_keyspace' - -.. setting:: CASSANDRA_COLUMN_FAMILY - -CASSANDRA_COLUMN_FAMILY -~~~~~~~~~~~~~~~~~~~~~~~ - -The column family in which to store the results. e.g.:: - - CASSANDRA_COLUMN_FAMILY = 'tasks' - -.. setting:: CASSANDRA_READ_CONSISTENCY - -CASSANDRA_READ_CONSISTENCY -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The read consistency used. Values can be ``ONE``, ``QUORUM`` or ``ALL``. - -.. setting:: CASSANDRA_WRITE_CONSISTENCY - -CASSANDRA_WRITE_CONSISTENCY -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The write consistency used. Values can be ``ONE``, ``QUORUM`` or ``ALL``. - -.. setting:: CASSANDRA_DETAILED_MODE - -CASSANDRA_DETAILED_MODE -~~~~~~~~~~~~~~~~~~~~~~~ - -Enable or disable detailed mode. Default is :const:`False`. -This mode allows to use the power of Cassandra wide columns to -store all states for a task as a wide column, instead of only the last one. - -To use this mode, you need to configure your ColumnFamily to -use the ``TimeUUID`` type as a comparator:: - - create column family task_results with comparator = TimeUUIDType; - -CASSANDRA_OPTIONS -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Options to be passed to the `pycassa connection pool`_ (optional). - -.. _`pycassa connection pool`: http://pycassa.github.com/pycassa/api/pycassa/pool.html - -Example configuration -~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: python - - CASSANDRA_SERVERS = ['localhost:9160'] - CASSANDRA_KEYSPACE = 'celery' - CASSANDRA_COLUMN_FAMILY = 'task_results' - CASSANDRA_READ_CONSISTENCY = 'ONE' - CASSANDRA_WRITE_CONSISTENCY = 'ONE' - CASSANDRA_DETAILED_MODE = True - CASSANDRA_OPTIONS = { - 'timeout': 300, - 'max_retries': 10 - } - -.. _conf-riak-result-backend: +.. _conf-riak-result-backend: Riak backend settings --------------------- @@ -736,18 +853,18 @@ Riak backend settings $ pip install riak -This backend requires the :setting:`CELERY_RESULT_BACKEND` +This backend requires the :setting:`result_backend` setting to be set to a Riak URL:: - CELERY_RESULT_BACKEND = "riak://host:port/bucket" + result_backend = "riak://host:port/bucket" For example:: - CELERY_RESULT_BACKEND = "riak://localhost/celery + result_backend = "riak://localhost/celery which is the same as:: - CELERY_RESULT_BACKEND = "riak://" + result_backend = "riak://" The fields of the URL are defined as follows: @@ -766,10 +883,10 @@ The bucket needs to be a string with ascii characters only. Altenatively, this backend can be configured with the following configuration directives. -.. setting:: CELERY_RIAK_BACKEND_SETTINGS +.. setting:: riak_backend_settings -CELERY_RIAK_BACKEND_SETTINGS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +riak_backend_settings +~~~~~~~~~~~~~~~~~~~~~ This is a dict supporting the following keys: @@ -784,7 +901,7 @@ This is a dict supporting the following keys: * protocol The protocol to use to connect to the Riak server. This is not configurable - via :setting:`CELERY_RESULT_BACKEND` + via :setting:`result_backend` .. _conf-ironcache-result-backend: @@ -802,9 +919,9 @@ IronCache backend settings $ pip install iron_celery -IronCache is configured via the URL provided in :setting:`CELERY_RESULT_BACKEND`, for example:: +IronCache is configured via the URL provided in :setting:`result_backend`, for example:: - CELERY_RESULT_BACKEND = 'ironcache://project_id:token@' + result_backend = 'ironcache://project_id:token@' Or to change the cache name:: @@ -812,7 +929,6 @@ Or to change the cache name:: For more information, see: https://github.com/iron-io/iron_celery - .. _conf-couchbase-result-backend: Couchbase backend settings @@ -829,16 +945,15 @@ Couchbase backend settings $ pip install couchbase -This backend can be configured via the :setting:`CELERY_RESULT_BACKEND` +This backend can be configured via the :setting:`result_backend` set to a couchbase URL:: - CELERY_RESULT_BACKEND = 'couchbase://username:password@host:port/bucket' - + result_backend = 'couchbase://username:password@host:port/bucket' -.. setting:: CELERY_COUCHBASE_BACKEND_SETTINGS +.. setting:: couchbase_backend_settings -CELERY_COUCHBASE_BACKEND_SETTINGS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +couchbase_backend_settings +~~~~~~~~~~~~~~~~~~~~~~~~~~ This is a dict supporting the following keys: @@ -858,7 +973,6 @@ This is a dict supporting the following keys: * password Password to authenticate to the Couchbase server (optional). - .. _conf-couchdb-result-backend: CouchDB backend settings @@ -875,11 +989,10 @@ CouchDB backend settings $ pip install pycouchdb -This backend can be configured via the :setting:`CELERY_RESULT_BACKEND` +This backend can be configured via the :setting:`result_backend` set to a couchdb URL:: - CELERY_RESULT_BACKEND = 'couchdb://username:password@host:port/container' - + result_backend = 'couchdb://username:password@host:port/container' The URL is formed out of the following parts: @@ -915,27 +1028,27 @@ AMQP backend settings expire results. If you are running an older version of RabbitMQ you should disable result expiration like this: - CELERY_TASK_RESULT_EXPIRES = None + result_expires = None -.. setting:: CELERY_RESULT_EXCHANGE +.. setting:: result_exchange -CELERY_RESULT_EXCHANGE -~~~~~~~~~~~~~~~~~~~~~~ +result_exchange +~~~~~~~~~~~~~~~ Name of the exchange to publish results in. Default is `celeryresults`. -.. setting:: CELERY_RESULT_EXCHANGE_TYPE +.. setting:: result_exchange_type -CELERY_RESULT_EXCHANGE_TYPE -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +result_exchange_type +~~~~~~~~~~~~~~~~~~~~ The exchange type of the result exchange. Default is to use a `direct` exchange. -.. setting:: CELERY_RESULT_PERSISTENT +.. setting:: result_persistent -CELERY_RESULT_PERSISTENT -~~~~~~~~~~~~~~~~~~~~~~~~ +result_persistent +~~~~~~~~~~~~~~~~~ If set to :const:`True`, result messages will be persistent. This means the messages will not be lost after a broker restart. The default is for the @@ -946,9 +1059,8 @@ Example configuration .. code-block:: python - CELERY_RESULT_BACKEND = 'amqp' - CELERY_TASK_RESULT_EXPIRES = 18000 # 5 hours. - + result_backend = 'amqp' + result_expires = 18000 # 5 hours. .. _conf-messaging: @@ -957,10 +1069,10 @@ Message Routing .. _conf-messaging-routing: -.. setting:: CELERY_QUEUES +.. setting:: task_queues -CELERY_QUEUES -~~~~~~~~~~~~~ +task_queues +~~~~~~~~~~~ Most users will not want to specify this setting and should rather use the :ref:`automatic routing facilities `. @@ -977,11 +1089,11 @@ Also see :ref:`routing-basics` for more information. The default is a queue/exchange/binding key of ``celery``, with exchange type ``direct``. -See also :setting:`CELERY_ROUTES` +See also :setting:`task_routes` -.. setting:: CELERY_ROUTES +.. setting:: task_routes -CELERY_ROUTES +task_routes ~~~~~~~~~~~~~ A list of routers, or a single router used to route tasks to queues. @@ -998,14 +1110,17 @@ Examples: .. code-block:: python - CELERY_ROUTES = {"celery.ping": "default", - "mytasks.add": "cpu-bound", - "video.encode": { - "queue": "video", - "exchange": "media" - "routing_key": "media.video.encode"}} + task_routes = { + "celery.ping": "default", + "mytasks.add": "cpu-bound", + "video.encode": { + "queue": "video", + "exchange": "media" + "routing_key": "media.video.encode", + }, + } - CELERY_ROUTES = ("myapp.tasks.Router", {"celery.ping": "default}) + task_routes = ("myapp.tasks.Router", {"celery.ping": "default}) Where ``myapp.tasks.Router`` could be: @@ -1018,7 +1133,7 @@ Where ``myapp.tasks.Router`` could be: return "default" ``route_for_task`` may return a string or a dict. A string then means -it's a queue name in :setting:`CELERY_QUEUES`, a dict means it's a custom route. +it's a queue name in :setting:`task_queues`, a dict means it's a custom route. When sending tasks, the routers are consulted in order. The first router that doesn't return ``None`` is the route to use. The message options @@ -1047,19 +1162,27 @@ the final message options will be: (and any default message options defined in the :class:`~celery.task.base.Task` class) -Values defined in :setting:`CELERY_ROUTES` have precedence over values defined in -:setting:`CELERY_QUEUES` when merging the two. +Values defined in :setting:`task_routes` have precedence over values defined in +:setting:`task_queues` when merging the two. With the follow settings: .. code-block:: python - CELERY_QUEUES = {"cpubound": {"exchange": "cpubound", - "routing_key": "cpubound"}} + task_queues = { + "cpubound": { + "exchange": "cpubound", + "routing_key": "cpubound", + }, + } - CELERY_ROUTES = {"tasks.add": {"queue": "cpubound", - "routing_key": "tasks.add", - "serializer": "json"}} + task_routes = { + "tasks.add": { + "queue": "cpubound", + "routing_key": "tasks.add", + "serializer": "json", + }, + } The final routing options for ``tasks.add`` will become: @@ -1071,11 +1194,10 @@ The final routing options for ``tasks.add`` will become: See :ref:`routers` for more examples. +.. setting:: task_queue_ha_policy -.. setting:: CELERY_QUEUE_HA_POLICY - -CELERY_QUEUE_HA_POLICY -~~~~~~~~~~~~~~~~~~~~~~ +task_queue_ha_policy +~~~~~~~~~~~~~~~~~~~~ :brokers: RabbitMQ This will set the default HA policy for a queue, and the value @@ -1083,25 +1205,24 @@ can either be a string (usually ``all``): .. code-block:: python - CELERY_QUEUE_HA_POLICY = 'all' + task_queue_ha_policy = 'all' Using 'all' will replicate the queue to all current nodes, Or you can give it a list of nodes to replicate to: .. code-block:: python - CELERY_QUEUE_HA_POLICY = ['rabbit@host1', 'rabbit@host2'] - + task_queue_ha_policy = ['rabbit@host1', 'rabbit@host2'] Using a list will implicitly set ``x-ha-policy`` to 'nodes' and ``x-ha-policy-params`` to the given list of nodes. See http://www.rabbitmq.com/ha.html for more information. -.. setting:: CELERY_WORKER_DIRECT +.. setting:: worker_direct -CELERY_WORKER_DIRECT -~~~~~~~~~~~~~~~~~~~~ +worker_direct +~~~~~~~~~~~~~ This option enables so that every worker has a dedicated queue, so that tasks can be routed to specific workers. @@ -1117,30 +1238,29 @@ becomes:: Then you can route the task to the task by specifying the hostname as the routing key and the ``C.dq`` exchange:: - CELERY_ROUTES = { + task_routes = { 'tasks.add': {'exchange': 'C.dq', 'routing_key': 'w1@example.com'} } -.. setting:: CELERY_CREATE_MISSING_QUEUES +.. setting:: task_create_missing_queues -CELERY_CREATE_MISSING_QUEUES -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +task_create_missing_queues +~~~~~~~~~~~~~~~~~~~~~~~~~~ If enabled (default), any queues specified that are not defined in -:setting:`CELERY_QUEUES` will be automatically created. See +:setting:`task_queues` will be automatically created. See :ref:`routing-automatic`. -.. setting:: CELERY_DEFAULT_QUEUE +.. setting:: task_default_queue -CELERY_DEFAULT_QUEUE -~~~~~~~~~~~~~~~~~~~~ +task_default_queue +~~~~~~~~~~~~~~~~~~ The name of the default queue used by `.apply_async` if the message has no route or no custom queue has been specified. - -This queue must be listed in :setting:`CELERY_QUEUES`. -If :setting:`CELERY_QUEUES` is not specified then it is automatically +This queue must be listed in :setting:`task_queues`. +If :setting:`task_queues` is not specified then it is automatically created containing one queue entry, where this name is used as the name of that queue. @@ -1150,39 +1270,39 @@ The default is: `celery`. :ref:`routing-changing-default-queue` -.. setting:: CELERY_DEFAULT_EXCHANGE +.. setting:: task_default_exchange -CELERY_DEFAULT_EXCHANGE -~~~~~~~~~~~~~~~~~~~~~~~ +task_default_exchange +~~~~~~~~~~~~~~~~~~~~~ Name of the default exchange to use when no custom exchange is -specified for a key in the :setting:`CELERY_QUEUES` setting. +specified for a key in the :setting:`task_queues` setting. The default is: `celery`. -.. setting:: CELERY_DEFAULT_EXCHANGE_TYPE +.. setting:: task_default_exchange_type -CELERY_DEFAULT_EXCHANGE_TYPE -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +task_default_exchange_type +~~~~~~~~~~~~~~~~~~~~~~~~~~ Default exchange type used when no custom exchange type is specified -for a key in the :setting:`CELERY_QUEUES` setting. +for a key in the :setting:`task_queues` setting. The default is: `direct`. -.. setting:: CELERY_DEFAULT_ROUTING_KEY +.. setting:: task_default_routing_key -CELERY_DEFAULT_ROUTING_KEY -~~~~~~~~~~~~~~~~~~~~~~~~~~ +task_default_routing_key +~~~~~~~~~~~~~~~~~~~~~~~~ The default routing key used when no custom routing key -is specified for a key in the :setting:`CELERY_QUEUES` setting. +is specified for a key in the :setting:`task_queues` setting. The default is: `celery`. -.. setting:: CELERY_DEFAULT_DELIVERY_MODE +.. setting:: task_default_delivery_mode -CELERY_DEFAULT_DELIVERY_MODE -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +task_default_delivery_mode +~~~~~~~~~~~~~~~~~~~~~~~~~~ Can be `transient` or `persistent`. The default is to send persistent messages. @@ -1192,59 +1312,9 @@ persistent messages. Broker Settings --------------- -.. setting:: CELERY_ACCEPT_CONTENT +.. setting:: broker_url -CELERY_ACCEPT_CONTENT -~~~~~~~~~~~~~~~~~~~~~ - -A whitelist of content-types/serializers to allow. - -If a message is received that is not in this list then -the message will be discarded with an error. - -By default any content type is enabled (including pickle and yaml) -so make sure untrusted parties do not have access to your broker. -See :ref:`guide-security` for more. - -Example:: - - # using serializer name - CELERY_ACCEPT_CONTENT = ['json'] - - # or the actual content-type (MIME) - CELERY_ACCEPT_CONTENT = ['application/json'] - -.. setting:: BROKER_FAILOVER_STRATEGY - -BROKER_FAILOVER_STRATEGY -~~~~~~~~~~~~~~~~~~~~~~~~ - -Default failover strategy for the broker Connection object. If supplied, -may map to a key in 'kombu.connection.failover_strategies', or be a reference -to any method that yields a single item from a supplied list. - -Example:: - - # Random failover strategy - def random_failover_strategy(servers): - it = list(it) # don't modify callers list - shuffle = random.shuffle - for _ in repeat(None): - shuffle(it) - yield it[0] - - BROKER_FAILOVER_STRATEGY=random_failover_strategy - -.. setting:: BROKER_TRANSPORT - -BROKER_TRANSPORT -~~~~~~~~~~~~~~~~ -:Aliases: ``BROKER_BACKEND`` -:Deprecated aliases: ``CARROT_BACKEND`` - -.. setting:: BROKER_URL - -BROKER_URL +broker_url ~~~~~~~~~~ Default broker URL. This must be an URL in the form of:: @@ -1264,23 +1334,44 @@ It can also be a fully qualified path to your own transport implementation. More than broker URL, of the same transport, can also be specified. The broker URLs can be passed in as a single string that is semicolon delimited:: - BROKER_URL = 'transport://userid:password@hostname:port//;transport://userid:password@hostname:port//' + broker_url = 'transport://userid:password@hostname:port//;transport://userid:password@hostname:port//' Or as a list:: - BROKER_URL = [ + broker_url = [ 'transport://userid:password@localhost:port//', 'transport://userid:password@hostname:port//' ] -The brokers will then be used in the :setting:`BROKER_FAILOVER_STRATEGY`. +The brokers will then be used in the :setting:`broker_failover_strategy`. See :ref:`kombu:connection-urls` in the Kombu documentation for more information. -.. setting:: BROKER_HEARTBEAT +.. setting:: broker_failover_strategy + +broker_failover_strategy +~~~~~~~~~~~~~~~~~~~~~~~~ + +Default failover strategy for the broker Connection object. If supplied, +may map to a key in 'kombu.connection.failover_strategies', or be a reference +to any method that yields a single item from a supplied list. + +Example:: + + # Random failover strategy + def random_failover_strategy(servers): + it = list(it) # don't modify callers list + shuffle = random.shuffle + for _ in repeat(None): + shuffle(it) + yield it[0] + + broker_failover_strategy = random_failover_strategy + +.. setting:: broker_heartbeat -BROKER_HEARTBEAT +broker_heartbeat ~~~~~~~~~~~~~~~~ :transports supported: ``pyamqp`` @@ -1293,29 +1384,28 @@ Heartbeats are disabled by default. If the heartbeat value is 10 seconds, then the heartbeat will be monitored at the interval specified -by the :setting:`BROKER_HEARTBEAT_CHECKRATE` setting, which by default is +by the :setting:`broker_heartbeat_checkrate` setting, which by default is double the rate of the heartbeat value (so for the default 10 seconds, the heartbeat is checked every 5 seconds). -.. setting:: BROKER_HEARTBEAT_CHECKRATE +.. setting:: broker_heartbeat_checkrate -BROKER_HEARTBEAT_CHECKRATE +broker_heartbeat_checkrate ~~~~~~~~~~~~~~~~~~~~~~~~~~ :transports supported: ``pyamqp`` At intervals the worker will monitor that the broker has not missed too many heartbeats. The rate at which this is checked is calculated -by dividing the :setting:`BROKER_HEARTBEAT` value with this value, +by dividing the :setting:`broker_heartbeat` value with this value, so if the heartbeat is 10.0 and the rate is the default 2.0, the check will be performed every 5 seconds (twice the heartbeat sending rate). -.. setting:: BROKER_USE_SSL +.. setting:: broker_use_ssl -BROKER_USE_SSL +broker_use_ssl ~~~~~~~~~~~~~~ :transports supported: ``pyamqp``, ``redis`` - Toggles SSL usage on broker connection and SSL settings. If ``True`` the connection will use SSL with default SSL settings. @@ -1334,7 +1424,7 @@ certificate authority: import ssl - BROKER_USE_SSL = { + broker_use_ssl = { 'keyfile': '/var/ssl/private/worker-key.pem', 'certfile': '/var/ssl/amqp-server-cert.pem', 'ca_certs': '/var/ssl/myca.pem', @@ -1343,14 +1433,14 @@ certificate authority: .. warning:: - Be careful using ``BROKER_USE_SSL=True``, it is possible that your default + Be careful using ``broker_use_ssl=True``, it is possible that your default configuration do not validate the server cert at all, please read Python `ssl module security considerations `_. -.. setting:: BROKER_POOL_LIMIT +.. setting:: broker_pool_limit -BROKER_POOL_LIMIT +broker_pool_limit ~~~~~~~~~~~~~~~~~ .. versionadded:: 2.3 @@ -1368,30 +1458,30 @@ connections will be established and closed for every use. Default (since 2.5) is to use a pool of 10 connections. -.. setting:: BROKER_CONNECTION_TIMEOUT +.. setting:: broker_connection_timeout -BROKER_CONNECTION_TIMEOUT +broker_connection_timeout ~~~~~~~~~~~~~~~~~~~~~~~~~ The default timeout in seconds before we give up establishing a connection to the AMQP server. Default is 4 seconds. -.. setting:: BROKER_CONNECTION_RETRY +.. setting:: broker_connection_retry -BROKER_CONNECTION_RETRY +broker_connection_retry ~~~~~~~~~~~~~~~~~~~~~~~ Automatically try to re-establish the connection to the AMQP broker if lost. The time between retries is increased for each retry, and is -not exhausted before :setting:`BROKER_CONNECTION_MAX_RETRIES` is +not exhausted before :setting:`broker_connection_max_retries` is exceeded. This behavior is on by default. -.. setting:: BROKER_CONNECTION_MAX_RETRIES +.. setting:: broker_connection_max_retries -BROKER_CONNECTION_MAX_RETRIES +broker_connection_max_retries ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Maximum number of retries before we give up re-establishing a connection @@ -1401,16 +1491,16 @@ If this is set to :const:`0` or :const:`None`, we will retry forever. Default is 100 retries. -.. setting:: BROKER_LOGIN_METHOD +.. setting:: broker_login_method -BROKER_LOGIN_METHOD +broker_login_method ~~~~~~~~~~~~~~~~~~~ Set custom amqp login method, default is ``AMQPLAIN``. -.. setting:: BROKER_TRANSPORT_OPTIONS +.. setting:: broker_transport_options -BROKER_TRANSPORT_OPTIONS +broker_transport_options ~~~~~~~~~~~~~~~~~~~~~~~~ .. versionadded:: 2.2 @@ -1424,232 +1514,81 @@ transports): .. code-block:: python - BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 18000} # 5 hours - -.. _conf-task-execution: - -Task execution settings ------------------------ - -.. setting:: CELERY_ALWAYS_EAGER + broker_transport_options = {'visibility_timeout': 18000} # 5 hours -CELERY_ALWAYS_EAGER -~~~~~~~~~~~~~~~~~~~ +.. _conf-worker: -If this is :const:`True`, all tasks will be executed locally by blocking until -the task returns. ``apply_async()`` and ``Task.delay()`` will return -an :class:`~celery.result.EagerResult` instance, which emulates the API -and behavior of :class:`~celery.result.AsyncResult`, except the result -is already evaluated. +Worker +------ -That is, tasks will be executed locally instead of being sent to -the queue. +.. setting:: imports -.. setting:: CELERY_EAGER_PROPAGATES_EXCEPTIONS +imports +~~~~~~~ -CELERY_EAGER_PROPAGATES_EXCEPTIONS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +A sequence of modules to import when the worker starts. -If this is :const:`True`, eagerly executed tasks (applied by `task.apply()`, -or when the :setting:`CELERY_ALWAYS_EAGER` setting is enabled), will -propagate exceptions. +This is used to specify the task modules to import, but also +to import signal handlers and additional remote control commands, etc. -It's the same as always running ``apply()`` with ``throw=True``. +The modules will be imported in the original order. -.. setting:: CELERY_IGNORE_RESULT +.. setting:: include -CELERY_IGNORE_RESULT -~~~~~~~~~~~~~~~~~~~~ +include +~~~~~~~ -Whether to store the task return values or not (tombstones). -If you still want to store errors, just not successful return values, -you can set :setting:`CELERY_STORE_ERRORS_EVEN_IF_IGNORED`. +Exact same semantics as :setting:`imports`, but can be used as a means +to have different import categories. -.. setting:: CELERY_MESSAGE_COMPRESSION +The modules in this setting are imported after the modules in +:setting:`imports`. -CELERY_MESSAGE_COMPRESSION -~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _conf-concurrency: -Default compression used for task messages. -Can be ``gzip``, ``bzip2`` (if available), or any custom -compression schemes registered in the Kombu compression registry. +.. setting:: worker_concurrency -The default is to send uncompressed messages. +worker_concurrency +~~~~~~~~~~~~~~~~~~ -.. setting:: CELERY_TASK_PROTOCOL +The number of concurrent worker processes/threads/green threads executing +tasks. -CELERY_TASK_PROTOCOL -~~~~~~~~~~~~~~~~~~~~ +If you're doing mostly I/O you can have more processes, +but if mostly CPU-bound, try to keep it close to the +number of CPUs on your machine. If not set, the number of CPUs/cores +on the host will be used. -Default task message protocol version. -Supports protocols: 1 and 2 (default is 1 for backwards compatibility). +Defaults to the number of available CPUs. -.. setting:: CELERY_TASK_RESULT_EXPIRES +.. setting:: worker_prefetch_multiplier -CELERY_TASK_RESULT_EXPIRES +worker_prefetch_multiplier ~~~~~~~~~~~~~~~~~~~~~~~~~~ -Time (in seconds, or a :class:`~datetime.timedelta` object) for when after -stored task tombstones will be deleted. - -A built-in periodic task will delete the results after this time -(``celery.backend_cleanup``), assuming that ``celery beat`` is -enabled. The task runs daily at 4am. +How many messages to prefetch at a time multiplied by the number of +concurrent processes. The default is 4 (four messages for each +process). The default setting is usually a good choice, however -- if you +have very long running tasks waiting in the queue and you have to start the +workers, note that the first worker to start will receive four times the +number of messages initially. Thus the tasks may not be fairly distributed +to the workers. -A value of :const:`None` or 0 means results will never expire (depending -on backend specifications). +To disable prefetching, set :setting:`worker_prefetch_multiplier` to 1. +Changing that setting to 0 will allow the worker to keep consuming +as many messages as it wants. -Default is to expire after 1 day. +For more on prefetching, read :ref:`optimizing-prefetch-limit` .. note:: - For the moment this only works with the amqp, database, cache, redis and MongoDB - backends. - - When using the database or MongoDB backends, `celery beat` must be - running for the results to be expired. - -.. setting:: CELERY_MAX_CACHED_RESULTS - -CELERY_MAX_CACHED_RESULTS -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Result backends caches ready results used by the client. - -This is the total number of results to cache before older results are evicted. -The default is 5000. 0 or None means no limit, and a value of :const:`-1` -will disable the cache. - -.. setting:: CELERY_TRACK_STARTED - -CELERY_TRACK_STARTED -~~~~~~~~~~~~~~~~~~~~ - -If :const:`True` the task will report its status as "started" when the -task is executed by a worker. The default value is :const:`False` as -the normal behaviour is to not report that level of granularity. Tasks -are either pending, finished, or waiting to be retried. Having a "started" -state can be useful for when there are long running tasks and there is a -need to report which task is currently running. - -.. setting:: CELERY_TASK_SERIALIZER - -CELERY_TASK_SERIALIZER -~~~~~~~~~~~~~~~~~~~~~~ - -A string identifying the default serialization method to use. Can be -`pickle` (default), `json`, `yaml`, `msgpack` or any custom serialization -methods that have been registered with :mod:`kombu.serialization.registry`. - -.. seealso:: - - :ref:`calling-serializers`. - -.. setting:: CELERY_TASK_PUBLISH_RETRY - -CELERY_TASK_PUBLISH_RETRY -~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 2.2 - -Decides if publishing task messages will be retried in the case -of connection loss or other connection errors. -See also :setting:`CELERY_TASK_PUBLISH_RETRY_POLICY`. - -Enabled by default. - -.. setting:: CELERY_TASK_PUBLISH_RETRY_POLICY - -CELERY_TASK_PUBLISH_RETRY_POLICY -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 2.2 - -Defines the default policy when retrying publishing a task message in -the case of connection loss or other connection errors. - -See :ref:`calling-retry` for more information. - -.. setting:: CELERY_DEFAULT_RATE_LIMIT - -CELERY_DEFAULT_RATE_LIMIT -~~~~~~~~~~~~~~~~~~~~~~~~~ - -The global default rate limit for tasks. - -This value is used for tasks that does not have a custom rate limit -The default is no rate limit. - -.. setting:: CELERY_DISABLE_RATE_LIMITS - -CELERY_DISABLE_RATE_LIMITS -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Disable all rate limits, even if tasks has explicit rate limits set. + Tasks with ETA/countdown are not affected by prefetch limits. -.. setting:: CELERY_ACKS_LATE +.. setting:: worker_lost_wait -CELERY_ACKS_LATE +worker_lost_wait ~~~~~~~~~~~~~~~~ -Late ack means the task messages will be acknowledged **after** the task -has been executed, not *just before*, which is the default behavior. - -.. seealso:: - - FAQ: :ref:`faq-acks_late-vs-retry`. - -.. setting:: CELERY_REJECT_ON_WORKER_LOST - -CELERY_REJECT_ON_WORKER_LOST -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Even if :attr:`acks_late` is enabled, the worker will -acknowledge tasks when the worker process executing them abrubtly -exits or is signalled (e.g. :sig:`KILL`/:sig:`INT`, etc). - -Setting this to true allows the message to be requeued instead, -so that the task will execute again by the same worker, or another -worker. - -.. warning:: - - Enabling this can cause message loops; make sure you know - what you're doing. - -.. _conf-worker: - -Worker ------- - -.. setting:: CELERY_IMPORTS - -CELERY_IMPORTS -~~~~~~~~~~~~~~ - -A sequence of modules to import when the worker starts. - -This is used to specify the task modules to import, but also -to import signal handlers and additional remote control commands, etc. - -The modules will be imported in the original order. - -.. setting:: CELERY_INCLUDE - -CELERY_INCLUDE -~~~~~~~~~~~~~~ - -Exact same semantics as :setting:`CELERY_IMPORTS`, but can be used as a means -to have different import categories. - -The modules in this setting are imported after the modules in -:setting:`CELERY_IMPORTS`. - -.. setting:: CELERYD_WORKER_LOST_WAIT - -CELERYD_WORKER_LOST_WAIT -~~~~~~~~~~~~~~~~~~~~~~~~ - In some cases a worker may be killed without proper cleanup, and the worker may have published a result before terminating. This value specifies how long we wait for any missing results before @@ -1657,18 +1596,18 @@ raising a :exc:`@WorkerLostError` exception. Default is 10.0 -.. setting:: CELERYD_MAX_TASKS_PER_CHILD +.. setting:: worker_max_tasks_per_child -CELERYD_MAX_TASKS_PER_CHILD +worker_max_tasks_per_child ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Maximum number of tasks a pool worker process can execute before it's replaced with a new one. Default is no limit. -.. setting:: CELERYD_MAX_MEMORY_PER_CHILD +.. setting:: worker_max_memory_per_child -CELERYD_MAX_MEMORY_PER_CHILD -~~~~~~~~~~~~~~~~~~~~~ +worker_max_memory_per_child +~~~~~~~~~~~~~~~~~~~~~~~~~~~ Maximum amount of resident memory that may be consumed by a worker before it will be replaced by a new worker. If a single @@ -1676,50 +1615,17 @@ task causes a worker to exceed this limit, the task will be completed, and the worker will be replaced afterwards. Default: no limit. -.. setting:: CELERYD_TASK_TIME_LIMIT - -CELERYD_TASK_TIME_LIMIT -~~~~~~~~~~~~~~~~~~~~~~~ - -Task hard time limit in seconds. The worker processing the task will -be killed and replaced with a new one when this is exceeded. - -.. setting:: CELERYD_TASK_SOFT_TIME_LIMIT - -CELERYD_TASK_SOFT_TIME_LIMIT -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Task soft time limit in seconds. - -The :exc:`~@SoftTimeLimitExceeded` exception will be -raised when this is exceeded. The task can catch this to -e.g. clean up before the hard time limit comes. - -Example: - -.. code-block:: python - - from celery.exceptions import SoftTimeLimitExceeded - - @app.task - def mytask(): - try: - return do_work() - except SoftTimeLimitExceeded: - cleanup_in_a_hurry() - -.. setting:: CELERY_STORE_ERRORS_EVEN_IF_IGNORED +.. setting:: worker_disable_rate_limits -CELERY_STORE_ERRORS_EVEN_IF_IGNORED -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +worker_disable_rate_limits +~~~~~~~~~~~~~~~~~~~~~~~~~~ -If set, the worker stores all task errors in the result store even if -:attr:`Task.ignore_result ` is on. +Disable all rate limits, even if tasks has explicit rate limits set. -.. setting:: CELERYD_STATE_DB +.. setting:: worker_state_db -CELERYD_STATE_DB -~~~~~~~~~~~~~~~~ +worker_state_db +~~~~~~~~~~~~~~~ Name of the file used to stores persistent worker state (like revoked tasks). Can be a relative or absolute path, but be aware that the suffix `.db` @@ -1730,10 +1636,10 @@ Can also be set via the :option:`--statedb` argument to Not enabled by default. -.. setting:: CELERYD_TIMER_PRECISION +.. setting:: worker_timer_precision -CELERYD_TIMER_PRECISION -~~~~~~~~~~~~~~~~~~~~~~~ +worker_timer_precision +~~~~~~~~~~~~~~~~~~~~~~ Set the maximum time in seconds that the ETA scheduler can sleep between rechecking the schedule. Default is 1 second. @@ -1741,94 +1647,92 @@ rechecking the schedule. Default is 1 second. Setting this value to 1 second means the schedulers precision will be 1 second. If you need near millisecond precision you can set this to 0.1. -.. setting:: CELERY_ENABLE_REMOTE_CONTROL +.. setting:: worker_enable_remote_control -CELERY_ENABLE_REMOTE_CONTROL +worker_enable_remote_control ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Specify if remote control of the workers is enabled. Default is :const:`True`. - .. _conf-error-mails: Error E-Mails ------------- -.. setting:: CELERY_SEND_TASK_ERROR_EMAILS +.. setting:: task_send_error_emails -CELERY_SEND_TASK_ERROR_EMAILS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +task_send_error_emails +~~~~~~~~~~~~~~~~~~~~~~ The default value for the `Task.send_error_emails` attribute, which if set to :const:`True` means errors occurring during task execution will be -sent to :setting:`ADMINS` by email. +sent to :setting:`admins` by email. Disabled by default. -.. setting:: ADMINS +.. setting:: admins -ADMINS +admins ~~~~~~ List of `(name, email_address)` tuples for the administrators that should receive error emails. -.. setting:: SERVER_EMAIL +.. setting:: server_email -SERVER_EMAIL +server_email ~~~~~~~~~~~~ The email address this worker sends emails from. Default is celery@localhost. -.. setting:: EMAIL_HOST +.. setting:: email_host -EMAIL_HOST +email_host ~~~~~~~~~~ The mail server to use. Default is ``localhost``. -.. setting:: EMAIL_HOST_USER +.. setting:: email_host_user -EMAIL_HOST_USER +email_host_user ~~~~~~~~~~~~~~~ User name (if required) to log on to the mail server with. -.. setting:: EMAIL_HOST_PASSWORD +.. setting:: email_host_password -EMAIL_HOST_PASSWORD +email_host_password ~~~~~~~~~~~~~~~~~~~ Password (if required) to log on to the mail server with. -.. setting:: EMAIL_PORT +.. setting:: email_port -EMAIL_PORT +email_port ~~~~~~~~~~ The port the mail server is listening on. Default is `25`. +.. setting:: email_use_ssl -.. setting:: EMAIL_USE_SSL - -EMAIL_USE_SSL +email_use_ssl ~~~~~~~~~~~~~ Use SSL when connecting to the SMTP server. Disabled by default. -.. setting:: EMAIL_USE_TLS +.. setting:: email_use_tls -EMAIL_USE_TLS +email_use_tls ~~~~~~~~~~~~~ Use TLS when connecting to the SMTP server. Disabled by default. -.. setting:: EMAIL_TIMEOUT +.. setting:: email_timeout -EMAIL_TIMEOUT +email_timeout ~~~~~~~~~~~~~ Timeout in seconds for when we give up trying to connect @@ -1836,15 +1740,14 @@ to the SMTP server when sending emails. The default is 2 seconds. -EMAIL_CHARSET +.. setting:: email_charset + +email_charset ~~~~~~~~~~~~~ .. versionadded:: 4.0 Charset for outgoing emails. Default is "us-ascii". -.. setting:: EMAIL_CHARSET - - .. _conf-example-error-mail-config: Example E-Mail configuration @@ -1856,40 +1759,40 @@ george@vandelay.com and kramer@vandelay.com: .. code-block:: python # Enables error emails. - CELERY_SEND_TASK_ERROR_EMAILS = True + task_send_error_emails = True # Name and email addresses of recipients - ADMINS = ( + admins = ( ('George Costanza', 'george@vandelay.com'), ('Cosmo Kramer', 'kosmo@vandelay.com'), ) # Email address used as sender (From field). - SERVER_EMAIL = 'no-reply@vandelay.com' + server_email = 'no-reply@vandelay.com' # Mailserver configuration - EMAIL_HOST = 'mail.vandelay.com' - EMAIL_PORT = 25 - # EMAIL_HOST_USER = 'servers' - # EMAIL_HOST_PASSWORD = 's3cr3t' + email_host = 'mail.vandelay.com' + email_port = 25 + # email_host_user = 'servers' + # email_host_password = 's3cr3t' .. _conf-events: Events ------ -.. setting:: CELERY_SEND_EVENTS +.. setting:: worker_send_events -CELERY_SEND_EVENTS +worker_send_events ~~~~~~~~~~~~~~~~~~ Send task-related events so that tasks can be monitored using tools like `flower`. Sets the default value for the workers :option:`-E` argument. -.. setting:: CELERY_SEND_TASK_SENT_EVENT +.. setting:: task_send_sent_event -CELERY_SEND_TASK_SENT_EVENT -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +task_send_sent_event +~~~~~~~~~~~~~~~~~~~~ .. versionadded:: 2.2 @@ -1898,9 +1801,9 @@ tracked before they are consumed by a worker. Disabled by default. -.. setting:: CELERY_EVENT_QUEUE_TTL +.. setting:: event_queue_ttl -CELERY_EVENT_QUEUE_TTL +event_queue_ttl ~~~~~~~~~~~~~~~~~~~~~~ :transports supported: ``amqp`` @@ -1912,83 +1815,50 @@ will be deleted after 10 seconds. Disabled by default. -.. setting:: CELERY_EVENT_QUEUE_EXPIRES +.. setting:: event_queue_expires -CELERY_EVENT_QUEUE_EXPIRES -~~~~~~~~~~~~~~~~~~~~~~~~~~ +event_queue_expires +~~~~~~~~~~~~~~~~~~~ :transports supported: ``amqp`` - Expiry time in seconds (int/float) for when after a monitor clients event queue will be deleted (``x-expires``). Default is never, relying on the queue autodelete setting. -.. setting:: CELERY_EVENT_SERIALIZER +.. setting:: event_serializer -CELERY_EVENT_SERIALIZER -~~~~~~~~~~~~~~~~~~~~~~~ +event_serializer +~~~~~~~~~~~~~~~~ Message serialization format used when sending event messages. Default is ``json``. See :ref:`calling-serializers`. -.. _conf-broadcast: - -Broadcast Commands ------------------- - -.. setting:: CELERY_BROADCAST_QUEUE - -CELERY_BROADCAST_QUEUE -~~~~~~~~~~~~~~~~~~~~~~ - -Name prefix for the queue used when listening for broadcast messages. -The workers host name will be appended to the prefix to create the final -queue name. - -Default is ``celeryctl``. - -.. setting:: CELERY_BROADCAST_EXCHANGE - -CELERY_BROADCAST_EXCHANGE -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Name of the exchange used for broadcast messages. - -Default is ``celeryctl``. - -.. setting:: CELERY_BROADCAST_EXCHANGE_TYPE - -CELERY_BROADCAST_EXCHANGE_TYPE -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Exchange type used for broadcast messages. Default is ``fanout``. - .. _conf-logging: Logging ------- -.. setting:: CELERYD_HIJACK_ROOT_LOGGER +.. setting:: worker_hijack_root_logger -CELERYD_HIJACK_ROOT_LOGGER -~~~~~~~~~~~~~~~~~~~~~~~~~~ +worker_hijack_root_logger +~~~~~~~~~~~~~~~~~~~~~~~~~ .. versionadded:: 2.2 By default any previously configured handlers on the root logger will be removed. If you want to customize your own logging handlers, then you can disable this behavior by setting -`CELERYD_HIJACK_ROOT_LOGGER = False`. +`worker_hijack_root_logger = False`. .. note:: Logging can also be customized by connecting to the :signal:`celery.signals.setup_logging` signal. -.. setting:: CELERYD_LOG_COLOR +.. setting:: worker_log_color -CELERYD_LOG_COLOR +worker_log_color ~~~~~~~~~~~~~~~~~ Enables/disables colors in logging output by the Celery apps. @@ -1998,10 +1868,10 @@ By default colors are enabled if 1) the app is logging to a real terminal, and not a file. 2) the app is not running on Windows. -.. setting:: CELERYD_LOG_FORMAT +.. setting:: worker_log_format -CELERYD_LOG_FORMAT -~~~~~~~~~~~~~~~~~~ +worker_log_format +~~~~~~~~~~~~~~~~~ The format to use for log messages. @@ -2010,10 +1880,10 @@ Default is `[%(asctime)s: %(levelname)s/%(processName)s] %(message)s` See the Python :mod:`logging` module for more information about log formats. -.. setting:: CELERYD_TASK_LOG_FORMAT +.. setting:: worker_task_log_format -CELERYD_TASK_LOG_FORMAT -~~~~~~~~~~~~~~~~~~~~~~~ +worker_task_log_format +~~~~~~~~~~~~~~~~~~~~~~ The format to use for log messages logged in tasks. Can be overridden using the :option:`--loglevel` option to :mod:`~celery.bin.worker`. @@ -2026,9 +1896,9 @@ Default is:: See the Python :mod:`logging` module for more information about log formats. -.. setting:: CELERY_REDIRECT_STDOUTS +.. setting:: worker_redirect_stdouts -CELERY_REDIRECT_STDOUTS +worker_redirect_stdouts ~~~~~~~~~~~~~~~~~~~~~~~ If enabled `stdout` and `stderr` will be redirected @@ -2037,9 +1907,9 @@ to the current logger. Enabled by default. Used by :program:`celery worker` and :program:`celery beat`. -.. setting:: CELERY_REDIRECT_STDOUTS_LEVEL +.. setting:: worker_redirect_stdouts_level -CELERY_REDIRECT_STDOUTS_LEVEL +worker_redirect_stdouts_level ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The log level output to `stdout` and `stderr` is logged as. @@ -2053,30 +1923,30 @@ Default is :const:`WARNING`. Security -------- -.. setting:: CELERY_SECURITY_KEY +.. setting:: security_key -CELERY_SECURITY_KEY -~~~~~~~~~~~~~~~~~~~ +security_key +~~~~~~~~~~~~ .. versionadded:: 2.5 The relative or absolute path to a file containing the private key used to sign messages when :ref:`message-signing` is used. -.. setting:: CELERY_SECURITY_CERTIFICATE +.. setting:: security_certificate -CELERY_SECURITY_CERTIFICATE -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +security_certificate +~~~~~~~~~~~~~~~~~~~~ .. versionadded:: 2.5 The relative or absolute path to an X.509 certificate file used to sign messages when :ref:`message-signing` is used. -.. setting:: CELERY_SECURITY_CERT_STORE +.. setting:: security_cert_store -CELERY_SECURITY_CERT_STORE -~~~~~~~~~~~~~~~~~~~~~~~~~~ +security_cert_store +~~~~~~~~~~~~~~~~~~~ .. versionadded:: 2.5 @@ -2089,10 +1959,10 @@ The directory containing X.509 certificates used for Custom Component Classes (advanced) ----------------------------------- -.. setting:: CELERYD_POOL +.. setting:: worker_pool -CELERYD_POOL -~~~~~~~~~~~~ +worker_pool +~~~~~~~~~~~ Name of the pool class used by the worker. @@ -2104,20 +1974,20 @@ Name of the pool class used by the worker. Default is ``celery.concurrency.prefork:TaskPool``. -.. setting:: CELERYD_POOL_RESTARTS +.. setting:: worker_pool_restarts -CELERYD_POOL_RESTARTS -~~~~~~~~~~~~~~~~~~~~~ +worker_pool_restarts +~~~~~~~~~~~~~~~~~~~~ If enabled the worker pool can be restarted using the :control:`pool_restart` remote control command. Disabled by default. -.. setting:: CELERYD_AUTOSCALER +.. setting:: worker_autoscaler -CELERYD_AUTOSCALER -~~~~~~~~~~~~~~~~~~ +worker_autoscaler +~~~~~~~~~~~~~~~~~ .. versionadded:: 2.2 @@ -2125,60 +1995,60 @@ Name of the autoscaler class to use. Default is ``celery.worker.autoscale:Autoscaler``. -.. setting:: CELERYD_AUTORELOADER +.. setting:: worker_autoreloader -CELERYD_AUTORELOADER -~~~~~~~~~~~~~~~~~~~~ +worker_autoreloader +~~~~~~~~~~~~~~~~~~~ Name of the autoreloader class used by the worker to reload Python modules and files that have changed. Default is: ``celery.worker.autoreload:Autoreloader``. -.. setting:: CELERYD_CONSUMER +.. setting:: worker_consumer -CELERYD_CONSUMER -~~~~~~~~~~~~~~~~ +worker_consumer +~~~~~~~~~~~~~~~ Name of the consumer class used by the worker. Default is :class:`celery.worker.consumer.Consumer` -.. setting:: CELERYD_TIMER +.. setting:: worker_timer -CELERYD_TIMER -~~~~~~~~~~~~~~~~~~~~~ +worker_timer +~~~~~~~~~~~~ Name of the ETA scheduler class used by the worker. -Default is :class:`celery.utils.timer2.Timer`, or one overrided +Default is :class:`kombu.async.hub.timer.Timer`, or one overrided by the pool implementation. .. _conf-celerybeat: -Periodic Task Server: celery beat ---------------------------------- +Beat Settings (:program:`celery beat`) +-------------------------------------- -.. setting:: CELERYBEAT_SCHEDULE +.. setting:: beat_schedule -CELERYBEAT_SCHEDULE -~~~~~~~~~~~~~~~~~~~ +beat_schedule +~~~~~~~~~~~~~ The periodic task schedule used by :mod:`~celery.bin.beat`. See :ref:`beat-entries`. -.. setting:: CELERYBEAT_SCHEDULER +.. setting:: beat_scheduler -CELERYBEAT_SCHEDULER -~~~~~~~~~~~~~~~~~~~~ +beat_scheduler +~~~~~~~~~~~~~~ The default scheduler class. Default is ``celery.beat:PersistentScheduler``. Can also be set via the :option:`-S` argument to :mod:`~celery.bin.beat`. -.. setting:: CELERYBEAT_SCHEDULE_FILENAME +.. setting:: beat_schedule_filename -CELERYBEAT_SCHEDULE_FILENAME -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +beat_schedule_filename +~~~~~~~~~~~~~~~~~~~~~~ Name of the file used by `PersistentScheduler` to store the last run times of periodic tasks. Can be a relative or absolute path, but be aware that the @@ -2187,10 +2057,10 @@ suffix `.db` may be appended to the file name (depending on Python version). Can also be set via the :option:`--schedule` argument to :mod:`~celery.bin.beat`. -.. setting:: CELERYBEAT_SYNC_EVERY +.. setting:: beat_sync_every -CELERYBEAT_SYNC_EVERY -~~~~~~~~~~~~~~~~~~~~~ +beat_sync_every +~~~~~~~~~~~~~~~ The number of periodic tasks that can be called before another database sync is issued. @@ -2198,10 +2068,10 @@ Defaults to 0 (sync based on timing - default of 3 minutes as determined by scheduler.sync_every). If set to 1, beat will call sync after every task message sent. -.. setting:: CELERYBEAT_MAX_LOOP_INTERVAL +.. setting:: beat_max_loop_interval -CELERYBEAT_MAX_LOOP_INTERVAL -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +beat_max_loop_interval +~~~~~~~~~~~~~~~~~~~~~~ The maximum number of seconds :mod:`~celery.bin.beat` can sleep between checking the schedule. diff --git a/docs/django/first-steps-with-django.rst b/docs/django/first-steps-with-django.rst index 4fb551487..70786d81c 100644 --- a/docs/django/first-steps-with-django.rst +++ b/docs/django/first-steps-with-django.rst @@ -76,7 +76,13 @@ but there's probably no reason for that when using Django. We also add the Django settings module as a configuration source for Celery. This means that you don't have to use multiple configuration files, and instead configure Celery directly -from the Django settings. +from the Django settings; but you can also separate them if wanted. + +The uppercase namespace means that all Celery configuration options +must be specified in uppercase instead of lowercase, and start with +``CELERY_``, so e.g. the :setting:`task_always_eager`` setting +becomes ``CELERY_TASK_ALWAYS_EAGER``, and the :setting:`broker_url` +setting becomes ``CELERY_BROKER_URL``. You can pass the object directly here, but using a string is better since then the worker doesn't have to serialize the object when using Windows @@ -84,7 +90,7 @@ or execv: .. code-block:: python - app.config_from_object('django.conf:settings') + app.config_from_object('django.conf:settings', namespace='CELERY_') Next, a common practice for reusable apps is to define all tasks in a separate ``tasks.py`` module, and Celery does have a way to @@ -106,7 +112,7 @@ of your installed apps, following the ``tasks.py`` convention:: This way you do not have to manually add the individual modules -to the :setting:`CELERY_IMPORTS` setting. The ``lambda`` so that the +to the :setting:`CELERY_IMPORTS ` setting. The ``lambda`` so that the autodiscovery can happen only when needed, and so that importing your module will not evaluate the Django settings object. @@ -176,7 +182,7 @@ To use this with your project you need to follow these four steps: .. code-block:: python app.conf.update( - CELERY_RESULT_BACKEND='djcelery.backends.database:DatabaseBackend', + result_backend='djcelery.backends.database:DatabaseBackend', ) For the cache backend you can use: @@ -184,7 +190,7 @@ To use this with your project you need to follow these four steps: .. code-block:: python app.conf.update( - CELERY_RESULT_BACKEND='djcelery.backends.cache:CacheBackend', + result_backend='djcelery.backends.cache:CacheBackend', ) If you have connected Celery to your Django settings then you can diff --git a/docs/faq.rst b/docs/faq.rst index 7efb678d5..c2ae478d5 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -449,10 +449,10 @@ data. Note that this is not just something you should be aware of with Celery, for example also Django uses pickle for its cache client. -For the task messages you can set the :setting:`CELERY_TASK_SERIALIZER` +For the task messages you can set the :setting:`task_serializer` setting to "json" or "yaml" instead of pickle. -Similarly for task results you can set :setting:`CELERY_RESULT_SERIALIZER`. +Similarly for task results you can set :setting:`result_serializer`. For more details of the formats used and the lookup order when checking which format to use for a task see :ref:`calling-serializers` @@ -461,7 +461,7 @@ Can messages be encrypted? -------------------------- **Answer**: Some AMQP brokers supports using SSL (including RabbitMQ). -You can enable this using the :setting:`BROKER_USE_SSL` setting. +You can enable this using the :setting:`broker_use_ssl` setting. It is also possible to add additional encryption and security to messages, if you have a need for this then you should contact the :ref:`mailing-list`. @@ -517,7 +517,7 @@ as a message. If you don't collect these results, they will build up and RabbitMQ will eventually run out of memory. Results expire after 1 day by default. It may be a good idea -to lower this value by configuring the :setting:`CELERY_TASK_RESULT_EXPIRES` +to lower this value by configuring the :setting:`result_expires` setting. If you don't use the results for a task, make sure you set the @@ -565,7 +565,7 @@ Tasks How can I reuse the same connection when calling tasks? ------------------------------------------------------- -**Answer**: See the :setting:`BROKER_POOL_LIMIT` setting. +**Answer**: See the :setting:`broker_pool_limit` setting. The connection pool is enabled by default since version 2.5. .. _faq-sudo-subprocess: diff --git a/docs/getting-started/brokers/beanstalk.rst b/docs/getting-started/brokers/beanstalk.rst index 4f0ed7df5..c31c630b2 100644 --- a/docs/getting-started/brokers/beanstalk.rst +++ b/docs/getting-started/brokers/beanstalk.rst @@ -34,7 +34,7 @@ Configuration Configuration is easy, set the transport, and configure the location of your Beanstalk database:: - BROKER_URL = 'beanstalk://localhost:11300' + broker_url = 'beanstalk://localhost:11300' Where the URL is in the format of:: diff --git a/docs/getting-started/brokers/couchdb.rst b/docs/getting-started/brokers/couchdb.rst index 8708fbcf7..b7dba3e05 100644 --- a/docs/getting-started/brokers/couchdb.rst +++ b/docs/getting-started/brokers/couchdb.rst @@ -32,7 +32,7 @@ Configuration Configuration is easy, set the transport, and configure the location of your CouchDB database:: - BROKER_URL = 'couchdb://localhost:5984/database_name' + broker_url = 'couchdb://localhost:5984/database_name' Where the URL is in the format of:: diff --git a/docs/getting-started/brokers/django.rst b/docs/getting-started/brokers/django.rst index f6c0d6b2b..df4669ea1 100644 --- a/docs/getting-started/brokers/django.rst +++ b/docs/getting-started/brokers/django.rst @@ -26,7 +26,7 @@ configuration values. #. Set your broker transport:: - BROKER_URL = 'django://' + CELERY_BROKER_URL = 'django://' #. Add :mod:`kombu.transport.django` to `INSTALLED_APPS`:: diff --git a/docs/getting-started/brokers/ironmq.rst b/docs/getting-started/brokers/ironmq.rst index 7fa8e2f31..4816bebba 100644 --- a/docs/getting-started/brokers/ironmq.rst +++ b/docs/getting-started/brokers/ironmq.rst @@ -31,7 +31,7 @@ First, you'll need to import the iron_celery library right after you import Cele You have to specify IronMQ in the broker URL:: - BROKER_URL = 'ironmq://ABCDEFGHIJKLMNOPQRST:ZYXK7NiynGlTogH8Nj+P9nlE73sq3@' + broker_url = 'ironmq://ABCDEFGHIJKLMNOPQRST:ZYXK7NiynGlTogH8Nj+P9nlE73sq3@' where the URL format is:: diff --git a/docs/getting-started/brokers/mongodb.rst b/docs/getting-started/brokers/mongodb.rst index 96c396c94..cd4d478b7 100644 --- a/docs/getting-started/brokers/mongodb.rst +++ b/docs/getting-started/brokers/mongodb.rst @@ -32,7 +32,7 @@ Configuration Configuration is easy, set the transport, and configure the location of your MongoDB database:: - BROKER_URL = 'mongodb://localhost:27017/database_name' + broker_url = 'mongodb://localhost:27017/database_name' Where the URL is in the format of:: diff --git a/docs/getting-started/brokers/rabbitmq.rst b/docs/getting-started/brokers/rabbitmq.rst index f5c077493..cf2902885 100644 --- a/docs/getting-started/brokers/rabbitmq.rst +++ b/docs/getting-started/brokers/rabbitmq.rst @@ -16,7 +16,7 @@ the broker instance you want to use: .. code-block:: python - BROKER_URL = 'amqp://guest:guest@localhost:5672//' + broker_url = 'amqp://guest:guest@localhost:5672//' For a description of broker URLs and a full list of the various broker configuration options available to Celery, diff --git a/docs/getting-started/brokers/redis.rst b/docs/getting-started/brokers/redis.rst index 21726b6d1..ac6ef7c85 100644 --- a/docs/getting-started/brokers/redis.rst +++ b/docs/getting-started/brokers/redis.rst @@ -25,7 +25,7 @@ Configuration Configuration is easy, just configure the location of your Redis database:: - BROKER_URL = 'redis://localhost:6379/0' + broker_url = 'redis://localhost:6379/0' Where the URL is in the format of:: @@ -47,9 +47,9 @@ The visibility timeout defines the number of seconds to wait for the worker to acknowledge the task before the message is redelivered to another worker. Be sure to see :ref:`redis-caveats` below. -This option is set via the :setting:`BROKER_TRANSPORT_OPTIONS` setting:: +This option is set via the :setting:`broker_transport_options` setting:: - BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 3600} # 1 hour. + broker_transport_options = {'visibility_timeout': 3600} # 1 hour. The default visibility timeout for Redis is 1 hour. @@ -61,7 +61,7 @@ Results If you also want to store the state and return values of tasks in Redis, you should configure these settings:: - CELERY_RESULT_BACKEND = 'redis://localhost:6379/0' + result_backend = 'redis://localhost:6379/0' For a complete list of options supported by the Redis result backend, see :ref:`conf-redis-result-backend` @@ -81,7 +81,7 @@ Broadcast messages will be seen by all virtual hosts by default. You have to set a transport option to prefix the messages so that they will only be received by the active virtual host:: - BROKER_TRANSPORT_OPTIONS = {'fanout_prefix': True} + broker_transport_options = {'fanout_prefix': True} Note that you will not be able to communicate with workers running older versions or workers that does not have this setting enabled. @@ -99,7 +99,7 @@ Workers will receive all task related events by default. To avoid this you must set the ``fanout_patterns`` fanout option so that the workers may only subscribe to worker related events:: - BROKER_TRANSPORT_OPTIONS = {'fanout_patterns': True} + broker_transport_options = {'fanout_patterns': True} Note that this change is backward incompatible so all workers in the cluster must have this option enabled, or else they will not be able to @@ -131,7 +131,7 @@ as this is a concept separate from ETA/countdown. You can increase this timeout by configuring a transport option with the same name:: - BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 43200} + broker_transport_options = {'visibility_timeout': 43200} The value must be an int describing the number of seconds. diff --git a/docs/getting-started/brokers/sqlalchemy.rst b/docs/getting-started/brokers/sqlalchemy.rst index 47f9b96d0..37f8d7f57 100644 --- a/docs/getting-started/brokers/sqlalchemy.rst +++ b/docs/getting-started/brokers/sqlalchemy.rst @@ -24,9 +24,9 @@ Configuration Celery needs to know the location of your database, which should be the usual SQLAlchemy connection string, but with 'sqla+' prepended to it:: - BROKER_URL = 'sqla+sqlite:///celerydb.sqlite' + broker_url = 'sqla+sqlite:///celerydb.sqlite' -This transport uses only the :setting:`BROKER_URL` setting, which have to be +This transport uses only the :setting:`broker_url` setting, which have to be an SQLAlchemy database URI. @@ -37,16 +37,16 @@ Here's a list of examples using a selection of other `SQLAlchemy Connection Stri .. code-block:: python # sqlite (filename) - BROKER_URL = 'sqla+sqlite:///celerydb.sqlite' + broker_url = 'sqla+sqlite:///celerydb.sqlite' # mysql - BROKER_URL = 'sqla+mysql://scott:tiger@localhost/foo' + broker_url = 'sqla+mysql://scott:tiger@localhost/foo' # postgresql - BROKER_URL = 'sqla+postgresql://scott:tiger@localhost/mydatabase' + broker_url = 'sqla+postgresql://scott:tiger@localhost/mydatabase' # oracle - BROKER_URL = 'sqla+oracle://scott:tiger@127.0.0.1:1521/sidname' + broker_url = 'sqla+oracle://scott:tiger@127.0.0.1:1521/sidname' .. _`SQLAlchemy: Supported Databases`: http://www.sqlalchemy.org/docs/core/engines.html#supported-databases diff --git a/docs/getting-started/brokers/sqs.rst b/docs/getting-started/brokers/sqs.rst index b9ec699cf..cc44b280f 100644 --- a/docs/getting-started/brokers/sqs.rst +++ b/docs/getting-started/brokers/sqs.rst @@ -32,7 +32,7 @@ Configuration You have to specify SQS in the broker URL:: - BROKER_URL = 'sqs://ABCDEFGHIJKLMNOPQRST:ZYXK7NiynGlTogH8Nj+P9nlE73sq3@' + broker_url = 'sqs://ABCDEFGHIJKLMNOPQRST:ZYXK7NiynGlTogH8Nj+P9nlE73sq3@' where the URL format is:: @@ -57,9 +57,9 @@ Region ------ The default region is ``us-east-1`` but you can select another region -by configuring the :setting:`BROKER_TRANSPORT_OPTIONS` setting:: +by configuring the :setting:`broker_transport_options` setting:: - BROKER_TRANSPORT_OPTIONS = {'region': 'eu-west-1'} + broker_transport_options = {'region': 'eu-west-1'} .. seealso:: @@ -74,9 +74,9 @@ The visibility timeout defines the number of seconds to wait for the worker to acknowledge the task before the message is redelivered to another worker. Also see caveats below. -This option is set via the :setting:`BROKER_TRANSPORT_OPTIONS` setting:: +This option is set via the :setting:`broker_transport_options` setting:: - BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 3600} # 1 hour. + broker_transport_options = {'visibility_timeout': 3600} # 1 hour. The default visibility timeout is 30 seconds. @@ -91,10 +91,10 @@ sleep for one second whenever there are no more messages to read. You should note that **more frequent polling is also more expensive, so increasing the polling interval can save you money**. -The polling interval can be set via the :setting:`BROKER_TRANSPORT_OPTIONS` +The polling interval can be set via the :setting:`broker_transport_options` setting:: - BROKER_TRANSPORT_OPTIONS = {'polling_interval': 0.3} + broker_transport_options = {'polling_interval': 0.3} Very frequent polling intervals can cause *busy loops*, which results in the worker using a lot of CPU time. If you need sub-millisecond precision you @@ -106,9 +106,9 @@ Queue Prefix By default Celery will not assign any prefix to the queue names, If you have other services using SQS you can configure it do so -using the :setting:`BROKER_TRANSPORT_OPTIONS` setting:: +using the :setting:`broker_transport_options` setting:: - BROKER_TRANSPORT_OPTIONS = {'queue_name_prefix': 'celery-'} + broker_transport_options = {'queue_name_prefix': 'celery-'} .. _sqs-caveats: @@ -137,7 +137,7 @@ Caveats The maximum visibility timeout supported by AWS as of this writing is 12 hours (43200 seconds):: - BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 43200} + broker_transport_options = {'visibility_timeout': 43200} - SQS does not yet support worker remote control commands. diff --git a/docs/getting-started/first-steps-with-celery.rst b/docs/getting-started/first-steps-with-celery.rst index 0231137de..661b8bf0c 100644 --- a/docs/getting-started/first-steps-with-celery.rst +++ b/docs/getting-started/first-steps-with-celery.rst @@ -225,7 +225,7 @@ built-in result backends to choose from: `SQLAlchemy`_/`Django`_ ORM, For this example you will use the `rpc` result backend, which sends states back as transient messages. The backend is specified via the ``backend`` argument to -:class:`@Celery`, (or via the :setting:`CELERY_RESULT_BACKEND` setting if +:class:`@Celery`, (or via the :setting:`task_result_backend` setting if you choose to use a configuration module):: app = Celery('tasks', backend='rpc://', broker='amqp://') @@ -289,22 +289,22 @@ can be configured. You can read about the options in the The configuration can be set on the app directly or by using a dedicated configuration module. As an example you can configure the default serializer used for serializing -task payloads by changing the :setting:`CELERY_TASK_SERIALIZER` setting: +task payloads by changing the :setting:`task_serializer` setting: .. code-block:: python - app.conf.CELERY_TASK_SERIALIZER = 'json' + app.conf.task_serializer = 'json' If you are configuring many settings at once you can use ``update``: .. code-block:: python app.conf.update( - CELERY_TASK_SERIALIZER='json', - CELERY_ACCEPT_CONTENT=['json'], # Ignore other content - CELERY_RESULT_SERIALIZER='json', - CELERY_TIMEZONE='Europe/Oslo', - CELERY_ENABLE_UTC=True, + task_serializer='json', + accept_content=['json'], # Ignore other content + result_serializer='json', + timezone='Europe/Oslo', + enable_utc=True, ) For larger projects using a dedicated configuration module is useful, @@ -332,14 +332,14 @@ current directory or on the Python path, it could look like this: .. code-block:: python - BROKER_URL = 'amqp://' - CELERY_RESULT_BACKEND = 'rpc://' + broker_url = 'amqp://' + result_backend = 'rpc://' - CELERY_TASK_SERIALIZER = 'json' - CELERY_RESULT_SERIALIZER = 'json' - CELERY_ACCEPT_CONTENT=['json'] - CELERY_TIMEZONE = 'Europe/Oslo' - CELERY_ENABLE_UTC = True + task_serializer = 'json' + result_serializer = 'json' + accept_content = ['json'] + timezone = 'Europe/Oslo' + enable_utc = True To verify that your configuration file works properly, and doesn't contain any syntax errors, you can try to import it: @@ -357,7 +357,7 @@ route a misbehaving task to a dedicated queue: .. code-block:: python - CELERY_ROUTES = { + task_routes = { 'tasks.add': 'low-priority', } @@ -369,7 +369,7 @@ instead, so that only 10 tasks of this type can be processed in a minute .. code-block:: python - CELERY_ANNOTATIONS = { + task_annotations = { 'tasks.add': {'rate_limit': '10/m'} } @@ -384,7 +384,7 @@ for the task at runtime: new rate limit set successfully See :ref:`guide-routing` to read more about task routing, -and the :setting:`CELERY_ANNOTATIONS` setting for more about annotations, +and the :setting:`task_annotations` setting for more about annotations, or :ref:`guide-monitoring` for more about remote control commands, and how to monitor what your workers are doing. @@ -435,7 +435,7 @@ the task id after all). Enabling this option will force the worker to skip updating states. -2) Make sure the :setting:`CELERY_IGNORE_RESULT` setting is not enabled. +2) Make sure the :setting:`task_ignore_result` setting is not enabled. 3) Make sure that you do not have any old workers still running. diff --git a/docs/getting-started/next-steps.rst b/docs/getting-started/next-steps.rst index 981b096a5..29cc8ed84 100644 --- a/docs/getting-started/next-steps.rst +++ b/docs/getting-started/next-steps.rst @@ -369,7 +369,7 @@ states. The stages of a typical task can be:: PENDING -> STARTED -> SUCCESS The started state is a special state that is only recorded if the -:setting:`CELERY_TRACK_STARTED` setting is enabled, or if the +:setting:`task_track_started` setting is enabled, or if the ``@task(track_started=True)`` option is set for the task. The pending state is actually not a recorded state, but rather @@ -605,13 +605,13 @@ Routing Celery supports all of the routing facilities provided by AMQP, but it also supports simple routing where messages are sent to named queues. -The :setting:`CELERY_ROUTES` setting enables you to route tasks by name +The :setting:`task_routes` setting enables you to route tasks by name and keep everything centralized in one location: .. code-block:: python app.conf.update( - CELERY_ROUTES = { + task_routes = { 'proj.tasks.add': {'queue': 'hipri'}, }, ) @@ -732,11 +732,11 @@ All times and dates, internally and in messages uses the UTC timezone. When the worker receives a message, for example with a countdown set it converts that UTC time to local time. If you wish to use a different timezone than the system timezone then you must -configure that using the :setting:`CELERY_TIMEZONE` setting: +configure that using the :setting:`timezone` setting: .. code-block:: python - app.conf.CELERY_TIMEZONE = 'Europe/London' + app.conf.timezone = 'Europe/London' Optimization ============ diff --git a/docs/glossary.rst b/docs/glossary.rst index d3158c543..32ad2395e 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -74,7 +74,7 @@ Glossary prefetch multiplier The :term:`prefetch count` is configured by using the - :setting:`CELERYD_PREFETCH_MULTIPLIER` setting, which is multiplied + :setting:`worker_prefetch_multiplier` setting, which is multiplied by the number of pool slots (threads/processes/greenthreads). prefetch count diff --git a/docs/internals/app-overview.rst b/docs/internals/app-overview.rst index 0213ac91a..4dd82a791 100644 --- a/docs/internals/app-overview.rst +++ b/docs/internals/app-overview.rst @@ -57,8 +57,8 @@ Getting access to the configuration: .. code-block:: python - celery.conf.CELERY_ALWAYS_EAGER = True - celery.conf["CELERY_ALWAYS_EAGER"] = True + celery.conf.task_always_eager = True + celery.conf["task_always_eager"] = True Controlling workers:: @@ -135,15 +135,15 @@ Aliases (Pending deprecation) * celery.conf.* -> {app.conf} **NOTE**: All configuration keys are now named the same - as in the configuration. So the key "CELERY_ALWAYS_EAGER" + as in the configuration. So the key "task_always_eager" is accessed as:: - >>> app.conf.CELERY_ALWAYS_EAGER + >>> app.conf.task_always_eager instead of:: >>> from celery import conf - >>> conf.ALWAYS_EAGER + >>> conf.always_eager * .get_queues -> {app.amqp.get_queues} diff --git a/docs/internals/deprecation.rst b/docs/internals/deprecation.rst index 746e7ae24..0f16b29ac 100644 --- a/docs/internals/deprecation.rst +++ b/docs/internals/deprecation.rst @@ -33,7 +33,7 @@ Removals for version 4.0 .. _deprecations-v4.0: -Removals for version 4.0 +Removals for version 5.0 ======================== Old Task API @@ -145,7 +145,7 @@ The task attributes: - ``delivery_mode`` - ``priority`` -is deprecated and must be set by :setting:`CELERY_ROUTES` instead. +is deprecated and must be set by :setting:`task_routes` instead. :mod:`celery.result` -------------------- @@ -228,11 +228,11 @@ Settings ===================================== ===================================== **Setting name** **Replace with** ===================================== ===================================== -``BROKER_HOST`` :setting:`BROKER_URL` -``BROKER_PORT`` :setting:`BROKER_URL` -``BROKER_USER`` :setting:`BROKER_URL` -``BROKER_PASSWORD`` :setting:`BROKER_URL` -``BROKER_VHOST`` :setting:`BROKER_URL` +``BROKER_HOST`` :setting:`broker_url` +``BROKER_PORT`` :setting:`broker_url` +``BROKER_USER`` :setting:`broker_url` +``BROKER_PASSWORD`` :setting:`broker_url` +``BROKER_VHOST`` :setting:`broker_url` ===================================== ===================================== @@ -242,14 +242,14 @@ Settings ===================================== ===================================== **Setting name** **Replace with** ===================================== ===================================== -``CELERY_REDIS_HOST`` :setting:`CELERY_RESULT_BACKEND` -``CELERY_REDIS_PORT`` :setting:`CELERY_RESULT_BACKEND` -``CELERY_REDIS_DB`` :setting:`CELERY_RESULT_BACKEND` -``CELERY_REDIS_PASSWORD`` :setting:`CELERY_RESULT_BACKEND` -``REDIS_HOST`` :setting:`CELERY_RESULT_BACKEND` -``REDIS_PORT`` :setting:`CELERY_RESULT_BACKEND` -``REDIS_DB`` :setting:`CELERY_RESULT_BACKEND` -``REDIS_PASSWORD`` :setting:`CELERY_RESULT_BACKEND` +``CELERY_REDIS_HOST`` :setting:`result_backend` +``CELERY_REDIS_PORT`` :setting:`result_backend` +``CELERY_REDIS_DB`` :setting:`result_backend` +``CELERY_REDIS_PASSWORD`` :setting:`result_backend` +``REDIS_HOST`` :setting:`result_backend` +``REDIS_PORT`` :setting:`result_backend` +``REDIS_DB`` :setting:`result_backend` +``REDIS_PASSWORD`` :setting:`result_backend` ===================================== ===================================== Logging Settings @@ -273,7 +273,7 @@ Other Settings **Setting name** **Replace with** ===================================== ===================================== ``CELERY_TASK_ERROR_WITELIST`` Annotate ``Task.ErrorMail`` -``CELERY_AMQP_TASK_RESULT_EXPIRES`` :setting:`CELERY_TASK_RESULT_EXPIRES` +``CELERY_AMQP_TASK_RESULT_EXPIRES`` :setting:`result_expires` ===================================== ===================================== @@ -287,12 +287,12 @@ Removals for version 2.0 ===================================== ===================================== **Setting name** **Replace with** ===================================== ===================================== -`CELERY_AMQP_CONSUMER_QUEUES` `CELERY_QUEUES` -`CELERY_AMQP_CONSUMER_QUEUES` `CELERY_QUEUES` -`CELERY_AMQP_EXCHANGE` `CELERY_DEFAULT_EXCHANGE` -`CELERY_AMQP_EXCHANGE_TYPE` `CELERY_DEFAULT_AMQP_EXCHANGE_TYPE` -`CELERY_AMQP_CONSUMER_ROUTING_KEY` `CELERY_QUEUES` -`CELERY_AMQP_PUBLISHER_ROUTING_KEY` `CELERY_DEFAULT_ROUTING_KEY` +`CELERY_AMQP_CONSUMER_QUEUES` `task_queues` +`CELERY_AMQP_CONSUMER_QUEUES` `task_queues` +`CELERY_AMQP_EXCHANGE` `task_default_exchange` +`CELERY_AMQP_EXCHANGE_TYPE` `task_default_exchange_type` +`CELERY_AMQP_CONSUMER_ROUTING_KEY` `task_queues` +`CELERY_AMQP_PUBLISHER_ROUTING_KEY` `task_default_routing_key` ===================================== ===================================== * :envvar:`CELERY_LOADER` definitions without class name. @@ -303,4 +303,4 @@ Removals for version 2.0 * :meth:`TaskSet.run`. Use :meth:`celery.task.base.TaskSet.apply_async` instead. -* The module :mod:`celery.task.rest`; use :mod:`celery.task.http` instead. +* The module :mod:`celery.task.rest`; use :mod:`celery.task.httpY` instead. diff --git a/docs/userguide/application.rst b/docs/userguide/application.rst index 4f7dcff2d..0c2df9030 100644 --- a/docs/userguide/application.rst +++ b/docs/userguide/application.rst @@ -117,18 +117,18 @@ or you can use a dedicated configuration module. The configuration is available as :attr:`@conf`:: - >>> app.conf.CELERY_TIMEZONE + >>> app.conf.timezone 'Europe/London' where you can also set configuration values directly:: - >>> app.conf.CELERY_ENABLE_UTC = True + >>> app.conf.enable_utc = True and update several keys at once by using the ``update`` method:: >>> app.conf.update( - ... CELERY_ENABLE_UTC=True, - ... CELERY_TIMEZONE='Europe/London', + ... enable_utc=True, + ... timezone='Europe/London', ...) The configuration object consists of multiple dictionaries @@ -175,8 +175,8 @@ The ``celeryconfig`` module may then look like this: .. code-block:: python - CELERY_ENABLE_UTC = True - CELERY_TIMEZONE = 'Europe/London' + enable_utc = True + timezone = 'Europe/London' Example 2: Using a configuration module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -207,8 +207,8 @@ Example 3: Using a configuration class/object app = Celery() class Config: - CELERY_ENABLE_UTC = True - CELERY_TIMEZONE = 'Europe/London' + enable_utc = True + timezone = 'Europe/London' app.config_from_object(Config) # or using the fully qualified name of the object: diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index e33e2aa9d..47cc7e1af 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -247,8 +247,8 @@ To disable retry you can set the ``retry`` execution option to :const:`False`: .. hlist:: :columns: 2 - - :setting:`CELERY_TASK_PUBLISH_RETRY` - - :setting:`CELERY_TASK_PUBLISH_RETRY_POLICY` + - :setting:`task_publish_retry` + - :setting:`task_publish_retry_policy` Retry Policy ------------ @@ -315,7 +315,7 @@ so every message in Celery has a ``content_type`` header that describes the serialization method used to encode it. The default serializer is :mod:`pickle`, but you can -change this using the :setting:`CELERY_TASK_SERIALIZER` setting, +change this using the :setting:`task_serializer` setting, or for each individual task, or even per message. There's built-in support for :mod:`pickle`, `JSON`, `YAML` @@ -382,7 +382,7 @@ to use when sending a task: 1. The `serializer` execution option. 2. The :attr:`@-Task.serializer` attribute - 3. The :setting:`CELERY_TASK_SERIALIZER` setting. + 3. The :setting:`task_serializer` setting. Example setting a custom serializer for a single task invocation: @@ -405,7 +405,7 @@ to use when sending a task: 1. The `compression` execution option. 2. The :attr:`@-Task.compression` attribute. - 3. The :setting:`CELERY_MESSAGE_COMPRESSION` attribute. + 3. The :setting:`task_compression` attribute. Example specifying the compression used when calling a task:: @@ -424,7 +424,7 @@ Connections The connection pool is enabled by default since version 2.5. - See the :setting:`BROKER_POOL_LIMIT` setting for more information. + See the :setting:`broker_pool_limit` setting for more information. You can handle the connection manually by creating a publisher: @@ -475,7 +475,7 @@ the workers :option:`-Q` argument: .. seealso:: Hard-coding queue names in code is not recommended, the best practice - is to use configuration routers (:setting:`CELERY_ROUTES`). + is to use configuration routers (:setting:`task_routes`). To find out more about routing, please see :ref:`guide-routing`. diff --git a/docs/userguide/canvas.rst b/docs/userguide/canvas.rst index b55fe5770..97cb06449 100644 --- a/docs/userguide/canvas.rst +++ b/docs/userguide/canvas.rst @@ -865,8 +865,8 @@ Important Notes ~~~~~~~~~~~~~~~ Tasks used within a chord must *not* ignore their results. In practice this -means that you must enable a :const:`CELERY_RESULT_BACKEND` in order to use -chords. Additionally, if :const:`CELERY_IGNORE_RESULT` is set to :const:`True` +means that you must enable a :const:`result_backend` in order to use +chords. Additionally, if :const:`task_ignore_result` is set to :const:`True` in your configuration, be sure that the individual tasks to be used within the chord are defined with :const:`ignore_result=False`. This applies to both Task subclasses and decorated tasks. diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index 6fd606017..083e9dacf 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -653,7 +653,7 @@ task-sent queue, exchange, routing_key)`` Sent when a task message is published and -the :setting:`CELERY_SEND_TASK_SENT_EVENT` setting is enabled. +the :setting:`task_send_sent_event` setting is enabled. .. event:: task-received diff --git a/docs/userguide/optimizing.rst b/docs/userguide/optimizing.rst index 673951083..7d37c9865 100644 --- a/docs/userguide/optimizing.rst +++ b/docs/userguide/optimizing.rst @@ -75,7 +75,7 @@ Broker Connection Pools The broker connection pool is enabled by default since version 2.5. -You can tweak the :setting:`BROKER_POOL_LIMIT` setting to minimize +You can tweak the :setting:`broker_pool_limit` setting to minimize contention, and the value should be based on the number of active threads/greenthreads using broker connections. @@ -96,18 +96,18 @@ to improve performance: from kombu import Exchange, Queue - CELERY_QUEUES = ( + task_queues = ( Queue('celery', routing_key='celery'), Queue('transient', Exchange('transient', delivery_mode=1), routing_key='transient', durable=False), ) -or by using :setting:`CELERY_ROUTES`: +or by using :setting:`task_routes`: .. code-block:: python - CELERY_ROUTES = { + task_routes = { 'proj.tasks.add': {'queue': 'celery', 'delivery_mode': 'transient'} } @@ -117,7 +117,7 @@ A value of 1 means that the message will not be written to disk, and a value of 2 (default) means that the message can be written to disk. To direct a task to your new transient queue you can specify the queue -argument (or use the :setting:`CELERY_ROUTES` setting): +argument (or use the :setting:`task_routes` setting): .. code-block:: python @@ -145,7 +145,7 @@ available worker nodes that may be able to process them sooner [*]_, or that the messages may not even fit in memory. The workers' default prefetch count is the -:setting:`CELERYD_PREFETCH_MULTIPLIER` setting multiplied by the number +:setting:`worker_prefetch_multiplier` setting multiplied by the number of concurrency slots[*]_ (processes/threads/greenthreads). If you have many tasks with a long duration you want @@ -169,7 +169,7 @@ the tasks according to the run-time. (see :ref:`guide-routing`). nodes starting. If there are 3 offline nodes and one active node, all messages will be delivered to the active node. -.. [*] This is the concurrency setting; :setting:`CELERYD_CONCURRENCY` or the +.. [*] This is the concurrency setting; :setting:`worker_concurrency` or the :option:`-c` option to the :program:`celery worker` program. @@ -195,8 +195,8 @@ You can enable this behavior by using the following configuration options: .. code-block:: python - CELERY_ACKS_LATE = True - CELERYD_PREFETCH_MULTIPLIER = 1 + task_acks_late = True + worker_prefetch_multiplier = 1 .. _prefork-pool-prefetch: diff --git a/docs/userguide/periodic-tasks.rst b/docs/userguide/periodic-tasks.rst index 0d6bb84e9..319fefc29 100644 --- a/docs/userguide/periodic-tasks.rst +++ b/docs/userguide/periodic-tasks.rst @@ -13,7 +13,7 @@ Introduction :program:`celery beat` is a scheduler. It kicks off tasks at regular intervals, which are then executed by the worker nodes available in the cluster. -By default the entries are taken from the :setting:`CELERYBEAT_SCHEDULE` setting, +By default the entries are taken from the :setting:`beat_schedule` setting, but custom stores can also be used, like storing the entries in an SQL database. @@ -28,18 +28,18 @@ Time Zones ========== The periodic task schedules uses the UTC time zone by default, -but you can change the time zone used using the :setting:`CELERY_TIMEZONE` +but you can change the time zone used using the :setting:`timezone` setting. An example time zone could be `Europe/London`: .. code-block:: python - CELERY_TIMEZONE = 'Europe/London' + timezone = 'Europe/London' This setting must be added to your app, either by configuration it directly -using (``app.conf.CELERY_TIMEZONE = 'Europe/London'``), or by adding +using (``app.conf.timezone = 'Europe/London'``), or by adding it to your configuration module if you have set one up using ``app.config_from_object``. See :ref:`celerytut-configuration` for more information about configuration options. @@ -58,7 +58,7 @@ schedule manually. For Django users the time zone specified in the ``TIME_ZONE`` setting will be used, or you can specify a custom time zone for Celery alone - by using the :setting:`CELERY_TIMEZONE` setting. + by using the :setting:`timezone` setting. The database scheduler will not reset when timezone related settings change, so you must do this manually: @@ -107,14 +107,14 @@ Setting these up from within the ``on_after_configure`` handler means that we will not evaluate the app at module level when using ``test.s()``. The `@add_periodic_task` function will add the entry to the -:setting:`CELERYBEAT_SCHEDULE` setting behind the scenes, which also +:setting:`beat_schedule` setting behind the scenes, which also can be used to set up periodic tasks manually: Example: Run the `tasks.add` task every 30 seconds. .. code-block:: python - CELERYBEAT_SCHEDULE = { + beat_schedule = { 'add-every-30-seconds': { 'task': 'tasks.add', 'schedule': 30.0, @@ -122,7 +122,7 @@ Example: Run the `tasks.add` task every 30 seconds. }, } - CELERY_TIMEZONE = 'UTC' + timezone = 'UTC' .. note:: @@ -203,7 +203,7 @@ the :class:`~celery.schedules.crontab` schedule type: from celery.schedules import crontab - CELERYBEAT_SCHEDULE = { + beat_schedule = { # Executes every Monday morning at 7:30 A.M 'add-every-monday-morning': { 'task': 'tasks.add', @@ -285,7 +285,7 @@ sunset, dawn or dusk, you can use the from celery.schedules import solar - CELERYBEAT_SCHEDULE = { + beat_schedule = { # Executes at sunset in Melbourne 'add-at-melbourne-sunset': { 'task': 'tasks.add', diff --git a/docs/userguide/remote-tasks.rst b/docs/userguide/remote-tasks.rst index d8fe3587a..d36867e43 100644 --- a/docs/userguide/remote-tasks.rst +++ b/docs/userguide/remote-tasks.rst @@ -53,7 +53,7 @@ Enabling the HTTP task ---------------------- To enable the HTTP dispatch task you have to add :mod:`celery.task.http` -to :setting:`CELERY_IMPORTS`, or start the worker with ``-I +to :setting:`imports`, or start the worker with ``-I celery.task.http``. diff --git a/docs/userguide/routing.rst b/docs/userguide/routing.rst index 0c4ca10ba..6e882ad70 100644 --- a/docs/userguide/routing.rst +++ b/docs/userguide/routing.rst @@ -25,17 +25,17 @@ Automatic routing ----------------- The simplest way to do routing is to use the -:setting:`CELERY_CREATE_MISSING_QUEUES` setting (on by default). +:setting:`task_create_missing_queues` setting (on by default). With this setting on, a named queue that is not already defined in -:setting:`CELERY_QUEUES` will be created automatically. This makes it easy to +:setting:`task_queues` will be created automatically. This makes it easy to perform simple routing tasks. Say you have two servers, `x`, and `y` that handles regular tasks, and one server `z`, that only handles feed related tasks. You can use this configuration:: - CELERY_ROUTES = {'feed.tasks.import_feed': {'queue': 'feeds'}} + task_routes = {'feed.tasks.import_feed': {'queue': 'feeds'}} With this route enabled import feed tasks will be routed to the `"feeds"` queue, while all other tasks will be routed to the default queue @@ -66,8 +66,8 @@ configuration: from kombu import Exchange, Queue - CELERY_DEFAULT_QUEUE = 'default' - CELERY_QUEUES = ( + task_default_queue = 'default' + task_queues = ( Queue('default', Exchange('default'), routing_key='default'), ) @@ -105,27 +105,27 @@ configuration: from kombu import Queue - CELERY_DEFAULT_QUEUE = 'default' - CELERY_QUEUES = ( + task_default_queue = 'default' + task_queues = ( Queue('default', routing_key='task.#'), Queue('feed_tasks', routing_key='feed.#'), ) - CELERY_DEFAULT_EXCHANGE = 'tasks' - CELERY_DEFAULT_EXCHANGE_TYPE = 'topic' - CELERY_DEFAULT_ROUTING_KEY = 'task.default' + task_default_exchange = 'tasks' + task_default_exchange_type = 'topic' + task_default_routing_key = 'task.default' -:setting:`CELERY_QUEUES` is a list of :class:`~kombu.entitity.Queue` +:setting:`task_queues` is a list of :class:`~kombu.entitity.Queue` instances. If you don't set the exchange or exchange type values for a key, these -will be taken from the :setting:`CELERY_DEFAULT_EXCHANGE` and -:setting:`CELERY_DEFAULT_EXCHANGE_TYPE` settings. +will be taken from the :setting:`task_default_exchange` and +:setting:`task_default_exchange_type` settings. To route a task to the `feed_tasks` queue, you can add an entry in the -:setting:`CELERY_ROUTES` setting: +:setting:`task_routes` setting: .. code-block:: python - CELERY_ROUTES = { + task_routes = { 'feeds.tasks.import_feed': { 'queue': 'feed_tasks', 'routing_key': 'feed.import', @@ -170,7 +170,7 @@ just specify a custom exchange and exchange type: from kombu import Exchange, Queue - CELERY_QUEUES = ( + task_queues = ( Queue('feed_tasks', routing_key='feed.#'), Queue('regular_tasks', routing_key='task.#'), Queue('image_tasks', exchange=Exchange('mediatasks', type='direct'), @@ -249,7 +249,7 @@ The steps required to send and receive messages are: 3. Bind the queue to the exchange. Celery automatically creates the entities necessary for the queues in -:setting:`CELERY_QUEUES` to work (except if the queue's `auto_declare` +:setting:`task_queues` to work (except if the queue's `auto_declare` setting is set to :const:`False`). Here's an example queue configuration with three queues; @@ -259,14 +259,14 @@ One for video, one for images and one default queue for everything else: from kombu import Exchange, Queue - CELERY_QUEUES = ( + task_queues = ( Queue('default', Exchange('default'), routing_key='default'), Queue('videos', Exchange('media'), routing_key='media.video'), Queue('images', Exchange('media'), routing_key='media.image'), ) - CELERY_DEFAULT_QUEUE = 'default' - CELERY_DEFAULT_EXCHANGE_TYPE = 'direct' - CELERY_DEFAULT_ROUTING_KEY = 'default' + task_default_queue = 'default' + task_default_exchange_type = 'direct' + task_default_routing_key = 'default' .. _amqp-exchange-types: @@ -459,7 +459,7 @@ Routing Tasks Defining queues --------------- -In Celery available queues are defined by the :setting:`CELERY_QUEUES` setting. +In Celery available queues are defined by the :setting:`task_queues` setting. Here's an example queue configuration with three queues; One for video, one for images and one default queue for everything else: @@ -469,21 +469,21 @@ One for video, one for images and one default queue for everything else: default_exchange = Exchange('default', type='direct') media_exchange = Exchange('media', type='direct') - CELERY_QUEUES = ( + task_queues = ( Queue('default', default_exchange, routing_key='default'), Queue('videos', media_exchange, routing_key='media.video'), Queue('images', media_exchange, routing_key='media.image') ) - CELERY_DEFAULT_QUEUE = 'default' - CELERY_DEFAULT_EXCHANGE = 'default' - CELERY_DEFAULT_ROUTING_KEY = 'default' + task_default_queue = 'default' + task_default_exchange = 'default' + task_default_routing_key = 'default' -Here, the :setting:`CELERY_DEFAULT_QUEUE` will be used to route tasks that +Here, the :setting:`task_default_queue` will be used to route tasks that doesn't have an explicit route. The default exchange, exchange type and routing key will be used as the default routing values for tasks, and as the default values for entries -in :setting:`CELERY_QUEUES`. +in :setting:`task_queues`. .. _routing-task-destination: @@ -492,7 +492,7 @@ Specifying task destination The destination for a task is decided by the following (in order): -1. The :ref:`routers` defined in :setting:`CELERY_ROUTES`. +1. The :ref:`routers` defined in :setting:`task_routes`. 2. The routing arguments to :func:`Task.apply_async`. 3. Routing related attributes defined on the :class:`~celery.task.base.Task` itself. @@ -524,7 +524,7 @@ All you need to define a new router is to create a class with a return None If you return the ``queue`` key, it will expand with the defined settings of -that queue in :setting:`CELERY_QUEUES`: +that queue in :setting:`task_queues`: .. code-block:: javascript @@ -540,27 +540,27 @@ becomes --> 'routing_key': 'video.compress'} -You install router classes by adding them to the :setting:`CELERY_ROUTES` +You install router classes by adding them to the :setting:`task_routes` setting: .. code-block:: python - CELERY_ROUTES = (MyRouter(),) + task_routes = (MyRouter(),) Router classes can also be added by name: .. code-block:: python - CELERY_ROUTES = ('myapp.routers.MyRouter',) + task_routes = ('myapp.routers.MyRouter',) For simple task name -> route mappings like the router example above, -you can simply drop a dict into :setting:`CELERY_ROUTES` to get the +you can simply drop a dict into :setting:`task_routes` to get the same behavior: .. code-block:: python - CELERY_ROUTES = ( + task_routes = ( {'myapp.tasks.compress_video': { 'queue': 'video', 'routing_key': 'video.compress', @@ -581,9 +581,8 @@ copies of tasks to all workers connected to it: from kombu.common import Broadcast - CELERY_QUEUES = (Broadcast('broadcast_tasks'),) - - CELERY_ROUTES = {'tasks.reload_cache': {'queue': 'broadcast_tasks'}} + task_queues = (Broadcast('broadcast_tasks'),) + task_routes = {'tasks.reload_cache': {'queue': 'broadcast_tasks'}} Now the ``tasks.reload_cache`` task will be sent to every worker consuming from this queue. diff --git a/docs/userguide/security.rst b/docs/userguide/security.rst index f000294bb..f1ebe3e18 100644 --- a/docs/userguide/security.rst +++ b/docs/userguide/security.rst @@ -47,7 +47,7 @@ this is something you should look at enabling. See for example http://www.rabbitmq.com/access-control.html. If supported by your broker backend, you can enable end-to-end SSL encryption -and authentication using :setting:`BROKER_USE_SSL`. +and authentication using :setting:`broker_use_ssl`. Client ------ @@ -104,7 +104,7 @@ unauthenticated. .. [*] http://nadiana.com/python-pickle-insecure You can disable untrusted content by specifying -a white-list of accepted content-types in the :setting:`CELERY_ACCEPT_CONTENT` +a white-list of accepted content-types in the :setting:`accept_content` setting: .. versionadded:: 3.0.18 @@ -117,7 +117,7 @@ setting: .. code-block:: python - CELERY_ACCEPT_CONTENT = ['json'] + accept_content = ['json'] This accepts a list of serializer names and content-types, so you could @@ -125,7 +125,7 @@ also specify the content type for json: .. code-block:: python - CELERY_ACCEPT_CONTENT = ['application/json'] + accept_content = ['application/json'] Celery also comes with a special `auth` serializer that validates communication between Celery clients and workers, making sure @@ -151,12 +151,12 @@ and then later verified by the worker using a public certificate. Optimally certificates should be signed by an official `Certificate Authority`_, but they can also be self-signed. -To enable this you should configure the :setting:`CELERY_TASK_SERIALIZER` +To enable this you should configure the :setting:`task_serializer` setting to use the `auth` serializer. Also required is configuring the paths used to locate private keys and certificates on the file-system: -the :setting:`CELERY_SECURITY_KEY`, -:setting:`CELERY_SECURITY_CERTIFICATE` and :setting:`CELERY_SECURITY_CERT_STORE` +the :setting:`security_key`, +:setting:`security_certificate` and :setting:`security_cert_store` settings respectively. With these configured it is also necessary to call the :func:`celery.setup_security` function. Note that this will also @@ -168,11 +168,13 @@ with the private key and certificate files located in `/etc/ssl`. .. code-block:: python - CELERY_SECURITY_KEY = '/etc/ssl/private/worker.key' - CELERY_SECURITY_CERTIFICATE = '/etc/ssl/certs/worker.pem' - CELERY_SECURITY_CERT_STORE = '/etc/ssl/certs/*.pem' - from celery.security import setup_security - setup_security() + app = Celery() + app.conf.update( + security_key='/etc/ssl/private/worker.key' + security_certificate='/etc/ssl/certs/worker.pem' + security_cert_store='/etc/ssl/certs/*.pem', + ) + app.setup_security() .. note:: diff --git a/docs/userguide/signals.rst b/docs/userguide/signals.rst index 8be7f37c2..cae2f7865 100644 --- a/docs/userguide/signals.rst +++ b/docs/userguide/signals.rst @@ -309,7 +309,7 @@ import_modules ~~~~~~~~~~~~~~ This signal is sent when a program (worker, beat, shell) etc, asks -for modules in the :setting:`CELERY_INCLUDE` and :setting:`CELERY_IMPORTS` +for modules in the :setting:`include` and :setting:`imports` settings to be imported. Sender is the app instance. @@ -369,7 +369,7 @@ to setup worker specific configuration: @celeryd_init.connect(sender='worker12@example.com') def configure_worker12(conf=None, **kwargs): - conf.CELERY_DEFAULT_RATE_LIMIT = '10/m' + conf.task_default_rate_limit = '10/m' or to set up configuration for multiple workers you can omit specifying a sender when you connect: @@ -381,9 +381,9 @@ sender when you connect: @celeryd_init.connect def configure_workers(sender=None, conf=None, **kwargs): if sender in ('worker1@example.com', 'worker2@example.com'): - conf.CELERY_DEFAULT_RATE_LIMIT = '10/m' + conf.task_default_rate_limit = '10/m' if sender == 'worker3@example.com': - conf.CELERYD_PREFETCH_MULTIPLIER = 0 + conf.worker_prefetch_multiplier = 0 Provides arguments: diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 9fe417af4..0f0183189 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -296,11 +296,11 @@ The request defines the following attributes: the client, and not by a worker. :eta: The original ETA of the task (if any). - This is in UTC time (depending on the :setting:`CELERY_ENABLE_UTC` + This is in UTC time (depending on the :setting:`enable_utc` setting). :expires: The original expiry time of the task (if any). - This is in UTC time (depending on the :setting:`CELERY_ENABLE_UTC` + This is in UTC time (depending on the :setting:`enable_utc` setting). :logfile: The file the worker logs to. See `Logging`_. @@ -323,7 +323,7 @@ The request defines the following attributes: :errback: A list of signatures to be called if this task fails. -:utc: Set to true the caller has utc enabled (:setting:`CELERY_ENABLE_UTC`). +:utc: Set to true the caller has utc enabled (:setting:`enable_utc`). .. versionadded:: 3.1 @@ -381,7 +381,7 @@ module. You can also use :func:`print`, as anything written to standard out/-err will be redirected to the logging system (you can disable this, -see :setting:`CELERY_REDIRECT_STDOUTS`). +see :setting:`worker_redirect_stdouts`). .. note:: @@ -400,7 +400,7 @@ see :setting:`CELERY_REDIRECT_STDOUTS`). @app.task(bind=True) def add(self, x, y): old_outs = sys.stdout, sys.stderr - rlevel = self.app.conf.CELERY_REDIRECT_STDOUTS_LEVEL + rlevel = self.app.conf.worker_redirect_stdouts_level try: self.app.log.redirect_stdouts_to_logger(logger, rlevel) print('Adding {0} + {1}'.format(x, y)) @@ -637,8 +637,8 @@ General Example: `"100/m"` (hundred tasks a minute). This will enforce a minimum delay of 600ms between starting two tasks on the same worker instance. - - Default is the :setting:`CELERY_DEFAULT_RATE_LIMIT` setting, + + Default is the :setting:`task_default_rate_limit` setting, which if not specified means rate limiting for tasks is disabled by default. Note that this is a *per worker instance* rate limit, and not a global @@ -670,7 +670,7 @@ General .. attribute:: Task.send_error_emails Send an email whenever a task of this type fails. - Defaults to the :setting:`CELERY_SEND_TASK_ERROR_EMAILS` setting. + Defaults to the :setting:`task_send_error_emails` setting. See :ref:`conf-error-mails` for more information. .. attribute:: Task.ErrorMail @@ -681,7 +681,7 @@ General .. attribute:: Task.serializer A string identifying the default serialization - method to use. Defaults to the :setting:`CELERY_TASK_SERIALIZER` + method to use. Defaults to the :setting:`task_serializer` setting. Can be `pickle` `json`, `yaml`, or any custom serialization methods that have been registered with :mod:`kombu.serialization.registry`. @@ -692,7 +692,7 @@ General A string identifying the default compression scheme to use. - Defaults to the :setting:`CELERY_MESSAGE_COMPRESSION` setting. + Defaults to the :setting:`task_compression` setting. Can be `gzip`, or `bzip2`, or any custom compression schemes that have been registered with the :mod:`kombu.compression` registry. @@ -702,7 +702,7 @@ General The result store backend to use for this task. An instance of one of the backend classes in `celery.backends`. Defaults to `app.backend` which is - defined by the :setting:`CELERY_RESULT_BACKEND` setting. + defined by the :setting:`result_backend` setting. .. attribute:: Task.acks_late @@ -714,7 +714,7 @@ General crashes in the middle of execution, which may be acceptable for some applications. - The global default can be overridden by the :setting:`CELERY_ACKS_LATE` + The global default can be overridden by the :setting:`task_acks_late` setting. .. _task-track-started: @@ -733,7 +733,7 @@ General will be available in the state metadata (e.g. `result.info['pid']`) The global default can be overridden by the - :setting:`CELERY_TRACK_STARTED` setting. + :setting:`task_track_started` setting. .. seealso:: @@ -800,7 +800,7 @@ poll for new states. The messages are transient (non-persistent) by default, so the results will disappear if the broker restarts. You can configure the result backend to send -persistent messages using the :setting:`CELERY_RESULT_PERSISTENT` setting. +persistent messages using the :setting:`result_persistent` setting. Database Result Backend ~~~~~~~~~~~~~~~~~~~~~~~ @@ -1286,7 +1286,7 @@ This is the list of tasks built-in to celery. Note that tasks will only be registered when the module they are defined in is imported. The default loader imports any modules listed in the -:setting:`CELERY_IMPORTS` setting. +:setting:`imports` setting. The entity responsible for registering your task in the registry is the metaclass: :class:`~celery.task.base.TaskType`. @@ -1330,7 +1330,7 @@ wastes time and resources. def mytask(…): something() -Results can even be disabled globally using the :setting:`CELERY_IGNORE_RESULT` +Results can even be disabled globally using the :setting:`task_ignore_result` setting. .. _task-disable-rate-limits: @@ -1342,12 +1342,12 @@ Disabling rate limits altogether is recommended if you don't have any tasks using them. This is because the rate limit subsystem introduces quite a lot of complexity. -Set the :setting:`CELERY_DISABLE_RATE_LIMITS` setting to globally disable +Set the :setting:`worker_disable_rate_limits` setting to globally disable rate limits: .. code-block:: python - CELERY_DISABLE_RATE_LIMITS = True + worker_disable_rate_limits = True You find additional optimization tips in the :ref:`Optimizing Guide `. diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index d9332b2c9..e82934710 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -449,8 +449,8 @@ time limit kills it: except SoftTimeLimitExceeded: clean_up_in_a_hurry() -Time limits can also be set using the :setting:`CELERYD_TASK_TIME_LIMIT` / -:setting:`CELERYD_TASK_SOFT_TIME_LIMIT` settings. +Time limits can also be set using the :setting:`task_time_limit` / +:setting:`task_soft_time_limit` settings. .. note:: @@ -508,7 +508,7 @@ list of workers you can include the ``destination`` argument: .. warning:: This won't affect workers with the - :setting:`CELERY_DISABLE_RATE_LIMITS` setting enabled. + :setting:`worker_disable_rate_limits` setting enabled. .. _worker-maxtasksperchild: @@ -526,7 +526,7 @@ This is useful if you have memory leaks you have no control over for example from closed source C extensions. The option can be set using the workers `--maxtasksperchild` argument -or using the :setting:`CELERYD_MAX_TASKS_PER_CHILD` setting. +or using the :setting:`worker_max_tasks_per_child` setting. Max memory per child setting ============================ @@ -571,7 +571,7 @@ numbers: the maximum and minimum number of pool processes:: You can also define your own rules for the autoscaler by subclassing :class:`~celery.worker.autoscaler.Autoscaler`. Some ideas for metrics include load average or the amount of memory available. -You can specify a custom autoscaler with the :setting:`CELERYD_AUTOSCALER` setting. +You can specify a custom autoscaler with the :setting:`worker_autoscaler` setting. .. _worker-queues: @@ -580,7 +580,7 @@ Queues A worker instance can consume from any number of queues. By default it will consume from all queues defined in the -:setting:`CELERY_QUEUES` setting (which if not specified defaults to the +:setting:`task_queues` setting (which if not specified defaults to the queue named ``celery``). You can specify what queues to consume from at startup, @@ -590,10 +590,10 @@ by giving a comma separated list of queues to the :option:`-Q` option: $ celery -A proj worker -l info -Q foo,bar,baz -If the queue name is defined in :setting:`CELERY_QUEUES` it will use that +If the queue name is defined in :setting:`task_queues` it will use that configuration, but if it's not defined in the list of queues Celery will automatically generate a new queue for you (depending on the -:setting:`CELERY_CREATE_MISSING_QUEUES` option). +:setting:`task_create_missing_queues` option). You can also tell the worker to start and stop consuming from a queue at runtime using the remote control commands :control:`add_consumer` and @@ -731,7 +731,7 @@ pool support: *prefork, eventlet, gevent, threads, solo* Starting :program:`celery worker` with the :option:`--autoreload` option will enable the worker to watch for file system changes to all imported task modules imported (and also any non-task modules added to the -:setting:`CELERY_IMPORTS` setting or the :option:`-I|--include` option). +:setting:`imports` setting or the :option:`-I|--include` option). This is an experimental feature intended for use in development only, using auto-reload in production is discouraged as the behavior of reloading @@ -783,7 +783,7 @@ Pool Restart Command .. versionadded:: 2.5 -Requires the :setting:`CELERYD_POOL_RESTARTS` setting to be enabled. +Requires the :setting:`worker_pool_restarts` setting to be enabled. The remote control command :control:`pool_restart` sends restart requests to the workers child processes. It is particularly useful for forcing diff --git a/examples/celery_http_gateway/settings.py b/examples/celery_http_gateway/settings.py index 750f18a7b..a56e1061a 100644 --- a/examples/celery_http_gateway/settings.py +++ b/examples/celery_http_gateway/settings.py @@ -5,7 +5,6 @@ DEBUG = True TEMPLATE_DEBUG = DEBUG -CARROT_BACKEND = 'amqp' CELERY_RESULT_BACKEND = 'database' BROKER_URL = 'amqp://guest:guest@localhost:5672//' diff --git a/examples/django/proj/celery.py b/examples/django/proj/celery.py index dc3ad1415..02020c6eb 100644 --- a/examples/django/proj/celery.py +++ b/examples/django/proj/celery.py @@ -11,7 +11,7 @@ # Using a string here means the worker will not have to # pickle the object when using Windows. -app.config_from_object('django.conf:settings') +app.config_from_object('django.conf:settings', namespace='CELERY_') # load task modules from all registered Django app configs. app.autodiscover_tasks() diff --git a/examples/django/proj/settings.py b/examples/django/proj/settings.py index 9a6a7e8de..2b61b564e 100644 --- a/examples/django/proj/settings.py +++ b/examples/django/proj/settings.py @@ -6,7 +6,7 @@ # Celery settings -BROKER_URL = 'amqp://guest:guest@localhost//' +CELERY_BROKER_URL = 'amqp://guest:guest@localhost//' #: Only add pickle to this list if your broker is secured #: from unwanted access (see userguide/security.html) diff --git a/examples/eventlet/celeryconfig.py b/examples/eventlet/celeryconfig.py index 2dc32edc2..9e3d1ec7f 100644 --- a/examples/eventlet/celeryconfig.py +++ b/examples/eventlet/celeryconfig.py @@ -3,12 +3,12 @@ sys.path.insert(0, os.getcwd()) # ## Start worker with -P eventlet -# Never use the CELERYD_POOL setting as that will patch +# Never use the worker_pool setting as that will patch # the worker too late. -BROKER_URL = 'amqp://guest:guest@localhost:5672//' -CELERY_DISABLE_RATE_LIMITS = True -CELERY_RESULT_BACKEND = 'amqp' -CELERY_TASK_RESULT_EXPIRES = 30 * 60 +broker_url = 'amqp://guest:guest@localhost:5672//' +worker_disable_rate_limits = True +result_backend = 'amqp' +result_expires = 30 * 60 -CELERY_IMPORTS = ('tasks', 'webcrawler') +imports = ('tasks', 'webcrawler') diff --git a/examples/gevent/celeryconfig.py b/examples/gevent/celeryconfig.py index e3714f277..a7ea06aa4 100644 --- a/examples/gevent/celeryconfig.py +++ b/examples/gevent/celeryconfig.py @@ -3,11 +3,10 @@ sys.path.insert(0, os.getcwd()) # ## Note: Start worker with -P gevent, -# do not use the CELERYD_POOL option. +# do not use the worker_pool option. -BROKER_URL = 'amqp://guest:guest@localhost:5672//' -CELERY_DISABLE_RATE_LIMITS = True -CELERY_RESULT_BACKEND = 'amqp' -CELERY_TASK_RESULT_EXPIRES = 30 * 60 +broker_url = 'amqp://guest:guest@localhost:5672//' +result_backend = 'amqp' +result_expires = 30 * 60 -CELERY_IMPORTS = ('tasks',) +imports = ('tasks',) diff --git a/examples/next-steps/proj/celery.py b/examples/next-steps/proj/celery.py index db98708bd..d200c2d35 100644 --- a/examples/next-steps/proj/celery.py +++ b/examples/next-steps/proj/celery.py @@ -9,7 +9,7 @@ # Optional configuration, see the application user guide. app.conf.update( - CELERY_TASK_RESULT_EXPIRES=3600, + result_expires=3600, ) if __name__ == '__main__': diff --git a/extra/release/verify_config_reference.py b/extra/release/verify_config_reference.py index b4d37c893..59b6e56c6 100644 --- a/extra/release/verify_config_reference.py +++ b/extra/release/verify_config_reference.py @@ -6,19 +6,20 @@ from celery.app.defaults import NAMESPACES, flatten ignore = { - 'CELERYD_AGENT', - 'CELERYD_POOL_PUTLOCKS', - 'BROKER_HOST', - 'BROKER_USER', - 'BROKER_PASSWORD', - 'BROKER_VHOST', - 'BROKER_PORT', - 'CELERY_CHORD_PROPAGATES', - 'CELERY_REDIS_HOST', - 'CELERY_REDIS_PORT', - 'CELERY_REDIS_DB', - 'CELERY_REDIS_PASSWORD', - 'CELERYD_FORCE_EXECV', + 'worker_agent', + 'worker_pool_putlocks', + 'broker_host', + 'broker_user', + 'broker_password', + 'broker_vhost', + 'broker_port', + 'broker_transport', + 'chord_propagates', + 'redis_host', + 'redis_port', + 'redis_db', + 'redis_password', + 'worker_force_execv', } diff --git a/funtests/benchmarks/bench_worker.py b/funtests/benchmarks/bench_worker.py index 07e6e256b..d6535d6b3 100644 --- a/funtests/benchmarks/bench_worker.py +++ b/funtests/benchmarks/bench_worker.py @@ -20,12 +20,12 @@ app = Celery('bench_worker') app.conf.update( - BROKER_TRANSPORT=BROKER_TRANSPORT, - BROKER_POOL_LIMIT=10, - CELERYD_POOL='solo', - CELERYD_PREFETCH_MULTIPLIER=0, - CELERY_DEFAULT_DELIVERY_MODE=1, - CELERY_QUEUES={ + broker_transport=BROKER_TRANSPORT, + broker_pool_limit=10, + celeryd_pool='solo', + celeryd_prefetch_multiplier=0, + default_delivery_mode=1, + queues={ 'bench.worker': { 'exchange': 'bench.worker', 'routing_key': 'bench.worker', @@ -35,9 +35,9 @@ 'auto_delete': True, } }, - CELERY_TASK_SERIALIZER='json', - CELERY_DEFAULT_QUEUE='bench.worker', - CELERY_BACKEND=None, + task_serializer='json', + default_queue='bench.worker', + result_backend=None, ), diff --git a/funtests/stress/stress/app.py b/funtests/stress/stress/app.py index df028d39d..9ecd289fc 100644 --- a/funtests/stress/stress/app.py +++ b/funtests/stress/stress/app.py @@ -58,6 +58,8 @@ def _marker(s, sep='-'): @app.task def add(x, y): + import locale + print(locale.getdefaultlocale()) return x + y diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index f46b12de5..522e6c60c 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -50,88 +50,88 @@ def template_names(): @template() class default(object): - BROKER_HEARTBEAT = 30 - CELERY_ACCEPT_CONTENT = ['json'] - CELERY_DEFAULT_QUEUE = CSTRESS_QUEUE - CELERY_TASK_SERIALIZER = 'json' - CELERY_RESULT_SERIALIZER = 'json' - CELERY_RESULT_PERSISTENT = True - CELERY_TASK_RESULT_EXPIRES = 300 - CELERY_QUEUES = [ + accept_content = ['json'] + broker_url = os.environ.get('CSTRESS_BROKER', 'amqp://') + broker_heartbeat = 30 + result_backend = os.environ.get('CSTRESS_BACKEND', 'rpc://') + result_serializer = 'json' + result_persistent = True + result_expires = 300 + result_cache_max = -1 + task_default_queue = CSTRESS_QUEUE + task_queues = [ Queue(CSTRESS_QUEUE, exchange=Exchange(CSTRESS_QUEUE), routing_key=CSTRESS_QUEUE, durable=not CSTRESS_TRANS, no_ack=CSTRESS_TRANS), ] - CELERY_MAX_CACHED_RESULTS = -1 - BROKER_URL = os.environ.get('CSTRESS_BROKER', 'amqp://') - CELERY_RESULT_BACKEND = os.environ.get('CSTRESS_BACKEND', 'rpc://') - CELERYD_PREFETCH_MULTIPLIER = int(os.environ.get('CSTRESS_PREFETCH', 10)) - CELERY_TASK_PUBLISH_RETRY_POLICY = { + task_serializer = 'json' + task_publish_retry_policy = { 'max_retries': 100, 'interval_max': 2, 'interval_step': 0.1, } - CELERY_TASK_PROTOCOL = 2 + task_protocol = 2 if CSTRESS_TRANS: - CELERY_DEFAULT_DELIVERY_MODE = 1 + task_default_delivery_mode = 1 + worker_prefetch_multiplier = int(os.environ.get('CSTRESS_PREFETCH', 10)) @template() class redis(default): - BROKER_URL = os.environ.get('CSTRESS_BROKER', 'redis://') - CELERY_RESULT_BACKEND = os.environ.get( - 'CSTRESS_BACKEND', 'redis://?new_join=1', - ) - BROKER_TRANSPORT_OPTIONS = { + broker_url = os.environ.get('CSTRESS_BROKER', 'redis://') + broker_transport_options = { 'fanout_prefix': True, 'fanout_patterns': True, } + result_backend = os.environ.get( + 'CSTRESS_BACKEND', 'redis://?new_join=1', + ) @template() class redistore(default): - CELERY_RESULT_BACKEND = 'redis://?new_join=1' + result_backend = 'redis://?new_join=1' @template() class acks_late(default): - CELERY_ACKS_LATE = True + task_acks_late = True @template() class pickle(default): - CELERY_ACCEPT_CONTENT = ['pickle', 'json'] - CELERY_TASK_SERIALIZER = 'pickle' - CELERY_RESULT_SERIALIZER = 'pickle' + accept_content = ['pickle', 'json'] + task_serializer = 'pickle' + result_serializer = 'pickle' @template() class confirms(default): - BROKER_URL = 'pyamqp://' - BROKER_TRANSPORT_OPTIONS = {'confirm_publish': True} + broker_url = 'pyamqp://' + broker_transport_options = {'confirm_publish': True} @template() class events(default): - CELERY_SEND_EVENTS = True - CELERY_SEND_TASK_SENT_EVENT = True + task_send_events = True + task_send_sent_event = True @template() class execv(default): - CELERYD_FORCE_EXECV = True + worker_force_execv = True @template() class sqs(default): - BROKER_URL = 'sqs://' - BROKER_TRANSPORT_OPTIONS = { + broker_url = 'sqs://' + broker_transport_options = { 'region': os.environ.get('AWS_REGION', 'us-east-1'), } @template() class proto1(default): - CELERY_TASK_PROTOCOL = 1 + task_protocol = 1 diff --git a/funtests/suite/config.py b/funtests/suite/config.py index 8060126b7..8f895a1e3 100644 --- a/funtests/suite/config.py +++ b/funtests/suite/config.py @@ -1,18 +1,18 @@ import atexit import os -BROKER_URL = os.environ.get('BROKER_URL') or 'amqp://' -CELERY_RESULT_BACKEND = 'amqp://' -CELERY_SEND_TASK_ERROR_EMAILS = False +broker_url = os.environ.get('BROKER_URL') or 'amqp://' +result_backend = 'amqp://' +send_task_error_emails = False -CELERY_DEFAULT_QUEUE = 'testcelery' -CELERY_DEFAULT_EXCHANGE = 'testcelery' -CELERY_DEFAULT_ROUTING_KEY = 'testcelery' -CELERY_QUEUES = {'testcelery': {'routing_key': 'testcelery'}} +default_queue = 'testcelery' +default_exchange = 'testcelery' +default_routing_key = 'testcelery' +queues = {'testcelery': {'routing_key': 'testcelery'}} -CELERYD_LOG_COLOR = False +log_color = False -CELERY_IMPORTS = ('celery.tests.functional.tasks',) +imports = ('celery.tests.functional.tasks',) @atexit.register From ff17246e19e06250c570e8245fa81abe34d78b95 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 Oct 2015 18:01:47 -0700 Subject: [PATCH 0757/1103] Removes result.serializable and result_from_serializable as per deprecation timeline --- celery/result.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/celery/result.py b/celery/result.py index 4e3770164..5e3094ad0 100644 --- a/celery/result.py +++ b/celery/result.py @@ -94,7 +94,6 @@ def __init__(self, id, backend=None, task_name=None, def as_tuple(self): parent = self.parent return (self.id, parent and parent.as_tuple()), None - serializable = as_tuple # XXX compat def forget(self): """Forget about (and possibly remove the result of) this task.""" @@ -814,7 +813,6 @@ def __repr__(self): def as_tuple(self): return self.id, [r.as_tuple() for r in self.results] - serializable = as_tuple # XXX compat @property def children(self): @@ -940,4 +938,3 @@ def result_from_tuple(r, app=None): parent = result_from_tuple(parent, app) return Result(id, parent=parent) return r -from_serializable = result_from_tuple # XXX compat From e3cab12540f1be6d36702b9d44f111b3d952fd7d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 Oct 2015 18:06:07 -0700 Subject: [PATCH 0758/1103] Removes deprecated TaskSet and TaskSetResult, and the .task.sets module --- celery/app/base.py | 10 - celery/result.py | 32 +-- celery/task/__init__.py | 4 +- celery/task/sets.py | 88 -------- celery/tests/compat_modules/test_sets.py | 245 ----------------------- celery/tests/tasks/test_result.py | 26 --- 6 files changed, 2 insertions(+), 403 deletions(-) delete mode 100644 celery/task/sets.py delete mode 100644 celery/tests/compat_modules/test_sets.py diff --git a/celery/app/base.py b/celery/app/base.py index eb91173f8..440c6d0ff 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -996,16 +996,6 @@ def GroupResult(self): """ return self.subclass_with_self('celery.result:GroupResult') - @cached_property - def TaskSet(self): # XXX compat - """Deprecated! Please use :class:`celery.group` instead.""" - return self.subclass_with_self('celery.task.sets:TaskSet') - - @cached_property - def TaskSetResult(self): # XXX compat - """Deprecated! Please use :attr:`GroupResult` instead.""" - return self.subclass_with_self('celery.result:TaskSetResult') - @property def pool(self): """Broker connection pool: :class:`~@pool`. diff --git a/celery/result.py b/celery/result.py index 5e3094ad0..25cd831fb 100644 --- a/celery/result.py +++ b/celery/result.py @@ -756,8 +756,7 @@ def backend(self): class GroupResult(ResultSet): """Like :class:`ResultSet`, but with an associated id. - This type is returned by :class:`~celery.group`, and the - deprecated TaskSet, meth:`~celery.task.TaskSet.apply_async` method. + This type is returned by :class:`~celery.group`. It enables inspection of the tasks state and return values as a single entity. @@ -826,35 +825,6 @@ def restore(self, id, backend=None): ).restore_group(id) -class TaskSetResult(GroupResult): - """Deprecated version of :class:`GroupResult`""" - - def __init__(self, taskset_id, results=None, **kwargs): - # XXX supports the taskset_id kwarg. - # XXX previously the "results" arg was named "subtasks". - if 'subtasks' in kwargs: - results = kwargs['subtasks'] - GroupResult.__init__(self, taskset_id, results, **kwargs) - - def itersubtasks(self): - """Deprecated. Use ``iter(self.results)`` instead.""" - return iter(self.results) - - @property - def total(self): - """Deprecated: Use ``len(r)``.""" - return len(self) - - @property - def taskset_id(self): - """compat alias to :attr:`self.id`""" - return self.id - - @taskset_id.setter # noqa - def taskset_id(self, id): - self.id = id - - class EagerResult(AsyncResult): """Result that we know has already been executed.""" task_name = None diff --git a/celery/task/__init__.py b/celery/task/__init__.py index 4ab1a2feb..3d820166f 100644 --- a/celery/task/__init__.py +++ b/celery/task/__init__.py @@ -17,7 +17,7 @@ __all__ = [ 'BaseTask', 'Task', 'PeriodicTask', 'task', 'periodic_task', - 'group', 'chord', 'subtask', 'TaskSet', + 'group', 'chord', 'subtask', ] @@ -29,7 +29,6 @@ # they contain. from celery.canvas import group, chord, subtask from .base import BaseTask, Task, PeriodicTask, task, periodic_task - from .sets import TaskSet class module(LazyModule): @@ -44,7 +43,6 @@ def __call__(self, *args, **kwargs): 'celery.task.base': ['BaseTask', 'Task', 'PeriodicTask', 'task', 'periodic_task'], 'celery.canvas': ['group', 'chord', 'subtask'], - 'celery.task.sets': ['TaskSet'], }, base=module, __package__='celery.task', diff --git a/celery/task/sets.py b/celery/task/sets.py deleted file mode 100644 index 2ea0012c3..000000000 --- a/celery/task/sets.py +++ /dev/null @@ -1,88 +0,0 @@ -# -*- coding: utf-8 -*- -""" - celery.task.sets - ~~~~~~~~~~~~~~~~ - - Old ``group`` implementation, this module should - not be used anymore use :func:`celery.group` instead. - -""" -from __future__ import absolute_import - -from celery._state import get_current_worker_task -from celery.app import app_or_default -from celery.canvas import maybe_signature # noqa -from celery.utils import uuid, warn_deprecated - -from celery.canvas import subtask # noqa - -warn_deprecated( - 'celery.task.sets and TaskSet', removal='4.0', - alternative="""\ -Please use "group" instead (see the Canvas section in the userguide)\ -""") - - -class TaskSet(list): - """A task containing several subtasks, making it possible - to track how many, or when all of the tasks have been completed. - - :param tasks: A list of :class:`subtask` instances. - - Example:: - - >>> from myproj.tasks import refresh_feed - - >>> urls = ('http://cnn.com/rss', 'http://bbc.co.uk/rss') - >>> s = TaskSet(refresh_feed.s(url) for url in urls) - >>> taskset_result = s.apply_async() - >>> list_of_return_values = taskset_result.join() # *expensive* - - """ - app = None - - def __init__(self, tasks=None, app=None, Publisher=None): - self.app = app_or_default(app or self.app) - super(TaskSet, self).__init__( - maybe_signature(t, app=self.app) for t in tasks or [] - ) - self.Publisher = Publisher or self.app.amqp.Producer - self.total = len(self) # XXX compat - - def apply_async(self, connection=None, publisher=None, taskset_id=None): - """Apply TaskSet.""" - app = self.app - - if app.conf.task_always_eager: - return self.apply(taskset_id=taskset_id) - - with app.connection_or_acquire(connection) as conn: - setid = taskset_id or uuid() - pub = publisher or self.Publisher(conn) - results = self._async_results(setid, pub) - - result = app.TaskSetResult(setid, results) - parent = get_current_worker_task() - if parent: - parent.add_trail(result) - return result - - def _async_results(self, taskset_id, publisher): - return [task.apply_async(taskset_id=taskset_id, publisher=publisher) - for task in self] - - def apply(self, taskset_id=None): - """Applies the TaskSet locally by blocking until all tasks return.""" - setid = taskset_id or uuid() - return self.app.TaskSetResult(setid, self._sync_results(setid)) - - def _sync_results(self, taskset_id): - return [task.apply(taskset_id=taskset_id) for task in self] - - @property - def tasks(self): - return self - - @tasks.setter # noqa - def tasks(self, tasks): - self[:] = tasks diff --git a/celery/tests/compat_modules/test_sets.py b/celery/tests/compat_modules/test_sets.py deleted file mode 100644 index 4869716cb..000000000 --- a/celery/tests/compat_modules/test_sets.py +++ /dev/null @@ -1,245 +0,0 @@ -from __future__ import absolute_import - -import warnings - -from kombu.utils import json - -from celery import uuid -from celery.result import TaskSetResult -from celery.task import Task -from celery.canvas import Signature - -from celery.tests.tasks.test_result import make_mock_group -from celery.tests.case import AppCase, Mock, patch - - -class SetsCase(AppCase): - - def setup(self): - with warnings.catch_warnings(record=True): - from celery.task import sets - self.sets = sets - self.subtask = sets.subtask - self.TaskSet = sets.TaskSet - - class MockTask(Task): - app = self.app - name = 'tasks.add' - - def run(self, x, y, **kwargs): - return x + y - - @classmethod - def apply_async(cls, args, kwargs, **options): - return (args, kwargs, options) - - @classmethod - def apply(cls, args, kwargs, **options): - return (args, kwargs, options) - self.MockTask = MockTask - - -class test_TaskSetResult(AppCase): - - def setup(self): - self.size = 10 - self.ts = TaskSetResult(uuid(), make_mock_group(self.app, self.size)) - - def test_total(self): - self.assertEqual(self.ts.total, self.size) - - def test_compat_properties(self): - self.assertEqual(self.ts.taskset_id, self.ts.id) - self.ts.taskset_id = 'foo' - self.assertEqual(self.ts.taskset_id, 'foo') - - def test_compat_subtasks_kwarg(self): - x = TaskSetResult(uuid(), subtasks=[1, 2, 3]) - self.assertEqual(x.results, [1, 2, 3]) - - def test_itersubtasks(self): - it = self.ts.itersubtasks() - - for i, t in enumerate(it): - self.assertEqual(t.get(), i) - - -class test_App(AppCase): - - def test_TaskSet(self): - with warnings.catch_warnings(record=True): - ts = self.app.TaskSet() - self.assertListEqual(ts.tasks, []) - self.assertIs(ts.app, self.app) - - -class test_subtask(SetsCase): - - def test_behaves_like_type(self): - s = self.subtask('tasks.add', (2, 2), {'cache': True}, - {'routing_key': 'CPU-bound'}) - self.assertDictEqual(self.subtask(s), s) - - def test_task_argument_can_be_task_cls(self): - s = self.subtask(self.MockTask, (2, 2)) - self.assertEqual(s.task, self.MockTask.name) - - def test_apply_async(self): - s = self.MockTask.subtask( - (2, 2), {'cache': True}, {'routing_key': 'CPU-bound'}, - ) - args, kwargs, options = s.apply_async() - self.assertTupleEqual(args, (2, 2)) - self.assertDictEqual(kwargs, {'cache': True}) - self.assertDictEqual(options, {'routing_key': 'CPU-bound'}) - - def test_delay_argmerge(self): - s = self.MockTask.subtask( - (2,), {'cache': True}, {'routing_key': 'CPU-bound'}, - ) - args, kwargs, options = s.delay(10, cache=False, other='foo') - self.assertTupleEqual(args, (10, 2)) - self.assertDictEqual(kwargs, {'cache': False, 'other': 'foo'}) - self.assertDictEqual(options, {'routing_key': 'CPU-bound'}) - - def test_apply_async_argmerge(self): - s = self.MockTask.subtask( - (2,), {'cache': True}, {'routing_key': 'CPU-bound'}, - ) - args, kwargs, options = s.apply_async((10,), - {'cache': False, 'other': 'foo'}, - routing_key='IO-bound', - exchange='fast') - - self.assertTupleEqual(args, (10, 2)) - self.assertDictEqual(kwargs, {'cache': False, 'other': 'foo'}) - self.assertDictEqual(options, {'routing_key': 'IO-bound', - 'exchange': 'fast'}) - - def test_apply_argmerge(self): - s = self.MockTask.subtask( - (2,), {'cache': True}, {'routing_key': 'CPU-bound'}, - ) - args, kwargs, options = s.apply((10,), - {'cache': False, 'other': 'foo'}, - routing_key='IO-bound', - exchange='fast') - - self.assertTupleEqual(args, (10, 2)) - self.assertDictEqual(kwargs, {'cache': False, 'other': 'foo'}) - self.assertDictEqual( - options, {'routing_key': 'IO-bound', 'exchange': 'fast'}, - ) - - def test_is_JSON_serializable(self): - s = self.MockTask.subtask( - (2,), {'cache': True}, {'routing_key': 'CPU-bound'}, - ) - # tuples are not preserved, but this doesn't matter. - s.args = list(s.args) - self.assertEqual(s, self.subtask(json.loads(json.dumps(s)))) - - def test_repr(self): - s = self.MockTask.subtask((2,), {'cache': True}) - self.assertIn('2', repr(s)) - self.assertIn('cache=True', repr(s)) - - def test_reduce(self): - s = self.MockTask.subtask((2,), {'cache': True}) - cls, args = s.__reduce__() - self.assertDictEqual(dict(cls(*args)), dict(s)) - - -class test_TaskSet(SetsCase): - - def test_task_arg_can_be_iterable__compat(self): - ts = self.TaskSet([self.MockTask.subtask((i, i)) - for i in (2, 4, 8)], app=self.app) - self.assertEqual(len(ts), 3) - - def test_respects_ALWAYS_EAGER(self): - app = self.app - - class MockTaskSet(self.TaskSet): - applied = 0 - - def apply(self, *args, **kwargs): - self.applied += 1 - - ts = MockTaskSet( - [self.MockTask.subtask((i, i)) for i in (2, 4, 8)], - app=self.app, - ) - app.conf.task_always_eager = True - ts.apply_async() - self.assertEqual(ts.applied, 1) - app.conf.task_always_eager = False - - with patch('celery.task.sets.get_current_worker_task') as gwt: - parent = gwt.return_value = Mock() - ts.apply_async() - self.assertTrue(parent.add_trail.called) - - def test_apply_async(self): - applied = [0] - - class mocksubtask(Signature): - - def apply_async(self, *args, **kwargs): - applied[0] += 1 - - ts = self.TaskSet([mocksubtask(self.MockTask, (i, i)) - for i in (2, 4, 8)], app=self.app) - ts.apply_async() - self.assertEqual(applied[0], 3) - - class Publisher(object): - - def send(self, *args, **kwargs): - pass - - ts.apply_async(publisher=Publisher()) - - # setting current_task - - @self.app.task(shared=False) - def xyz(): - pass - - from celery._state import _task_stack - xyz.push_request() - _task_stack.push(xyz) - try: - ts.apply_async(publisher=Publisher()) - finally: - _task_stack.pop() - xyz.pop_request() - - def test_apply(self): - - applied = [0] - - class mocksubtask(Signature): - - def apply(self, *args, **kwargs): - applied[0] += 1 - - ts = self.TaskSet([mocksubtask(self.MockTask, (i, i)) - for i in (2, 4, 8)], app=self.app) - ts.apply() - self.assertEqual(applied[0], 3) - - def test_set_app(self): - ts = self.TaskSet([], app=self.app) - ts.app = 42 - self.assertEqual(ts.app, 42) - - def test_set_tasks(self): - ts = self.TaskSet([], app=self.app) - ts.tasks = [1, 2, 3] - self.assertEqual(ts, [1, 2, 3]) - - def test_set_Publisher(self): - ts = self.TaskSet([], app=self.app) - ts.Publisher = 42 - self.assertEqual(ts.Publisher, 42) diff --git a/celery/tests/tasks/test_result.py b/celery/tests/tasks/test_result.py index 590b0f494..067998885 100644 --- a/celery/tests/tasks/test_result.py +++ b/celery/tests/tasks/test_result.py @@ -8,7 +8,6 @@ from celery.result import ( AsyncResult, EagerResult, - TaskSetResult, result_from_tuple, ) from celery.utils import uuid @@ -409,31 +408,6 @@ def get_many(self, *args, **kwargs): for i, id in enumerate(self.ids)) -class test_TaskSetResult(AppCase): - - def setup(self): - self.size = 10 - self.ts = TaskSetResult(uuid(), make_mock_group(self.app, self.size)) - - def test_total(self): - self.assertEqual(self.ts.total, self.size) - - def test_compat_properties(self): - self.assertEqual(self.ts.taskset_id, self.ts.id) - self.ts.taskset_id = 'foo' - self.assertEqual(self.ts.taskset_id, 'foo') - - def test_compat_subtasks_kwarg(self): - x = TaskSetResult(uuid(), subtasks=[1, 2, 3]) - self.assertEqual(x.results, [1, 2, 3]) - - def test_itersubtasks(self): - it = self.ts.itersubtasks() - - for i, t in enumerate(it): - self.assertEqual(t.get(), i) - - class test_GroupResult(AppCase): def setup(self): From 0cc715aee6953c8cbc229bb56e61bead8218a27d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 Oct 2015 18:06:18 -0700 Subject: [PATCH 0759/1103] Fixes docs --- celery/app/utils.py | 1 - celery/backends/new_cassandra.py | 2 +- docs/configuration.rst | 12 +- docs/internals/deprecation.rst | 226 +++++++++++++++---------------- docs/reference/celery.rst | 12 +- 5 files changed, 126 insertions(+), 127 deletions(-) diff --git a/celery/app/utils.py b/celery/app/utils.py index 9a308cb0c..1775e94a5 100644 --- a/celery/app/utils.py +++ b/celery/app/utils.py @@ -188,7 +188,6 @@ def detect_settings(conf, preconf={}, ignore_keys=set(), prefix=None, # Majority of the settings are old. info, left = _old_settings_info, is_in_new elif is_in_old: - print('IS IN OLD: %r' % (is_in_old, )) # have old setting names, or a majority of the names are old. info, left = _old_settings_info, is_in_new if is_in_new and len(is_in_new) > len(is_in_old): diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index 48079e02f..8a2920b76 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -1,7 +1,7 @@ # -* coding: utf-8 -*- """ celery.backends.new_cassandra - ~~~~~~~~~~~~~~~~~~~~~~~~~ + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Apache Cassandra result store backend using DataStax driver diff --git a/docs/configuration.rst b/docs/configuration.rst index 1f76da414..301f3eba4 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -374,12 +374,8 @@ Can be one of the following: Use `MongoDB`_ to store the results. See :ref:`conf-mongodb-result-backend`. -* cassandra - Use `Cassandra`_ to store the results. - See :ref:`conf-cassandra-result-backend`. - * new_cassandra - Use `new_cassandra`_ to store the results, using newer database driver than _cassandra_. + Use `Cassandra`_ to store the results, using newer database driver than _cassandra_. See :ref:`conf-new_cassandra-result-backend`. * ironcache @@ -564,8 +560,6 @@ you to customize the table names: RPC backend settings -------------------- -.. _conf-amqp-result-backend: - .. setting:: result_persistent result_persistent @@ -1012,6 +1006,8 @@ The URL is formed out of the following parts: The default container the CouchDB server is writing to. Defaults to ``default``. +.. _conf-amqp-result-backend: + AMQP backend settings --------------------- @@ -1045,8 +1041,6 @@ result_exchange_type The exchange type of the result exchange. Default is to use a `direct` exchange. -.. setting:: result_persistent - result_persistent ~~~~~~~~~~~~~~~~~ diff --git a/docs/internals/deprecation.rst b/docs/internals/deprecation.rst index 0f16b29ac..e661a3c26 100644 --- a/docs/internals/deprecation.rst +++ b/docs/internals/deprecation.rst @@ -31,67 +31,6 @@ Removals for version 4.0 >>> from celery.result import result_from_tuple >>> result = result_from_tuple(tup) -.. _deprecations-v4.0: - -Removals for version 5.0 -======================== - -Old Task API ------------- - -.. _deprecate-compat-task-modules: - -Compat Task Modules -~~~~~~~~~~~~~~~~~~~ - -- Module ``celery.decorators`` will be removed: - - Which means you need to change:: - - from celery.decorators import task - -Into:: - - from celery import task - -- Module ``celery.task`` *may* be removed (not decided) - - This means you should change:: - - from celery.task import task - - into:: - - from celery import task - - -- and:: - - from celery.task import Task - - into:: - - from celery import Task - - -Note that the new :class:`~celery.Task` class no longer -uses classmethods for these methods: - - - delay - - apply_async - - retry - - apply - - AsyncResult - - subtask - -This also means that you can't call these methods directly -on the class, but have to instantiate the task first:: - - >>> MyTask.delay() # NO LONGER WORKS - - - >>> MyTask().delay() # WORKS! - - TaskSet ~~~~~~~ @@ -132,21 +71,6 @@ should be rewritten into:: def add(self, x, y): print("My task id is {0.request.id}".format(self)) - -Task attributes ---------------- - -The task attributes: - -- ``queue`` -- ``exchange`` -- ``exchange_type`` -- ``routing_key`` -- ``delivery_mode`` -- ``priority`` - -is deprecated and must be set by :setting:`task_routes` instead. - :mod:`celery.result` -------------------- @@ -182,43 +106,6 @@ The :signal:`task_sent` signal will be removed in version 4.0. Please use the :signal:`before_task_publish` and :signal:`after_task_publush` signals instead. - -Modules to Remove ------------------ - -- ``celery.execute`` - - This module only contains ``send_task``, which must be replaced with - :attr:`@send_task` instead. - -- ``celery.decorators`` - - See :ref:`deprecate-compat-task-modules` - -- ``celery.log`` - - Use :attr:`@log` instead. - -- ``celery.messaging`` - - Use :attr:`@amqp` instead. - -- ``celery.registry`` - - Use :mod:`celery.app.registry` instead. - -- ``celery.task.control`` - - Use :attr:`@control` instead. - -- ``celery.task.schedules`` - - Use :mod:`celery.schedules` instead. - -- ``celery.task.chords`` - - Use :func:`celery.chord` instead. - Settings -------- @@ -277,6 +164,119 @@ Other Settings ===================================== ===================================== + +.. _deprecations-v5.0: + +Removals for version 5.0 +======================== + +Old Task API +------------ + +.. _deprecate-compat-task-modules: + +Compat Task Modules +~~~~~~~~~~~~~~~~~~~ + +- Module ``celery.decorators`` will be removed: + + Which means you need to change:: + + from celery.decorators import task + +Into:: + + from celery import task + +- Module ``celery.task`` *may* be removed (not decided) + + This means you should change:: + + from celery.task import task + + into:: + + from celery import task + + -- and:: + + from celery.task import Task + + into:: + + from celery import Task + + +Note that the new :class:`~celery.Task` class no longer +uses classmethods for these methods: + + - delay + - apply_async + - retry + - apply + - AsyncResult + - subtask + +This also means that you can't call these methods directly +on the class, but have to instantiate the task first:: + + >>> MyTask.delay() # NO LONGER WORKS + + + >>> MyTask().delay() # WORKS! + + +Task attributes +--------------- + +The task attributes: + +- ``queue`` +- ``exchange`` +- ``exchange_type`` +- ``routing_key`` +- ``delivery_mode`` +- ``priority`` + +is deprecated and must be set by :setting:`task_routes` instead. + + +Modules to Remove +----------------- + +- ``celery.execute`` + + This module only contains ``send_task``, which must be replaced with + :attr:`@send_task` instead. + +- ``celery.decorators`` + + See :ref:`deprecate-compat-task-modules` + +- ``celery.log`` + + Use :attr:`@log` instead. + +- ``celery.messaging`` + + Use :attr:`@amqp` instead. + +- ``celery.registry`` + + Use :mod:`celery.app.registry` instead. + +- ``celery.task.control`` + + Use :attr:`@control` instead. + +- ``celery.task.schedules`` + + Use :mod:`celery.schedules` instead. + +- ``celery.task.chords`` + + Use :func:`celery.chord` instead. + .. _deprecations-v2.0: Removals for version 2.0 diff --git a/docs/reference/celery.rst b/docs/reference/celery.rst index d8e8626b6..4890bfdce 100644 --- a/docs/reference/celery.rst +++ b/docs/reference/celery.rst @@ -102,11 +102,17 @@ and creating Celery applications. .. automethod:: finalize - .. autodata:: on_configure + .. data:: on_configure - .. autodata:: on_after_configure + Signal sent when app is loading configuration. - .. autodata:: on_after_finalize + .. data:: on_after_configure + + Signal sent after app has prepared the configuration. + + .. data:: on_after_finalize + + Signal sent after app has been finalized. Canvas primitives ----------------- From 91457c48da7db86350fa1a4cda7b7505916351e7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 Oct 2015 18:09:45 -0700 Subject: [PATCH 0760/1103] Removes accept _magic_kwargs attribute --- celery/app/base.py | 3 --- celery/app/task.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 440c6d0ff..7a88c140e 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -187,9 +187,6 @@ class Celery(object): #: Signal sent after app has been finalized. on_after_finalize = None - #: ignored - accept_magic_kwargs = False - def __init__(self, main=None, loader=None, backend=None, amqp=None, events=None, log=None, control=None, set_as_current=True, tasks=None, broker=None, include=None, diff --git a/celery/app/task.py b/celery/app/task.py index 1d1baa4c9..3c1365f68 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -265,9 +265,6 @@ class Task(object): ('store_errors_even_if_ignored', 'task_store_errors_even_if_ignored'), ) - #: ignored - accept_magic_kwargs = False - _backend = None # set by backend property. __bound__ = False From 77b16348c435e6bb9ef20121c0ee24ad7ecf0ad2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 Oct 2015 18:10:20 -0700 Subject: [PATCH 0761/1103] Removes BaseAsyncResult as per deprecation timeline --- celery/result.py | 1 - 1 file changed, 1 deletion(-) diff --git a/celery/result.py b/celery/result.py index 25cd831fb..edbf5aa9a 100644 --- a/celery/result.py +++ b/celery/result.py @@ -403,7 +403,6 @@ def task_id(self): @task_id.setter # noqa def task_id(self, id): self.id = id -BaseAsyncResult = AsyncResult # for backwards compatibility. class ResultSet(ResultBase): From fa4514bb328060cd1982b5933ef233c842d9c721 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 Oct 2015 18:13:29 -0700 Subject: [PATCH 0762/1103] Removes .loaders.current_loader + .load_settings() as per deprecation timeline --- celery/loaders/__init__.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/celery/loaders/__init__.py b/celery/loaders/__init__.py index 2a39ba2ab..ad6d766c9 100644 --- a/celery/loaders/__init__.py +++ b/celery/loaders/__init__.py @@ -9,8 +9,6 @@ """ from __future__ import absolute_import -from celery._state import current_app -from celery.utils import deprecated from celery.utils.imports import symbol_by_name, import_from_cwd __all__ = ['get_loader_cls'] @@ -23,15 +21,3 @@ def get_loader_cls(loader): """Get loader class by name/alias""" return symbol_by_name(loader, LOADER_ALIASES, imp=import_from_cwd) - - -@deprecated(deprecation=2.5, removal=4.0, - alternative='celery.current_app.loader') -def current_loader(): - return current_app.loader - - -@deprecated(deprecation=2.5, removal=4.0, - alternative='celery.current_app.conf') -def load_settings(): - return current_app.conf From 5bd0ec59c236fda2ffbd6116ae8fef4d626ec559 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 Oct 2015 18:16:04 -0700 Subject: [PATCH 0763/1103] Cosmetics --- celery/result.py | 2 +- docs/internals/deprecation.rst | 125 ++++++++++++++++++--------------- 2 files changed, 68 insertions(+), 59 deletions(-) diff --git a/celery/result.py b/celery/result.py index edbf5aa9a..83b4c91d4 100644 --- a/celery/result.py +++ b/celery/result.py @@ -858,7 +858,7 @@ def get(self, timeout=None, propagate=True, **kwargs): if propagate: raise self.result return self.result - wait = get + wait = get # XXX Compat (remove 5.0) def forget(self): pass diff --git a/docs/internals/deprecation.rst b/docs/internals/deprecation.rst index e661a3c26..817aa9aa6 100644 --- a/docs/internals/deprecation.rst +++ b/docs/internals/deprecation.rst @@ -82,15 +82,6 @@ should be rewritten into:: - ``TaskSetResult.taskset_id`` -> ``GroupResult.id`` -Apply to: :class:`~celery.result.AsyncResult`, -:class:`~celery.result.EagerResult`:: - -- ``Result.wait()`` -> ``Result.get()`` - -- ``Result.task_id()`` -> ``Result.id`` - -- ``Result.status`` -> ``Result.state``. - :mod:`celery.loader` -------------------- @@ -99,46 +90,9 @@ Apply to: :class:`~celery.result.AsyncResult`, - ``load_settings()`` -> ``current_app.conf`` -Task_sent signal ----------------- - -The :signal:`task_sent` signal will be removed in version 4.0. -Please use the :signal:`before_task_publish` and :signal:`after_task_publush` -signals instead. - Settings -------- -``BROKER`` Settings -~~~~~~~~~~~~~~~~~~~ - -===================================== ===================================== -**Setting name** **Replace with** -===================================== ===================================== -``BROKER_HOST`` :setting:`broker_url` -``BROKER_PORT`` :setting:`broker_url` -``BROKER_USER`` :setting:`broker_url` -``BROKER_PASSWORD`` :setting:`broker_url` -``BROKER_VHOST`` :setting:`broker_url` -===================================== ===================================== - - -``REDIS`` Result Backend Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -===================================== ===================================== -**Setting name** **Replace with** -===================================== ===================================== -``CELERY_REDIS_HOST`` :setting:`result_backend` -``CELERY_REDIS_PORT`` :setting:`result_backend` -``CELERY_REDIS_DB`` :setting:`result_backend` -``CELERY_REDIS_PASSWORD`` :setting:`result_backend` -``REDIS_HOST`` :setting:`result_backend` -``REDIS_PORT`` :setting:`result_backend` -``REDIS_DB`` :setting:`result_backend` -``REDIS_PASSWORD`` :setting:`result_backend` -===================================== ===================================== - Logging Settings ~~~~~~~~~~~~~~~~ @@ -153,18 +107,6 @@ Logging Settings ``CELERYMON_LOG_FILE`` :option:`--loglevel`` ===================================== ===================================== -Other Settings -~~~~~~~~~~~~~~ - -===================================== ===================================== -**Setting name** **Replace with** -===================================== ===================================== -``CELERY_TASK_ERROR_WITELIST`` Annotate ``Task.ErrorMail`` -``CELERY_AMQP_TASK_RESULT_EXPIRES`` :setting:`result_expires` -===================================== ===================================== - - - .. _deprecations-v5.0: Removals for version 5.0 @@ -277,6 +219,73 @@ Modules to Remove Use :func:`celery.chord` instead. +Settings +-------- + +``BROKER`` Settings +~~~~~~~~~~~~~~~~~~~ + +===================================== ===================================== +**Setting name** **Replace with** +===================================== ===================================== +``BROKER_HOST`` :setting:`broker_url` +``BROKER_PORT`` :setting:`broker_url` +``BROKER_USER`` :setting:`broker_url` +``BROKER_PASSWORD`` :setting:`broker_url` +``BROKER_VHOST`` :setting:`broker_url` +===================================== ===================================== + +``REDIS`` Result Backend Settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +===================================== ===================================== +**Setting name** **Replace with** +===================================== ===================================== +``CELERY_REDIS_HOST`` :setting:`result_backend` +``CELERY_REDIS_PORT`` :setting:`result_backend` +``CELERY_REDIS_DB`` :setting:`result_backend` +``CELERY_REDIS_PASSWORD`` :setting:`result_backend` +``REDIS_HOST`` :setting:`result_backend` +``REDIS_PORT`` :setting:`result_backend` +``REDIS_DB`` :setting:`result_backend` +``REDIS_PASSWORD`` :setting:`result_backend` +===================================== ===================================== + + +Task_sent signal +---------------- + +The :signal:`task_sent` signal will be removed in version 4.0. +Please use the :signal:`before_task_publish` and :signal:`after_task_publush` +signals instead. + +Result +------ + +Apply to: :class:`~celery.result.AsyncResult`, +:class:`~celery.result.EagerResult`:: + +- ``Result.wait()`` -> ``Result.get()`` + +- ``Result.task_id()`` -> ``Result.id`` + +- ``Result.status`` -> ``Result.state``. + +.. _deprecations-v3.1: + + +Settings +~~~~~~~~ + +===================================== ===================================== +**Setting name** **Replace with** +===================================== ===================================== +``CELERY_TASK_ERROR_WITELIST`` Annotate ``Task.ErrorMail`` +``CELERY_AMQP_TASK_RESULT_EXPIRES`` :setting:`result_expires` +===================================== ===================================== + + + .. _deprecations-v2.0: Removals for version 2.0 From 3d09eeed0b316bef88b85cb0f9dd566ae4ba89ad Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 Oct 2015 18:18:09 -0700 Subject: [PATCH 0764/1103] flakes --- celery/tests/app/test_loaders.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/celery/tests/app/test_loaders.py b/celery/tests/app/test_loaders.py index 99812fb8c..9d80e08f8 100644 --- a/celery/tests/app/test_loaders.py +++ b/celery/tests/app/test_loaders.py @@ -5,18 +5,14 @@ import warnings from celery import loaders -from celery.exceptions import ( - NotConfigured, -) +from celery.exceptions import NotConfigured from celery.loaders import base from celery.loaders import default from celery.loaders.app import AppLoader from celery.utils.imports import NotAPackage from celery.utils.mail import SendmailWarning -from celery.tests.case import ( - AppCase, Case, Mock, depends_on_current_app, patch, with_environ, -) +from celery.tests.case import AppCase, Case, Mock, patch, with_environ class DummyLoader(base.BaseLoader): @@ -31,16 +27,6 @@ def test_get_loader_cls(self): self.assertEqual(loaders.get_loader_cls('default'), default.Loader) - @depends_on_current_app - def test_current_loader(self): - with self.assertPendingDeprecation(): - self.assertIs(loaders.current_loader(), self.app.loader) - - @depends_on_current_app - def test_load_settings(self): - with self.assertPendingDeprecation(): - self.assertIs(loaders.load_settings(), self.app.conf) - class test_LoaderBase(AppCase): message_options = {'subject': 'Subject', From e38e2ea4060c4c815ef5cb1732f9ca209b15525f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 Oct 2015 18:37:57 -0700 Subject: [PATCH 0765/1103] Fixed bug masked by PYTHONHASHSEED --- celery/app/defaults.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 0730a551f..a150870a9 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -49,7 +49,8 @@ def Namespace(__old__=None, **options): if __old__ is not None: for opt in values(options): - opt.old = opt.old | __old__ + if not opt.old: + opt.old = __old__ return options From 884ef24919cfa9cfb6af0eeafc420da7c92e641c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 Oct 2015 18:42:10 -0700 Subject: [PATCH 0766/1103] Master is now 4.0.0b1 and in feature-freeze --- Changelog | 4 +++- README.rst | 2 +- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Changelog b/Changelog index 183a24ffb..4da5b706f 100644 --- a/Changelog +++ b/Changelog @@ -11,8 +11,10 @@ an overview of what's new in Celery 4.0. .. _version-4.0.0: 4.0.0 -======= +===== :release-date: TBA +:status: *FROZEN* +:branch: master :release-by: See :ref:`whatsnew-4.0`. diff --git a/README.rst b/README.rst index 38671fab0..d79d2e996 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ .. image:: http://cloud.github.com/downloads/celery/celery/celery_128.png -:Version: 4.0.0a1 (0today8) +:Version: 4.0.0b1 (0today8) :Web: http://celeryproject.org/ :Download: http://pypi.python.org/pypi/celery/ :Source: http://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index 48ac71dd7..260a78738 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -18,7 +18,7 @@ ) SERIES = '0today8' -VERSION = version_info_t(4, 0, 0, 'a1', '') +VERSION = version_info_t(4, 0, 0, 'b1', '') __version__ = '{0.major}.{0.minor}.{0.micro}{0.releaselevel}'.format(VERSION) __author__ = 'Ask Solem' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 16e2d2b59..7986c52a4 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 4.0.0a1 (0today8) +:Version: 4.0.0b1 (0today8) :Web: http://celeryproject.org/ :Download: http://pypi.python.org/pypi/celery/ :Source: http://github.com/celery/celery/ From b5d6054d0d065e910f6b9e88dac49fc33d80f2e7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 29 Oct 2015 18:53:50 -0700 Subject: [PATCH 0767/1103] Fixes PyPy tests --- celery/_state.py | 4 ++++ celery/app/base.py | 4 +++- celery/tests/app/test_app.py | 16 +++++++++------- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/celery/_state.py b/celery/_state.py index 1fec88973..5047182e3 100644 --- a/celery/_state.py +++ b/celery/_state.py @@ -130,5 +130,9 @@ def _register_app(app): _apps.add(app) +def _deregister_app(app): + _apps.discard(app) + + def _get_active_apps(): return _apps diff --git a/celery/app/base.py b/celery/app/base.py index 7a88c140e..b0e7663db 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -29,7 +29,8 @@ from celery import signals from celery._state import ( _task_stack, get_current_app, _set_current_app, set_default_app, - _register_app, get_current_worker_task, connect_on_app_finalize, + _register_app, _deregister_app, + get_current_worker_task, connect_on_app_finalize, _announce_app_finalized, ) from celery.datastructures import AttributeDictMixin @@ -286,6 +287,7 @@ def close(self): pass """ self._maybe_close_pool() + _deregister_app(self) def on_init(self): """Optional callback called at init.""" diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index b04a3f1a3..85f0b3eb6 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -123,7 +123,6 @@ def test_with_config_source(self): def test_task_windows_execv(self): prev, _appbase._EXECV = _appbase._EXECV, True try: - @self.app.task(shared=False) def foo(): pass @@ -286,13 +285,16 @@ def test_pending_configuration__iter(self): self.assertTrue(app.configured) def test_pending_configuration__raises_ImproperlyConfigured(self): - with self.Celery() as app: + with self.Celery(set_as_current=False) as app: app.conf.worker_agent = 'foo://bar' app.conf.task_default_delivery_mode = 44 - app.conf.CELERY_ALWAYS_EAGER = True + app.conf.CELERY_ALWAYS_EAGER = 5 with self.assertRaises(ImproperlyConfigured): app.finalize() + with self.Celery() as app: + self.assertFalse(self.app.conf.task_always_eager) + def test_repr(self): self.assertTrue(repr(self.app)) @@ -509,12 +511,12 @@ class Config(object): def test_config_from_object__supports_old_names(self): class Config(object): - task_always_eager = 44 + task_always_eager = 45 task_default_delivery_mode = 301 self.app.config_from_object(Config()) - self.assertEqual(self.app.conf.CELERY_ALWAYS_EAGER, 44) - self.assertEqual(self.app.conf.task_always_eager, 44) + self.assertEqual(self.app.conf.CELERY_ALWAYS_EAGER, 45) + self.assertEqual(self.app.conf.task_always_eager, 45) self.assertEqual(self.app.conf.CELERY_DEFAULT_DELIVERY_MODE, 301) self.assertEqual(self.app.conf.task_default_delivery_mode, 301) self.assertEqual(self.app.conf.task_default_routing_key, 'testcelery') @@ -555,7 +557,7 @@ class Config(object): def test_config_from_object__mixing_old_and_new(self): class Config(object): - CELERY_ALWAYS_EAGER = 44 + CELERY_ALWAYS_EAGER = 46 CELERYD_AGENT = 'foo:Agent' CELERYD_CONSUMER = 'foo:Consumer' CELERYBEAT_SCHEDULE = '/foo/schedule' From be9d3df3ba1169c677abd4400386df1fc9abbfd5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 30 Oct 2015 12:38:16 -0700 Subject: [PATCH 0768/1103] [Stress] Added test for unicode printing/logging tasks. --- funtests/stress/stress/app.py | 1 + funtests/stress/stress/suite.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/funtests/stress/stress/app.py b/funtests/stress/stress/app.py index 9ecd289fc..658d48e5d 100644 --- a/funtests/stress/stress/app.py +++ b/funtests/stress/stress/app.py @@ -113,6 +113,7 @@ def retries(self): @app.task def print_unicode(): + logger.warning('hå它 valmuefrø') print('hiöäüß') diff --git a/funtests/stress/stress/suite.py b/funtests/stress/stress/suite.py index 2556ff16d..3902c82e4 100755 --- a/funtests/stress/stress/suite.py +++ b/funtests/stress/stress/suite.py @@ -19,7 +19,7 @@ from .app import ( marker, _marker, add, any_, exiting, kill, sleeping, - sleeping_ignore_limits, any_returning + sleeping_ignore_limits, any_returning, print_unicode, ) from .data import BIG, SMALL from .fbi import FBI @@ -267,6 +267,11 @@ def manyshort(self): self.join(group(add.s(i, i) for i in range(1000))(), timeout=10, propagate=True) + @testcase('all', 'green') + def unicodetask(self): + self.join(group(print_unicode.s() for _ in range(5))(), + timeout=1, propagate=True) + @testcase('all') def always_timeout(self): self.join( From 2d8b83723aae44ca26ffd96ae140393a8a330fbb Mon Sep 17 00:00:00 2001 From: Dennis Brakhane Date: Fri, 30 Oct 2015 20:05:11 +0100 Subject: [PATCH 0769/1103] Fix LRUCache.update for Python 3.5 Python 3.5's OrderedDict does not allow mutation while it is being iterated over. This breaks "update" if it is called with a dict larger than the maximum size. This commit changes the code to a version that does not iterate over the dict, and should also be a little bit faster. Closes #2897 --- CONTRIBUTORS.txt | 1 + celery/tests/utils/test_functional.py | 5 +++++ celery/utils/functional.py | 7 +++---- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index b62f1915c..1d4f33e49 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -197,3 +197,4 @@ Gerald Manipon, 2015/10/19 Krzysztof Bujniewicz, 2015/10/21 Sukrit Khera, 2015/10/26 Dave Smith, 2015/10/27 +Dennis Brakhane, 2015/10/30 diff --git a/celery/tests/utils/test_functional.py b/celery/tests/utils/test_functional.py index c60419d00..043646fe0 100644 --- a/celery/tests/utils/test_functional.py +++ b/celery/tests/utils/test_functional.py @@ -63,6 +63,11 @@ def test_least_recently_used(self): x[7] = 7 self.assertEqual(list(x.keys()), [3, 6, 7]) + def test_update_larger_than_cache_size(self): + x = LRUCache(2) + x.update({x: x for x in range(100)}) + self.assertEqual(list(x.keys()), [98, 99]) + def assertSafeIter(self, method, interval=0.01, size=10000): if sys.version_info >= (3, 5): raise SkipTest('Fails on Py3.5') diff --git a/celery/utils/functional.py b/celery/utils/functional.py index fbb4fc468..1af2914e5 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -20,7 +20,7 @@ from kombu.utils import cached_property from kombu.utils.functional import lazy, maybe_evaluate, is_list, maybe_list -from celery.five import UserDict, UserList, items, keys +from celery.five import UserDict, UserList, items, keys, range __all__ = ['LRUCache', 'is_list', 'maybe_list', 'memoize', 'mlazy', 'noop', 'first', 'firstmethod', 'chunks', 'padlist', 'mattrgetter', 'uniq', @@ -71,9 +71,8 @@ def update(self, *args, **kwargs): data.update(*args, **kwargs) if limit and len(data) > limit: # pop additional items in case limit exceeded - # negative overflow will lead to an empty list - for item in islice(iter(data), len(data) - limit): - data.pop(item) + for _ in range(len(data) - limit): + data.popitem(last=False) def popitem(self, last=True): with self.mutex: From 3d36b78eb4ad3d82a314556ee34c5cc8938f4665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Thu, 29 Oct 2015 13:36:08 +0100 Subject: [PATCH 0770/1103] fixes #2895 + tests --- celery/backends/new_cassandra.py | 76 ++++++++++++--------- celery/tests/backends/test_new_cassandra.py | 28 ++++++++ 2 files changed, 71 insertions(+), 33 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index 8a2920b76..a498ade07 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -140,40 +140,50 @@ def _get_connection(self, write=False): """ if self._connection is None: - self._connection = cassandra.cluster.Cluster(self.servers, - port=self.port) - self._session = self._connection.connect(self.keyspace) - - # We are forced to do concatenation below, as formatting would - # blow up on superficial %s that will be processed by Cassandra - self._write_stmt = cassandra.query.SimpleStatement( - Q_INSERT_RESULT.format( - table=self.table, expires=self.cqlexpires), - ) - self._write_stmt.consistency_level = self.write_consistency - - self._read_stmt = cassandra.query.SimpleStatement( - Q_SELECT_RESULT.format(table=self.table), - ) - self._read_stmt.consistency_level = self.read_consistency - - if write: - # Only possible writers "workers" are allowed to issue - # CREATE TABLE. This is to prevent conflicting situations - # where both task-creator and task-executor would issue it - # at the same time. - - # Anyway, if you are doing anything critical, you should - # have probably created this table in advance, in which case - # this query will be a no-op (instant fail with AlreadyExists) - self._make_stmt = cassandra.query.SimpleStatement( - Q_CREATE_RESULT_TABLE.format(table=self.table), + try: + self._connection = cassandra.cluster.Cluster(self.servers, + port=self.port) + self._session = self._connection.connect(self.keyspace) + + # We are forced to do concatenation below, as formatting would + # blow up on superficial %s that will be processed by Cassandra + self._write_stmt = cassandra.query.SimpleStatement( + Q_INSERT_RESULT.format( + table=self.table, expires=self.cqlexpires), ) - self._make_stmt.consistency_level = self.write_consistency - try: - self._session.execute(self._make_stmt) - except cassandra.AlreadyExists: - pass + self._write_stmt.consistency_level = self.write_consistency + + self._read_stmt = cassandra.query.SimpleStatement( + Q_SELECT_RESULT.format(table=self.table), + ) + self._read_stmt.consistency_level = self.read_consistency + + if write: + # Only possible writers "workers" are allowed to issue + # CREATE TABLE. This is to prevent conflicting situations + # where both task-creator and task-executor would issue it + # at the same time. + + # Anyway, if you are doing anything critical, you should + # have probably created this table in advance, in which case + # this query will be a no-op (instant fail with AlreadyExists) + self._make_stmt = cassandra.query.SimpleStatement( + Q_CREATE_RESULT_TABLE.format(table=self.table), + ) + self._make_stmt.consistency_level = self.write_consistency + try: + self._session.execute(self._make_stmt) + except cassandra.AlreadyExists: + pass + + except cassandra.OperationTimedOut: + # a heavily loaded or gone Cassandra cluster failed to respond. + # leave this class in a consistent state + self._connection = None + if self._session is not None: + self._session.shutdown() + + raise # we did fail after all - reraise def _store_result(self, task_id, result, status, traceback=None, request=None, **kwargs): diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 61b5fdfb6..678bc744d 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -102,3 +102,31 @@ def test_process_cleanup(self): self.assertIsNone(x._connection) self.assertIsNone(x._session) + + def test_timeouting_cluster(self): + """ + Tests behaviour when Cluster.connect raises cassandra.OperationTimedOut + """ + with mock_module(*CASSANDRA_MODULES): + from celery.backends import new_cassandra as mod + + class OTOExc(Exception): + pass + + class VeryFaultyCluster(object): + def __init__(self, *args, **kwargs): + pass + + def connect(self, *args, **kwargs): + raise OTOExc() + + mod.cassandra = Mock() + mod.cassandra.OperationTimedOut = OTOExc + mod.cassandra.cluster = Mock() + mod.cassandra.cluster.Cluster = VeryFaultyCluster + + x = mod.CassandraBackend(app=self.app) + + self.assertRaises(OTOExc, lambda: x._store_result('task_id', 'result', states.SUCCESS)) + self.assertIsNone(x._connection) + self.assertIsNone(x._session) From 38790108a48b40643c493cfb953f07185fe2c9ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Thu, 29 Oct 2015 13:42:58 +0100 Subject: [PATCH 0771/1103] process cleanup should be safe to invoke after connect fails --- celery/tests/backends/test_new_cassandra.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 678bc744d..6806d19f9 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -130,3 +130,6 @@ def connect(self, *args, **kwargs): self.assertRaises(OTOExc, lambda: x._store_result('task_id', 'result', states.SUCCESS)) self.assertIsNone(x._connection) self.assertIsNone(x._session) + + x.process_cleanup() # assert it doesn't raise + From cab69f9f8e909fc6c85c38c8c8b88ec26fa96b65 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 30 Oct 2015 12:45:50 -0700 Subject: [PATCH 0772/1103] Flakes for #2896 --- celery/backends/new_cassandra.py | 6 +++--- celery/tests/backends/test_new_cassandra.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index a498ade07..b25d234ce 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -164,9 +164,9 @@ def _get_connection(self, write=False): # where both task-creator and task-executor would issue it # at the same time. - # Anyway, if you are doing anything critical, you should - # have probably created this table in advance, in which case - # this query will be a no-op (instant fail with AlreadyExists) + # Anyway; if you're doing anything critical, you should + # have created this table in advance, in which case + # this query will be a no-op (AlreadyExists) self._make_stmt = cassandra.query.SimpleStatement( Q_CREATE_RESULT_TABLE.format(table=self.table), ) diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 6806d19f9..6f83db3dc 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -127,9 +127,9 @@ def connect(self, *args, **kwargs): x = mod.CassandraBackend(app=self.app) - self.assertRaises(OTOExc, lambda: x._store_result('task_id', 'result', states.SUCCESS)) + with self.assertRaises(OTOExc): + x._store_result('task_id', 'result', states.SUCCESS) self.assertIsNone(x._connection) self.assertIsNone(x._session) - x.process_cleanup() # assert it doesn't raise - + x.process_cleanup() # should not raise From c2c07b91fbba0520ba828ae857829bb5e1315c94 Mon Sep 17 00:00:00 2001 From: Kevin Harvey Date: Mon, 12 Oct 2015 13:16:22 -0500 Subject: [PATCH 0773/1103] Fixes a few grammatical and punctuation errors. --- docs/userguide/application.rst | 14 +++++++------- docs/userguide/tasks.rst | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/userguide/application.rst b/docs/userguide/application.rst index 0c2df9030..5cff4a2bc 100644 --- a/docs/userguide/application.rst +++ b/docs/userguide/application.rst @@ -12,7 +12,7 @@ The Celery library must be instantiated before use, this instance is called an application (or *app* for short). The application is thread-safe so that multiple Celery applications -with different configuration, components and tasks can co-exist in the +with different configurations, components and tasks can co-exist in the same process space. Let's create one now: @@ -32,12 +32,12 @@ current main module (``__main__``), and the memory address of the object Main Name ========= -Only one of these is important, and that is the main module name, -let's look at why that is. +Only one of these is important, and that is the main module name. +Let's look at why that is. When you send a task message in Celery, that message will not contain any source code, but only the name of the task you want to execute. -This works similarly to how host names works on the internet: every worker +This works similarly to how host names work on the internet: every worker maintains a mapping of task names to their actual functions, called the *task registry*. @@ -154,7 +154,7 @@ from a configuration object. This can be a configuration module, or any object with configuration attributes. -Note that any configuration that was previous set will be reset when +Note that any configuration that was previously set will be reset when :meth:`~@config_from_object` is called. If you want to set additional configuration you should do so after. @@ -333,7 +333,7 @@ Finalizing the object will: #. Make sure all tasks are bound to the current app. - Tasks are bound to apps so that it can read default + Tasks are bound to an app so that they can read default values from the configuration. .. _default-app: @@ -466,7 +466,7 @@ Abstract Tasks ============== All tasks created using the :meth:`~@task` decorator -will inherit from the applications base :attr:`~@Task` class. +will inherit from the application's base :attr:`~@Task` class. You can specify a different base class with the ``base`` argument: diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 0f0183189..ca074c685 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -154,7 +154,7 @@ if the module name is "tasks.py": Automatic naming and relative imports ------------------------------------- -Relative imports and automatic name generation does not go well together, +Relative imports and automatic name generation do not go well together, so if you're using relative imports you should set the name explicitly. For example if the client imports the module "myapp.tasks" as ".tasks", and @@ -682,7 +682,7 @@ General A string identifying the default serialization method to use. Defaults to the :setting:`task_serializer` - setting. Can be `pickle` `json`, `yaml`, or any custom + setting. Can be `pickle`, `json`, `yaml`, or any custom serialization methods that have been registered with :mod:`kombu.serialization.registry`. @@ -1264,7 +1264,7 @@ Handlers How it works ============ -Here comes the technical details, this part isn't something you need to know, +Here come the technical details. This part isn't something you need to know, but you may be interested. All defined tasks are listed in a registry. The registry contains @@ -1423,8 +1423,8 @@ Granularity ----------- The task granularity is the amount of computation needed by each subtask. -In general it is better to split the problem up into many small tasks, than -have a few long running tasks. +In general it is better to split the problem up into many small tasks rather +than have a few long running tasks. With smaller tasks you can process more tasks in parallel and the tasks won't run long enough to block the worker from processing other waiting tasks. @@ -1596,7 +1596,7 @@ depending on state from the current transaction*: Example ======= -Let's take a real world example; A blog where comments posted needs to be +Let's take a real world example: a blog where comments posted need to be filtered for spam. When the comment is created, the spam filter runs in the background, so the user doesn't have to wait for it to finish. From 558733c17f3b2b27e5f76347cdb0762cab3ff012 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 4 Oct 2015 22:35:44 +0000 Subject: [PATCH 0774/1103] =?UTF-8?q?[docs]=20Fixed=20double=20=E2=80=9Cim?= =?UTF-8?q?ported=E2=80=9D=20in=20workers=20guide.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/userguide/workers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index e82934710..cbe93aee7 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -730,7 +730,7 @@ pool support: *prefork, eventlet, gevent, threads, solo* Starting :program:`celery worker` with the :option:`--autoreload` option will enable the worker to watch for file system changes to all imported task -modules imported (and also any non-task modules added to the +modules (and also any non-task modules added to the :setting:`imports` setting or the :option:`-I|--include` option). This is an experimental feature intended for use in development only, From 2ccf237436cd432418e7e870a88fb03714998370 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 30 Oct 2015 16:17:56 -0700 Subject: [PATCH 0775/1103] Redis: ?new_join=1 is now the default --- celery/app/task.py | 5 ++-- celery/backends/redis.py | 16 +++------- celery/tests/backends/test_redis.py | 45 ++++++++++------------------- funtests/stress/stress/templates.py | 6 ++-- 4 files changed, 24 insertions(+), 48 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index 3c1365f68..97fd005b3 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -788,8 +788,9 @@ def add_to_chord(self, sig, lazy=False): :param lazy: If enabled the new task will not actually be called, and ``sig.delay()`` must be called manually. - Currently only supported by the Redis result backend when - ``?new_join=1`` is enabled. + .. versionadded:: 4.0 + + Currently only supported by the Redis result backend. """ if not self.request.chord: diff --git a/celery/backends/redis.py b/celery/backends/redis.py index bf9d0e770..486a4bbec 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -58,7 +58,7 @@ class RedisBackend(KeyValueStoreBackend): def __init__(self, host=None, port=None, db=None, password=None, max_connections=None, url=None, - connection_pool=None, new_join=False, **kwargs): + connection_pool=None, **kwargs): super(RedisBackend, self).__init__(expires_type=int, **kwargs) _get = self.app.conf.get if self.redis is None: @@ -87,14 +87,6 @@ def __init__(self, host=None, port=None, db=None, password=None, self.connparams = self._params_from_url(url, self.connparams) self.url = url - try: - new_join = strtobool(self.connparams.pop('new_join')) - except KeyError: - pass - if new_join: - self.apply_chord = self._new_chord_apply - self.on_chord_part_return = self._new_chord_return - self.connection_errors, self.channel_errors = ( get_redis_error_classes() if get_redis_error_classes else ((), ())) @@ -185,13 +177,13 @@ def _unpack_chord_result(self, tup, decode, raise ChordError('Dependency {0} raised {1!r}'.format(tid, retval)) return retval - def _new_chord_apply(self, header, partial_args, group_id, body, - result=None, options={}, **kwargs): + def apply_chord(self, header, partial_args, group_id, body, + result=None, options={}, **kwargs): # avoids saving the group in the redis db. options['task_id'] = group_id return header(*partial_args, **options or {}) - def _new_chord_return(self, request, state, result, propagate=None): + def on_chord_part_return(self, request, state, result, propagate=None): app = self.app tid, gid = request.id, request.group if not gid or not tid: diff --git a/celery/tests/backends/test_redis.py b/celery/tests/backends/test_redis.py index 878caa542..cbb534f5e 100644 --- a/celery/tests/backends/test_redis.py +++ b/celery/tests/backends/test_redis.py @@ -128,7 +128,7 @@ def setup(self): def test_reduce(self): try: from celery.backends.redis import RedisBackend - x = RedisBackend(app=self.app, new_join=True) + x = RedisBackend(app=self.app) self.assertTrue(loads(dumps(x))) except ImportError: raise SkipTest('redis not installed') @@ -136,12 +136,11 @@ def test_reduce(self): def test_no_redis(self): self.Backend.redis = None with self.assertRaises(ImproperlyConfigured): - self.Backend(app=self.app, new_join=True) + self.Backend(app=self.app) def test_url(self): x = self.Backend( 'redis://:bosco@vandelay.com:123//1', app=self.app, - new_join=True, ) self.assertTrue(x.connparams) self.assertEqual(x.connparams['host'], 'vandelay.com') @@ -152,7 +151,6 @@ def test_url(self): def test_socket_url(self): x = self.Backend( 'socket:///tmp/redis.sock?virtual_host=/3', app=self.app, - new_join=True, ) self.assertTrue(x.connparams) self.assertEqual(x.connparams['path'], '/tmp/redis.sock') @@ -167,7 +165,6 @@ def test_socket_url(self): def test_compat_propertie(self): x = self.Backend( 'redis://:bosco@vandelay.com:123//1', app=self.app, - new_join=True, ) with self.assertPendingDeprecation(): self.assertEqual(x.host, 'vandelay.com') @@ -185,65 +182,53 @@ def test_conf_raises_KeyError(self): 'result_expires': None, 'accept_content': ['json'], }) - self.Backend(app=self.app, new_join=True) + self.Backend(app=self.app) def test_expires_defaults_to_config(self): self.app.conf.result_expires = 10 - b = self.Backend(expires=None, app=self.app, new_join=True) + b = self.Backend(expires=None, app=self.app) self.assertEqual(b.expires, 10) def test_expires_is_int(self): - b = self.Backend(expires=48, app=self.app, new_join=True) + b = self.Backend(expires=48, app=self.app) self.assertEqual(b.expires, 48) - def test_set_new_join_from_url_query(self): - b = self.Backend('redis://?new_join=True;foobar=1', app=self.app) - self.assertEqual(b.on_chord_part_return, b._new_chord_return) - self.assertEqual(b.apply_chord, b._new_chord_apply) - def test_add_to_chord(self): - b = self.Backend('redis://?new_join=True', app=self.app) + b = self.Backend('redis://', app=self.app) gid = uuid() b.add_to_chord(gid, 'sig') b.client.incr.assert_called_with(b.get_key_for_group(gid, '.t'), 1) - def test_default_is_old_join(self): - b = self.Backend(app=self.app) - self.assertNotEqual(b.on_chord_part_return, b._new_chord_return) - self.assertNotEqual(b.apply_chord, b._new_chord_apply) - def test_expires_is_None(self): - b = self.Backend(expires=None, app=self.app, new_join=True) + b = self.Backend(expires=None, app=self.app) self.assertEqual( b.expires, self.app.conf.result_expires.total_seconds(), ) def test_expires_is_timedelta(self): - b = self.Backend( - expires=timedelta(minutes=1), app=self.app, new_join=1, - ) + b = self.Backend(expires=timedelta(minutes=1), app=self.app) self.assertEqual(b.expires, 60) def test_apply_chord(self): - self.Backend(app=self.app, new_join=True).apply_chord( + self.Backend(app=self.app).apply_chord( group(app=self.app), (), 'group_id', {}, result=[self.app.AsyncResult(x) for x in [1, 2, 3]], ) def test_mget(self): - b = self.Backend(app=self.app, new_join=True) + b = self.Backend(app=self.app) self.assertTrue(b.mget(['a', 'b', 'c'])) b.client.mget.assert_called_with(['a', 'b', 'c']) def test_set_no_expire(self): - b = self.Backend(app=self.app, new_join=True) + b = self.Backend(app=self.app) b.expires = None b.set('foo', 'bar') @patch('celery.result.GroupResult.restore') def test_on_chord_part_return(self, restore): - b = self.Backend(app=self.app, new_join=True) + b = self.Backend(app=self.app) def create_task(): tid = uuid() @@ -271,10 +256,10 @@ def create_task(): ]) def test_process_cleanup(self): - self.Backend(app=self.app, new_join=True).process_cleanup() + self.Backend(app=self.app).process_cleanup() def test_get_set_forget(self): - b = self.Backend(app=self.app, new_join=True) + b = self.Backend(app=self.app) tid = uuid() b.store_result(tid, 42, states.SUCCESS) self.assertEqual(b.get_status(tid), states.SUCCESS) @@ -283,7 +268,7 @@ def test_get_set_forget(self): self.assertEqual(b.get_status(tid), states.PENDING) def test_set_expires(self): - b = self.Backend(expires=512, app=self.app, new_join=True) + b = self.Backend(expires=512, app=self.app) tid = uuid() key = b.get_key_for_task(tid) b.store_result(tid, 42, states.SUCCESS) diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index 522e6c60c..b36087c82 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -85,14 +85,12 @@ class redis(default): 'fanout_prefix': True, 'fanout_patterns': True, } - result_backend = os.environ.get( - 'CSTRESS_BACKEND', 'redis://?new_join=1', - ) + result_backend = os.environ.get('CSTRESS_BACKEND', 'redis://') @template() class redistore(default): - result_backend = 'redis://?new_join=1' + result_backend = 'redis://' @template() From 4a4bb4542dcbfeee0c95e683845b17049600be29 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 30 Oct 2015 16:19:41 -0700 Subject: [PATCH 0776/1103] Cosmetics --- celery/events/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/celery/events/__init__.py b/celery/events/__init__.py index 6a79802cc..44dfd158d 100644 --- a/celery/events/__init__.py +++ b/celery/events/__init__.py @@ -331,8 +331,7 @@ def _get_queue_arguments(self, ttl=None, expires=None): ttl if ttl is not None else conf.event_queue_ttl, ), 'x-expires': maybe_s_to_ms( - expires if expires is not None - else conf.event_queue_expires, + expires if expires is not None else conf.event_queue_expires, ), }) From d3e1282664bd6cf6c7898df10e7bc37ea90be6df Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 30 Oct 2015 17:45:56 -0700 Subject: [PATCH 0777/1103] Implements the new chain field in task protocol 2. Closes #1078 The chain is now stored in reverse order, so the first task in the list is the last. This means we can do a quick pop instead of a slow head remove. --- celery/app/amqp.py | 4 ++-- celery/app/base.py | 4 ++-- celery/app/task.py | 1 + celery/app/trace.py | 6 +++++ celery/canvas.py | 54 +++++++++++++++++++++++++++++++++------------ 5 files changed, 51 insertions(+), 18 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index b8b5a9e27..7cc80d931 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -297,7 +297,7 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, callbacks=None, errbacks=None, reply_to=None, time_limit=None, soft_time_limit=None, create_sent_event=False, root_id=None, parent_id=None, - shadow=None, now=None, timezone=None): + shadow=None, chain=None, now=None, timezone=None): args = args or () kwargs = kwargs or {} utc = self.utc @@ -354,7 +354,7 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, args, kwargs, { 'callbacks': callbacks, 'errbacks': errbacks, - 'chain': None, # TODO + 'chain': chain, 'chord': chord, }, ), diff --git a/celery/app/base.py b/celery/app/base.py index b0e7663db..c9cbd5059 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -612,7 +612,7 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, add_to_parent=True, group_id=None, retries=0, chord=None, reply_to=None, time_limit=None, soft_time_limit=None, root_id=None, parent_id=None, route_name=None, - shadow=None, **options): + shadow=None, chain=None, **options): """Send task by name. :param name: Name of task to call (e.g. `"tasks.add"`). @@ -639,7 +639,7 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, maybe_list(link), maybe_list(link_error), reply_to or self.oid, time_limit, soft_time_limit, self.conf.task_send_sent_event, - root_id, parent_id, shadow, + root_id, parent_id, shadow, chain, ) if connection: diff --git a/celery/app/task.py b/celery/app/task.py index 97fd005b3..23617d48c 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -86,6 +86,7 @@ class Context(object): taskset = None # compat alias to group group = None chord = None + chain = None utc = None called_directly = True callbacks = None diff --git a/celery/app/trace.py b/celery/app/trace.py index ffd63fa50..b2af0f95a 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -394,6 +394,12 @@ def trace_task(uuid, args, kwargs, request=None): group(sigs).apply_async((retval,)) else: signature(callbacks[0], app=app).delay(retval) + + # execute first task in chain + chain = task.request.chain + if chain: + signature(chain.pop(), app=app).apply_async( + (retval,), chain=chain) mark_as_done( uuid, retval, task_request, publish_result, ) diff --git a/celery/canvas.py b/celery/canvas.py index 2f9cb4483..1a29b2aba 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -27,8 +27,7 @@ from celery.result import GroupResult from celery.utils import abstract from celery.utils.functional import ( - maybe_list, is_list, regen, - chunks as _chunks, + maybe_list, is_list, noop, regen, chunks as _chunks, ) from celery.utils.text import truncate @@ -383,6 +382,7 @@ def __init__(self, *tasks, **options): Signature.__init__( self, 'celery.chain', (), {'tasks': tasks}, **options ) + self._use_link = options.pop('use_link', None) self.subtask_type = 'chain' self._frozen = None @@ -402,6 +402,7 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, task_id=None, link=None, link_error=None, publisher=None, producer=None, root_id=None, app=None, **options): app = app or self.app + use_link = self._use_link args = (tuple(args) + tuple(self.args) if args and not self.immutable else self.args) @@ -413,12 +414,22 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, task_id, group_id, chord, ) + if results: # make sure we can do a link() and link_error() on a chain object. - if link: - tasks[-1].set(link=link) - tasks[0].apply_async(**options) - return results[-1] + if self._use_link: + # old task protocol used link for chains, last is last. + if link: + tasks[-1].set(link=link) + tasks[0].apply_async(**options) + return results[-1] + else: + # -- using chain message field means last task is first. + if link: + tasks[0].set(link=link) + first_task = tasks.pop() + first_task.apply_async(chain=tasks, **options) + return results[0] def freeze(self, _id=None, group_id=None, chord=None, root_id=None): _, results = self._frozen = self.prepare_steps( @@ -432,12 +443,25 @@ def prepare_steps(self, args, tasks, last_task_id=None, group_id=None, chord_body=None, clone=True, from_dict=Signature.from_dict): app = app or self.app + # use chain message field for protocol 2 and later. + # this avoids pickle blowing the stack on the recursion + # required by linking task together in a tree structure. + # (why is pickle using recursion? or better yet why cannot python + # do tail call optimization making recursion actually useful?) + use_link = self._use_link + if use_link is None and app.conf.task_protocol > 1: + use_link = False steps = deque(tasks) + + steps_pop = steps.popleft if use_link else steps.pop + steps_extend = steps.extendleft if use_link else steps.extend + extend_order = reverse if use_link else noop + next_step = prev_task = prev_res = None tasks, results = [], [] i = 0 while steps: - task = steps.popleft() + task = steps_pop() if not isinstance(task, abstract.CallableSignature): task = from_dict(task, app=app) @@ -452,12 +476,12 @@ def prepare_steps(self, args, tasks, if isinstance(task, chain): # splice the chain - steps.extendleft(reversed(task.tasks)) + steps_extend(extend_order(task.tasks)) continue elif isinstance(task, group) and steps: # automatically upgrade group(...) | s to chord(group, s) try: - next_step = steps.popleft() + next_step = steps_pop() # for chords we freeze by pretending it's a normal # signature instead of a group. res = Signature.freeze(next_step, root_id=root_id) @@ -484,11 +508,13 @@ def prepare_steps(self, args, tasks, i += 1 if prev_task: - # link previous task to this task. - prev_task.link(task) - # set AsyncResult.parent - if not res.parent: - res.parent = prev_res + if use_link: + # link previous task to this task. + prev_task.link(task) + if not res.parent: + res.parent = prev_res + else: + prev_res.parent = res if link_error: task.set(link_error=link_error) From 336269a23ab06254913b8688b9162c232fffb3f4 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 30 Oct 2015 18:08:39 -0700 Subject: [PATCH 0778/1103] Fixes chain issues from last commit (holy that function is hairy now, need to write two of them) --- celery/canvas.py | 42 ++++++++++++++++--------------- celery/tests/app/test_builtins.py | 17 +++++++++++++ 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 1a29b2aba..3e8930efb 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -414,7 +414,6 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, task_id, group_id, chord, ) - if results: # make sure we can do a link() and link_error() on a chain object. if self._use_link: @@ -455,13 +454,15 @@ def prepare_steps(self, args, tasks, steps_pop = steps.popleft if use_link else steps.pop steps_extend = steps.extendleft if use_link else steps.extend - extend_order = reverse if use_link else noop + extend_order = reversed if use_link else noop next_step = prev_task = prev_res = None tasks, results = [], [] i = 0 while steps: task = steps_pop() + last_task = not steps if use_link else not i + first_task = not i if use_link else not steps if not isinstance(task, abstract.CallableSignature): task = from_dict(task, app=app) @@ -471,30 +472,29 @@ def prepare_steps(self, args, tasks, # first task gets partial args from chain if clone: task = task.clone(args) if not i else task.clone() - elif not i: + elif first_task: task.args = tuple(args) + tuple(task.args) if isinstance(task, chain): # splice the chain steps_extend(extend_order(task.tasks)) continue - elif isinstance(task, group) and steps: - # automatically upgrade group(...) | s to chord(group, s) - try: - next_step = steps_pop() - # for chords we freeze by pretending it's a normal - # signature instead of a group. - res = Signature.freeze(next_step, root_id=root_id) - task = chord( - task, body=next_step, - task_id=res.task_id, root_id=root_id, - ) - except IndexError: - pass # no callback, so keep as group. - - if steps: - res = task.freeze(root_id=root_id) - else: + elif isinstance(task, group): + if (steps if use_link else prev_task): + # automatically upgrade group(...) | s to chord(group, s) + try: + next_step = steps_pop() if use_link else prev_task + # for chords we freeze by pretending it's a normal + # signature instead of a group. + res = Signature.freeze(next_step, root_id=root_id) + task = chord( + task, body=next_step, + task_id=res.task_id, root_id=root_id, + ) + except IndexError: + pass # no callback, so keep as group. + + if last_task: # chain(task_id=id) means task id is set for the last task # in the chain. If the chord is part of a chord/group # then that chord/group must synchronize based on the @@ -504,6 +504,8 @@ def prepare_steps(self, args, tasks, last_task_id, root_id=root_id, group_id=group_id, chord=chord_body, ) + else: + res = task.freeze(root_id=root_id) root_id = res.id if root_id is None else root_id i += 1 diff --git a/celery/tests/app/test_builtins.py b/celery/tests/app/test_builtins.py index 02f8a2b5c..50608c05e 100644 --- a/celery/tests/app/test_builtins.py +++ b/celery/tests/app/test_builtins.py @@ -140,15 +140,32 @@ def test_group_to_chord(self): self.add.s(20) | self.add.s(30) ) + c._use_link = True tasks, _ = c.prepare_steps((), c.tasks) self.assertIsInstance(tasks[0], chord) self.assertTrue(tasks[0].body.options['link']) self.assertTrue(tasks[0].body.options['link'][0].options['link']) c2 = self.add.s(2, 2) | group(self.add.s(i, i) for i in range(10)) + c2._use_link = True tasks2, _ = c2.prepare_steps((), c2.tasks) self.assertIsInstance(tasks2[1], group) + def test_group_to_chord__protocol_2(self): + c = ( + group([self.add.s(i, i) for i in range(5)], app=self.app) | + self.add.s(10) | + self.add.s(20) | + self.add.s(30) + ) + c._use_link = False + tasks, _ = c.prepare_steps((), c.tasks) + self.assertIsInstance(tasks[-1], chord) + + c2 = self.add.s(2, 2) | group(self.add.s(i, i) for i in range(10)) + c2._use_link = False + tasks2, _ = c2.prepare_steps((), c2.tasks) + self.assertIsInstance(tasks2[0], group) def test_apply_options(self): class static(Signature): From ff4e1a9e8138c1f7da39f4033a5b0cba33ae81be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Sat, 31 Oct 2015 21:53:01 +0100 Subject: [PATCH 0779/1103] fixes #2900 --- celery/backends/new_cassandra.py | 5 ++-- celery/tests/backends/test_new_cassandra.py | 32 +++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index b25d234ce..67403702e 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -127,11 +127,12 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, self._read_stmt = None def process_cleanup(self): - if self._connection is not None: - self._connection = None if self._session is not None: self._session.shutdown() self._session = None + if self._connection is not None: + self._connection.shutdown() + self._connection = None def _get_connection(self, write=False): """Prepare the connection for action diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 6f83db3dc..81373e0e5 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -133,3 +133,35 @@ def connect(self, *args, **kwargs): self.assertIsNone(x._session) x.process_cleanup() # should not raise + + def test_please_free_memory(self): + """ + Ensure that Cluster object IS shut down. + """ + with mock_module(*CASSANDRA_MODULES): + from celery.backends import new_cassandra as mod + + class RAMHoggingCluster(object): + + objects_alive = 0 + + def __init__(self, *args, **kwargs): + pass + + def connect(self, *args, **kwargs): + RAMHoggingCluster.objects_alive += 1 + return Mock() + + def shutdown(self): + RAMHoggingCluster.objects_alive -= 1 + + mod.cassandra = Mock() + mod.cassandra.cluster = Mock() + mod.cassandra.cluster.Cluster = RAMHoggingCluster + + for x in xrange(0, 10): + x = mod.CassandraBackend(app=self.app) + x._store_result('task_id', 'result', states.SUCCESS) + x.process_cleanup() + + self.assertEquals(RAMHoggingCluster.objects_alive, 0) From 0e4890cfa4222b34fb467fee50f75b2c4b39022a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Sun, 1 Nov 2015 00:20:46 +0100 Subject: [PATCH 0780/1103] fixes #2900 --- celery/backends/new_cassandra.py | 18 ++++++++++-------- celery/tests/backends/test_new_cassandra.py | 7 +++++++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py index 67403702e..39c476883 100644 --- a/celery/backends/new_cassandra.py +++ b/celery/backends/new_cassandra.py @@ -125,14 +125,14 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, self._session = None self._write_stmt = None self._read_stmt = None + self._make_stmt = None def process_cleanup(self): - if self._session is not None: - self._session.shutdown() - self._session = None if self._connection is not None: - self._connection.shutdown() - self._connection = None + self._connection.shutdown() # also shuts down _session + + self._connection = None + self._session = None def _get_connection(self, write=False): """Prepare the connection for action @@ -172,6 +172,7 @@ def _get_connection(self, write=False): Q_CREATE_RESULT_TABLE.format(table=self.table), ) self._make_stmt.consistency_level = self.write_consistency + try: self._session.execute(self._make_stmt) except cassandra.AlreadyExists: @@ -180,10 +181,11 @@ def _get_connection(self, write=False): except cassandra.OperationTimedOut: # a heavily loaded or gone Cassandra cluster failed to respond. # leave this class in a consistent state - self._connection = None - if self._session is not None: - self._session.shutdown() + if self._connection is not None: + self._connection.shutdown() # also shuts down _session + self._connection = None + self._session = None raise # we did fail after all - reraise def _store_result(self, task_id, result, status, diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index 81373e0e5..b10055840 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -41,6 +41,7 @@ def test_init_with_and_without_LOCAL_QUROM(self): with mock_module(*CASSANDRA_MODULES): from celery.backends import new_cassandra as mod mod.cassandra = Mock() + cons = mod.cassandra.ConsistencyLevel = Object() cons.LOCAL_QUORUM = 'foo' @@ -68,6 +69,7 @@ def test_get_task_meta_for(self): with mock_module(*CASSANDRA_MODULES): from celery.backends import new_cassandra as mod mod.cassandra = Mock() + x = mod.CassandraBackend(app=self.app) x._connection = True session = x._session = Mock() @@ -120,6 +122,9 @@ def __init__(self, *args, **kwargs): def connect(self, *args, **kwargs): raise OTOExc() + def shutdown(self): + pass + mod.cassandra = Mock() mod.cassandra.OperationTimedOut = OTOExc mod.cassandra.cluster = Mock() @@ -134,6 +139,7 @@ def connect(self, *args, **kwargs): x.process_cleanup() # should not raise + def test_please_free_memory(self): """ Ensure that Cluster object IS shut down. @@ -156,6 +162,7 @@ def shutdown(self): RAMHoggingCluster.objects_alive -= 1 mod.cassandra = Mock() + mod.cassandra.cluster = Mock() mod.cassandra.cluster.Cluster = RAMHoggingCluster From 055bf9536ed841a8730b48ea0c948a9524e7ff4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Sun, 1 Nov 2015 00:49:57 +0100 Subject: [PATCH 0781/1103] love for Python 3 --- celery/tests/backends/test_new_cassandra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py index b10055840..5ecbc292a 100644 --- a/celery/tests/backends/test_new_cassandra.py +++ b/celery/tests/backends/test_new_cassandra.py @@ -166,7 +166,7 @@ def shutdown(self): mod.cassandra.cluster = Mock() mod.cassandra.cluster.Cluster = RAMHoggingCluster - for x in xrange(0, 10): + for x in range(0, 10): x = mod.CassandraBackend(app=self.app) x._store_result('task_id', 'result', states.SUCCESS) x.process_cleanup() From 3ea224dea1213e8ac85c3239d9142219b3bc2639 Mon Sep 17 00:00:00 2001 From: Rudy Attias Date: Mon, 2 Nov 2015 18:14:28 +0200 Subject: [PATCH 0782/1103] fixes broken little-worker example using the -L test breaks the ability to use multiple instances of init script as described in the top of file --- extra/generic-init.d/celeryd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/generic-init.d/celeryd b/extra/generic-init.d/celeryd index 875f300f2..873dd9f52 100755 --- a/extra/generic-init.d/celeryd +++ b/extra/generic-init.d/celeryd @@ -39,7 +39,7 @@ fi # Can be a runlevel symlink (e.g. S02celeryd) -if [ -L "$0" ]; then +if [[ `dirname $0` == /etc/rc*.d ]]; then SCRIPT_FILE=$(readlink "$0") else SCRIPT_FILE="$0" From 9a03964bf83a031c039175c37645c417806dd69c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 3 Nov 2015 11:53:26 -0800 Subject: [PATCH 0783/1103] Cleanup chain.prepare --- celery/canvas.py | 46 ++++++++++++------------------- celery/tests/app/test_builtins.py | 8 +++--- 2 files changed, 22 insertions(+), 32 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 3e8930efb..9f49b9707 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -415,20 +415,12 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, ) if results: - # make sure we can do a link() and link_error() on a chain object. - if self._use_link: - # old task protocol used link for chains, last is last. - if link: - tasks[-1].set(link=link) - tasks[0].apply_async(**options) - return results[-1] - else: - # -- using chain message field means last task is first. - if link: - tasks[0].set(link=link) - first_task = tasks.pop() - first_task.apply_async(chain=tasks, **options) - return results[0] + if link: + tasks[0].set(link=link) + first_task = tasks.pop() + first_task.apply_async( + chain=tasks if not use_link else None, **options) + return results[0] def freeze(self, _id=None, group_id=None, chord=None, root_id=None): _, results = self._frozen = self.prepare_steps( @@ -452,17 +444,15 @@ def prepare_steps(self, args, tasks, use_link = False steps = deque(tasks) - steps_pop = steps.popleft if use_link else steps.pop - steps_extend = steps.extendleft if use_link else steps.extend - extend_order = reversed if use_link else noop + steps_pop = steps.pop + steps_extend = steps.extend next_step = prev_task = prev_res = None tasks, results = [], [] i = 0 while steps: task = steps_pop() - last_task = not steps if use_link else not i - first_task = not i if use_link else not steps + is_first_task, is_last_task = not steps, not i if not isinstance(task, abstract.CallableSignature): task = from_dict(task, app=app) @@ -471,19 +461,19 @@ def prepare_steps(self, args, tasks, # first task gets partial args from chain if clone: - task = task.clone(args) if not i else task.clone() - elif first_task: + task = task.clone(args) if is_first_task else task.clone() + elif is_first_task: task.args = tuple(args) + tuple(task.args) if isinstance(task, chain): # splice the chain - steps_extend(extend_order(task.tasks)) + steps_extend(task.tasks) continue elif isinstance(task, group): - if (steps if use_link else prev_task): + if prev_task: # automatically upgrade group(...) | s to chord(group, s) try: - next_step = steps_pop() if use_link else prev_task + next_step = prev_task # for chords we freeze by pretending it's a normal # signature instead of a group. res = Signature.freeze(next_step, root_id=root_id) @@ -494,7 +484,7 @@ def prepare_steps(self, args, tasks, except IndexError: pass # no callback, so keep as group. - if last_task: + if is_last_task: # chain(task_id=id) means task id is set for the last task # in the chain. If the chord is part of a chord/group # then that chord/group must synchronize based on the @@ -512,9 +502,9 @@ def prepare_steps(self, args, tasks, if prev_task: if use_link: # link previous task to this task. - prev_task.link(task) + task.link(prev_task) if not res.parent: - res.parent = prev_res + prev_res.parent = res.parent else: prev_res.parent = res @@ -686,7 +676,7 @@ def _prepared(self, tasks, partial_args, group_id, root_id, app, dict=dict, task = from_dict(task, app=app) if isinstance(task, group): # needs yield_from :( - unroll = task._prepared( + unroll = task_prepared( task.tasks, partial_args, group_id, root_id, app, ) for taskN, resN in unroll: diff --git a/celery/tests/app/test_builtins.py b/celery/tests/app/test_builtins.py index 50608c05e..49849310b 100644 --- a/celery/tests/app/test_builtins.py +++ b/celery/tests/app/test_builtins.py @@ -142,14 +142,14 @@ def test_group_to_chord(self): ) c._use_link = True tasks, _ = c.prepare_steps((), c.tasks) - self.assertIsInstance(tasks[0], chord) - self.assertTrue(tasks[0].body.options['link']) - self.assertTrue(tasks[0].body.options['link'][0].options['link']) + self.assertIsInstance(tasks[-1], chord) + self.assertTrue(tasks[-1].body.options['link']) + self.assertTrue(tasks[-1].body.options['link'][0].options['link']) c2 = self.add.s(2, 2) | group(self.add.s(i, i) for i in range(10)) c2._use_link = True tasks2, _ = c2.prepare_steps((), c2.tasks) - self.assertIsInstance(tasks2[1], group) + self.assertIsInstance(tasks2[0], group) def test_group_to_chord__protocol_2(self): c = ( From 6066a45700440233b1a8b0db9e44b792b2ccb13e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 4 Nov 2015 18:10:31 -0800 Subject: [PATCH 0784/1103] Adds parent_id + root_id task message fields, and to events. Closes #1318 --- celery/app/amqp.py | 4 +- celery/app/base.py | 14 ++- celery/app/trace.py | 27 ++++-- celery/canvas.py | 130 +++++++++++++++++++--------- celery/events/state.py | 27 ++++-- celery/result.py | 4 +- celery/tests/app/test_builtins.py | 62 ++++++++++++- celery/tests/tasks/test_canvas.py | 36 +++++++- celery/worker/consumer.py | 1 - celery/worker/request.py | 8 +- docs/userguide/monitoring.rst | 4 +- funtests/stress/stress/app.py | 10 +++ funtests/stress/stress/suite.py | 111 +++++++++++++++++++++++- funtests/stress/stress/templates.py | 2 +- 14 files changed, 363 insertions(+), 77 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 7cc80d931..8d94d7f55 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -360,8 +360,8 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, ), sent_event={ 'uuid': task_id, - 'root': root_id, - 'parent': parent_id, + 'root_id': root_id, + 'parent_id': parent_id, 'name': name, 'args': argsrepr, 'kwargs': kwargsrepr, diff --git a/celery/app/base.py b/celery/app/base.py index c9cbd5059..1d34f08ea 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -622,6 +622,7 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, Otherwise supports the same arguments as :meth:`@-Task.apply_async`. """ + parent = have_parent = None amqp = self.amqp task_id = task_id or uuid() producer = producer or publisher # XXX compat @@ -633,6 +634,16 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, ), stacklevel=2) options = router.route(options, route_name or name, args, kwargs) + if root_id is None: + parent, have_parent = get_current_worker_task(), True + if parent: + root_id = parent.request.root_id or parent.request.id + if parent_id is None: + if not have_parent: + parent, have_parent = get_current_worker_task(), True + if parent: + parent_id = parent.request.id + message = amqp.create_task_message( task_id, name, args, kwargs, countdown, eta, group_id, expires, retries, chord, @@ -649,7 +660,8 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, amqp.send_task_message(P, name, message, **options) result = (result_cls or self.AsyncResult)(task_id) if add_to_parent: - parent = get_current_worker_task() + if not have_parent: + parent, have_parent = get_current_worker_task(), True if parent: parent.add_trail(result) return result diff --git a/celery/app/trace.py b/celery/app/trace.py index b2af0f95a..d337373a9 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -306,10 +306,11 @@ def on_error(request, exc, uuid, state=FAILURE, call_errbacks=True): I = Info(state, exc) R = I.handle_error_state(task, request, eager=eager) if call_errbacks: + root_id = request.root_id or uuid group( [signature(errback, app=app) for errback in request.errbacks or []], app=app, - ).apply_async((uuid,)) + ).apply_async((uuid,), parent_id=uuid, root_id=root_id) return I, R, I.state, I.retval def trace_task(uuid, args, kwargs, request=None): @@ -336,6 +337,7 @@ def trace_task(uuid, args, kwargs, request=None): push_task(task) task_request = Context(request or {}, args=args, called_directly=False, kwargs=kwargs) + root_id = task_request.root_id or uuid push_request(task_request) try: # -*- PRE -*- @@ -363,8 +365,7 @@ def trace_task(uuid, args, kwargs, request=None): I.handle_ignore(task, task_request) except Retry as exc: I, R, state, retval = on_error( - task_request, exc, uuid, RETRY, call_errbacks=False, - ) + task_request, exc, uuid, RETRY, call_errbacks=False) except Exception as exc: I, R, state, retval = on_error(task_request, exc, uuid) except BaseException as exc: @@ -389,17 +390,27 @@ def trace_task(uuid, args, kwargs, request=None): else: sigs.append(sig) for group_ in groups: - group.apply_async((retval,)) + group.apply_async( + (retval,), + parent_id=uuid, root_id=root_id, + ) if sigs: - group(sigs).apply_async((retval,)) + group(sigs).apply_async( + (retval,), + parent_id=uuid, root_id=root_id, + ) else: - signature(callbacks[0], app=app).delay(retval) + signature(callbacks[0], app=app).apply_async( + (retval,), parent_id=uuid, root_id=root_id, + ) # execute first task in chain - chain = task.request.chain + chain = task_request.chain if chain: signature(chain.pop(), app=app).apply_async( - (retval,), chain=chain) + (retval,), chain=chain, + parent_id=uuid, root_id=root_id, + ) mark_as_done( uuid, retval, task_request, publish_result, ) diff --git a/celery/canvas.py b/celery/canvas.py index 9f49b9707..ff43f05d6 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -216,13 +216,17 @@ def clone(self, args=(), kwargs={}, **opts): return s partial = clone - def freeze(self, _id=None, group_id=None, chord=None, root_id=None): + def freeze(self, _id=None, group_id=None, chord=None, + root_id=None, parent_id=None): opts = self.options try: tid = opts['task_id'] except KeyError: tid = opts['task_id'] = _id or uuid() - root_id = opts.setdefault('root_id', root_id) + if root_id: + opts['root_id'] = root_id + if parent_id: + opts['parent_id'] = parent_id if 'reply_to' not in opts: opts['reply_to'] = self.app.oid if group_id: @@ -251,6 +255,9 @@ def set(self, immutable=None, **options): def set_immutable(self, immutable): self.immutable = immutable + def set_parent_id(self, parent_id): + self.parent_id = parent_id + def apply_async(self, args=(), kwargs={}, route_name=None, **options): try: _apply = self._apply_async @@ -362,6 +369,8 @@ def _apply_async(self): except KeyError: return _partial(self.app.send_task, self['task']) id = _getitem_property('options.task_id') + parent_id = _getitem_property('options.parent_id') + root_id = _getitem_property('options.root_id') task = _getitem_property('task') args = _getitem_property('args') kwargs = _getitem_property('kwargs') @@ -399,8 +408,8 @@ def apply_async(self, args=(), kwargs={}, **options): dict(self.options, **options) if options else self.options)) def run(self, args=(), kwargs={}, group_id=None, chord=None, - task_id=None, link=None, link_error=None, - publisher=None, producer=None, root_id=None, app=None, **options): + task_id=None, link=None, link_error=None, publisher=None, + producer=None, root_id=None, parent_id=None, app=None, **options): app = app or self.app use_link = self._use_link args = (tuple(args) + tuple(self.args) @@ -410,7 +419,7 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, tasks, results = self._frozen else: tasks, results = self.prepare_steps( - args, self.tasks, root_id, link_error, app, + args, self.tasks, root_id, parent_id, link_error, app, task_id, group_id, chord, ) @@ -422,15 +431,16 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, chain=tasks if not use_link else None, **options) return results[0] - def freeze(self, _id=None, group_id=None, chord=None, root_id=None): + def freeze(self, _id=None, group_id=None, chord=None, + root_id=None, parent_id=None): _, results = self._frozen = self.prepare_steps( - self.args, self.tasks, root_id, None, + self.args, self.tasks, root_id, parent_id, None, self.app, _id, group_id, chord, clone=False, ) return results[-1] def prepare_steps(self, args, tasks, - root_id=None, link_error=None, app=None, + root_id=None, parent_id=None, link_error=None, app=None, last_task_id=None, group_id=None, chord_body=None, clone=True, from_dict=Signature.from_dict): app = app or self.app @@ -447,7 +457,8 @@ def prepare_steps(self, args, tasks, steps_pop = steps.pop steps_extend = steps.extend - next_step = prev_task = prev_res = None + next_step = prev_task = prev_prev_task = None + prev_res = prev_prev_res = None tasks, results = [], [] i = 0 while steps: @@ -469,21 +480,18 @@ def prepare_steps(self, args, tasks, # splice the chain steps_extend(task.tasks) continue - elif isinstance(task, group): - if prev_task: - # automatically upgrade group(...) | s to chord(group, s) - try: - next_step = prev_task - # for chords we freeze by pretending it's a normal - # signature instead of a group. - res = Signature.freeze(next_step, root_id=root_id) - task = chord( - task, body=next_step, - task_id=res.task_id, root_id=root_id, - ) - except IndexError: - pass # no callback, so keep as group. + if isinstance(task, group) and prev_task: + # automatically upgrade group(...) | s to chord(group, s) + # for chords we freeze by pretending it's a normal + # signature instead of a group. + tasks.pop() + results.pop() + prev_res = prev_prev_res + task = chord( + task, body=prev_task, + task_id=res.task_id, root_id=root_id, app=app, + ) if is_last_task: # chain(task_id=id) means task id is set for the last task # in the chain. If the chord is part of a chord/group @@ -496,26 +504,36 @@ def prepare_steps(self, args, tasks, ) else: res = task.freeze(root_id=root_id) - root_id = res.id if root_id is None else root_id + i += 1 if prev_task: + prev_task.set_parent_id(task.id) if use_link: # link previous task to this task. task.link(prev_task) - if not res.parent: + if not res.parent and prev_res: prev_res.parent = res.parent - else: + elif prev_res: prev_res.parent = res + if is_first_task and parent_id is not None: + task.set_parent_id(parent_id) + if link_error: task.set(link_error=link_error) tasks.append(task) results.append(res) - prev_task, prev_res = task, res + prev_prev_task, prev_task, prev_prev_res, prev_res = ( + prev_task, task, prev_res, res, + ) + if root_id is None and tasks: + root_id = tasks[-1].id + for task in reversed(tasks): + task.options['root_id'] = root_id return tasks, results def apply(self, args=(), kwargs={}, **options): @@ -634,13 +652,16 @@ def apply_chunks(cls, task, it, n, app=None): return cls(task, it, n, app=app)() -def _maybe_group(tasks): +def _maybe_group(tasks, app): + if isinstance(tasks, dict): + tasks = signature(tasks, app=app) + if isinstance(tasks, group): - tasks = list(tasks.tasks) + tasks = tasks.tasks elif isinstance(tasks, abstract.CallableSignature): tasks = [tasks] else: - tasks = [signature(t) for t in regen(tasks)] + tasks = [signature(t, app=app) for t in regen(tasks)] return tasks @@ -649,8 +670,9 @@ class group(Signature): tasks = _getitem_property('kwargs.tasks') def __init__(self, *tasks, **options): + app = options.get('app') if len(tasks) == 1: - tasks = _maybe_group(tasks[0]) + tasks = _maybe_group(tasks[0], app) Signature.__init__( self, 'celery.group', (), {'tasks': tasks}, **options ) @@ -662,6 +684,9 @@ def from_dict(self, d, app=None): d, group(d['kwargs']['tasks'], app=app, **d['options']), ) + def __len__(self): + return len(self.tasks) + def _prepared(self, tasks, partial_args, group_id, root_id, app, dict=dict, CallableSignature=abstract.CallableSignature, from_dict=Signature.from_dict): @@ -703,6 +728,10 @@ def _freeze_gid(self, options): options.pop('task_id', uuid())) return options, group_id, options.get('root_id') + def set_parent_id(self, parent_id): + for task in self.tasks: + task.set_parent_id(parent_id) + def apply_async(self, args=(), kwargs=None, add_to_parent=True, producer=None, **options): app = self.app @@ -757,7 +786,7 @@ def link_error(self, sig): def __call__(self, *partial_args, **options): return self.apply_async(partial_args, **options) - def _freeze_unroll(self, new_tasks, group_id, chord, root_id): + def _freeze_unroll(self, new_tasks, group_id, chord, root_id, parent_id): stack = deque(self.tasks) while stack: task = maybe_signature(stack.popleft(), app=self._app).clone() @@ -766,9 +795,11 @@ def _freeze_unroll(self, new_tasks, group_id, chord, root_id): else: new_tasks.append(task) yield task.freeze(group_id=group_id, - chord=chord, root_id=root_id) + chord=chord, root_id=root_id, + parent_id=parent_id) - def freeze(self, _id=None, group_id=None, chord=None, root_id=None): + def freeze(self, _id=None, group_id=None, chord=None, + root_id=None, parent_id=None): opts = self.options try: gid = opts['task_id'] @@ -779,11 +810,12 @@ def freeze(self, _id=None, group_id=None, chord=None, root_id=None): if chord: opts['chord'] = chord root_id = opts.setdefault('root_id', root_id) + parent_id = opts.setdefault('parent_id', parent_id) new_tasks = [] # Need to unroll subgroups early so that chord gets the # right result instance for chord_unlock etc. results = list(self._freeze_unroll( - new_tasks, group_id, chord, root_id, + new_tasks, group_id, chord, root_id, parent_id, )) if isinstance(self.tasks, MutableSequence): self.tasks[:] = new_tasks @@ -819,16 +851,29 @@ def app(self): class chord(Signature): def __init__(self, header, body=None, task='celery.chord', - args=(), kwargs={}, **options): + args=(), kwargs={}, app=None, **options): Signature.__init__( self, task, args, - dict(kwargs, header=_maybe_group(header), + dict(kwargs, header=_maybe_group(header, app), body=maybe_signature(body, app=self._app)), **options ) self.subtask_type = 'chord' - def freeze(self, *args, **kwargs): - return self.body.freeze(*args, **kwargs) + def freeze(self, _id=None, group_id=None, chord=None, + root_id=None, parent_id=None): + if not isinstance(self.tasks, group): + self.tasks = group(self.tasks) + self.tasks.freeze(parent_id=parent_id, root_id=root_id) + self.id = self.tasks.id + return self.body.freeze(_id, parent_id=self.id, root_id=root_id) + + def set_parent_id(self, parent_id): + tasks = self.tasks + if isinstance(tasks, group): + tasks = tasks.tasks + for task in tasks: + task.set_parent_id(parent_id) + self.parent_id = parent_id @classmethod def from_dict(self, d, app=None): @@ -848,7 +893,11 @@ def app(self): def _get_app(self, body=None): app = self._app if app is None: - app = self.tasks[0]._app + try: + tasks = self.tasks.tasks # is a group + except AttributeError: + tasks = self.tasks + app = tasks[0]._app if app is None and body is not None: app = body._app return app if app is not None else current_app @@ -900,6 +949,7 @@ def run(self, header, body, partial_args, app=None, interval=None, body.chord_size = self.__length_hint__() options = dict(self.options, **options) if options else self.options if options: + options.pop('task_id', None) body.options.update(options) results = header.freeze( diff --git a/celery/events/state.py b/celery/events/state.py index 549f8dfcf..bc03f0c78 100644 --- a/celery/events/state.py +++ b/celery/events/state.py @@ -233,11 +233,13 @@ class Task(object): state = states.PENDING clock = 0 - _fields = ('uuid', 'name', 'state', 'received', 'sent', 'started', - 'succeeded', 'failed', 'retried', 'revoked', 'args', 'kwargs', - 'eta', 'expires', 'retries', 'worker', 'result', 'exception', - 'timestamp', 'runtime', 'traceback', 'exchange', 'routing_key', - 'clock', 'client') + _fields = ( + 'uuid', 'name', 'state', 'received', 'sent', 'started', + 'succeeded', 'failed', 'retried', 'revoked', 'args', 'kwargs', + 'eta', 'expires', 'retries', 'worker', 'result', 'exception', + 'timestamp', 'runtime', 'traceback', 'exchange', 'routing_key', + 'clock', 'client', 'root_id', 'parent_id', + ) if not PYPY: __slots__ = ('__dict__', '__weakref__') @@ -249,12 +251,19 @@ class Task(object): #: that state. ``(RECEIVED, ('name', 'args')``, means the name and args #: fields are always taken from the RECEIVED state, and any values for #: these fields received before or after is simply ignored. - merge_rules = {states.RECEIVED: ('name', 'args', 'kwargs', - 'retries', 'eta', 'expires')} + merge_rules = { + states.RECEIVED: ( + 'name', 'args', 'kwargs', 'parent_id', + 'root_id' 'retries', 'eta', 'expires', + ), + } #: meth:`info` displays these fields by default. - _info_fields = ('args', 'kwargs', 'retries', 'result', 'eta', 'runtime', - 'expires', 'exception', 'exchange', 'routing_key') + _info_fields = ( + 'args', 'kwargs', 'retries', 'result', 'eta', 'runtime', + 'expires', 'exception', 'exchange', 'routing_key', + 'root_id', 'parent_id', + ) def __init__(self, uuid=None, **kwargs): self.uuid = uuid diff --git a/celery/result.py b/celery/result.py index 83b4c91d4..ddda0051e 100644 --- a/celery/result.py +++ b/celery/result.py @@ -122,7 +122,7 @@ def revoke(self, connection=None, terminate=False, signal=None, reply=wait, timeout=timeout) def get(self, timeout=None, propagate=True, interval=0.5, - no_ack=True, follow_parents=True, + no_ack=True, follow_parents=True, callback=None, EXCEPTION_STATES=states.EXCEPTION_STATES, PROPAGATE_STATES=states.PROPAGATE_STATES): """Wait until task is ready, and return its result. @@ -174,6 +174,8 @@ def get(self, timeout=None, propagate=True, interval=0.5, status = meta['status'] if status in PROPAGATE_STATES and propagate: raise meta['result'] + if callback is not None: + callback(self.id, meta['result']) return meta['result'] wait = get # deprecated alias to :meth:`get`. diff --git a/celery/tests/app/test_builtins.py b/celery/tests/app/test_builtins.py index 49849310b..7f7bac1e8 100644 --- a/celery/tests/app/test_builtins.py +++ b/celery/tests/app/test_builtins.py @@ -133,18 +133,72 @@ def test_apply_async(self): self.assertTrue(result.parent.parent) self.assertIsNone(result.parent.parent.parent) + def test_group_to_chord__freeze_parent_id(self): + def using_freeze(c): + c.freeze(parent_id='foo', root_id='root') + return c._frozen[0] + self.assert_group_to_chord_parent_ids(using_freeze) + + def assert_group_to_chord_parent_ids(self, freezefun): + c = ( + self.add.s(5, 5) | + group([self.add.s(i, i) for i in range(5)], app=self.app) | + self.add.si(10, 10) | + self.add.si(20, 20) | + self.add.si(30, 30) + ) + tasks = freezefun(c) + self.assertEqual(tasks[-1].parent_id, 'foo') + self.assertEqual(tasks[-1].root_id, 'root') + self.assertEqual(tasks[-2].parent_id, tasks[-1].id) + self.assertEqual(tasks[-2].root_id, 'root') + self.assertEqual(tasks[-2].body.parent_id, tasks[-2].tasks.id) + self.assertEqual(tasks[-2].body.parent_id, tasks[-2].id) + self.assertEqual(tasks[-2].body.root_id, 'root') + self.assertEqual(tasks[-2].tasks.tasks[0].parent_id, tasks[-1].id) + self.assertEqual(tasks[-2].tasks.tasks[0].root_id, 'root') + self.assertEqual(tasks[-2].tasks.tasks[1].parent_id, tasks[-1].id) + self.assertEqual(tasks[-2].tasks.tasks[1].root_id, 'root') + self.assertEqual(tasks[-2].tasks.tasks[2].parent_id, tasks[-1].id) + self.assertEqual(tasks[-2].tasks.tasks[2].root_id, 'root') + self.assertEqual(tasks[-2].tasks.tasks[3].parent_id, tasks[-1].id) + self.assertEqual(tasks[-2].tasks.tasks[3].root_id, 'root') + self.assertEqual(tasks[-2].tasks.tasks[4].parent_id, tasks[-1].id) + self.assertEqual(tasks[-2].tasks.tasks[4].root_id, 'root') + self.assertEqual(tasks[-3].parent_id, tasks[-2].body.id) + self.assertEqual(tasks[-3].root_id, 'root') + self.assertEqual(tasks[-4].parent_id, tasks[-3].id) + self.assertEqual(tasks[-4].root_id, 'root') + def test_group_to_chord(self): c = ( + self.add.s(5) | group([self.add.s(i, i) for i in range(5)], app=self.app) | self.add.s(10) | self.add.s(20) | self.add.s(30) ) c._use_link = True - tasks, _ = c.prepare_steps((), c.tasks) - self.assertIsInstance(tasks[-1], chord) - self.assertTrue(tasks[-1].body.options['link']) - self.assertTrue(tasks[-1].body.options['link'][0].options['link']) + tasks, results = c.prepare_steps((), c.tasks) + + self.assertEqual(tasks[-1].args[0], 5) + self.assertIsInstance(tasks[-2], chord) + self.assertEqual(len(tasks[-2].tasks), 5) + self.assertEqual(tasks[-2].parent_id, tasks[-1].id) + self.assertEqual(tasks[-2].root_id, tasks[-1].id) + self.assertEqual(tasks[-2].body.args[0], 10) + self.assertEqual(tasks[-2].body.parent_id, tasks[-2].id) + + self.assertEqual(tasks[-3].args[0], 20) + self.assertEqual(tasks[-3].root_id, tasks[-1].id) + self.assertEqual(tasks[-3].parent_id, tasks[-2].body.id) + + self.assertEqual(tasks[-4].args[0], 30) + self.assertEqual(tasks[-4].parent_id, tasks[-3].id) + self.assertEqual(tasks[-4].root_id, tasks[-1].id) + + self.assertTrue(tasks[-2].body.options['link']) + self.assertTrue(tasks[-2].body.options['link'][0].options['link']) c2 = self.add.s(2, 2) | group(self.add.s(i, i) for i in range(10)) c2._use_link = True diff --git a/celery/tests/tasks/test_canvas.py b/celery/tests/tasks/test_canvas.py index 287241d2d..52ed2ccb4 100644 --- a/celery/tests/tasks/test_canvas.py +++ b/celery/tests/tasks/test_canvas.py @@ -14,7 +14,7 @@ ) from celery.result import EagerResult -from celery.tests.case import AppCase, Mock +from celery.tests.case import AppCase, ContextMock, Mock SIG = Signature({'task': 'TASK', 'args': ('A1',), @@ -233,6 +233,40 @@ def test_empty_chain_returns_none(self): self.assertIsNone(chain(app=self.app)()) self.assertIsNone(chain(app=self.app).apply_async()) + def test_root_id_parent_id(self): + self.app.conf.task_protocol = 2 + c = chain(self.add.si(i, i) for i in range(4)) + c.freeze() + tasks, _ = c._frozen + for i, task in enumerate(tasks): + self.assertEqual(task.root_id, tasks[-1].id) + try: + self.assertEqual(task.parent_id, tasks[i + 1].id) + except IndexError: + assert i == len(tasks) - 1 + else: + valid_parents = i + self.assertEqual(valid_parents, len(tasks) - 2) + + self.assert_sent_with_ids(tasks[-1], tasks[-1].id, 'foo', + parent_id='foo') + self.assertTrue(tasks[-2].options['parent_id']) + self.assert_sent_with_ids(tasks[-2], tasks[-1].id, tasks[-1].id) + self.assert_sent_with_ids(tasks[-3], tasks[-1].id, tasks[-2].id) + self.assert_sent_with_ids(tasks[-4], tasks[-1].id, tasks[-3].id) + + + def assert_sent_with_ids(self, task, rid, pid, **options): + self.app.amqp.send_task_message = Mock(name='send_task_message') + self.app.backend = Mock() + self.app.producer_or_acquire = ContextMock() + + res = task.apply_async(**options) + self.assertTrue(self.app.amqp.send_task_message.called) + message = self.app.amqp.send_task_message.call_args[0][2] + self.assertEqual(message.headers['parent_id'], pid) + self.assertEqual(message.headers['root_id'], rid) + def test_call_no_tasks(self): x = chain() self.assertFalse(x()) diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 20d392288..bda4d8288 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -458,7 +458,6 @@ def create_task_handler(self): callbacks = self.on_task_message def on_task_received(message): - # payload will only be set for v1 protocol, since v2 # will defer deserializing the message body to the pool. payload = None diff --git a/celery/worker/request.py b/celery/worker/request.py index 73cbc86cd..b3cb81ad0 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -77,9 +77,9 @@ class Request(object): if not IS_PYPY: # pragma: no cover __slots__ = ( - 'app', 'type', 'name', 'id', 'on_ack', 'body', - 'hostname', 'eventer', 'connection_errors', 'task', 'eta', - 'expires', 'request_dict', 'on_reject', 'utc', + 'app', 'type', 'name', 'id', 'root_id', 'parent_id', + 'on_ack', 'body', 'hostname', 'eventer', 'connection_errors', + 'task', 'eta', 'expires', 'request_dict', 'on_reject', 'utc', 'content_type', 'content_encoding', 'argsrepr', 'kwargsrepr', '__weakref__', '__dict__', ) @@ -108,6 +108,8 @@ def __init__(self, message, on_ack=noop, self.id = headers['id'] type = self.type = self.name = headers['task'] + self.root_id = headers.get('root_id') + self.parent_id = headers.get('parent_id') if 'shadow' in headers: self.name = headers['shadow'] if 'timelimit' in headers: diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index 083e9dacf..8652f6bec 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -650,7 +650,7 @@ task-sent ~~~~~~~~~ :signature: ``task-sent(uuid, name, args, kwargs, retries, eta, expires, - queue, exchange, routing_key)`` + queue, exchange, routing_key, root_id, parent_id)`` Sent when a task message is published and the :setting:`task_send_sent_event` setting is enabled. @@ -661,7 +661,7 @@ task-received ~~~~~~~~~~~~~ :signature: ``task-received(uuid, name, args, kwargs, retries, eta, hostname, - timestamp)`` + timestamp, root_id, parent_id)`` Sent when the worker receives a task. diff --git a/funtests/stress/stress/app.py b/funtests/stress/stress/app.py index 658d48e5d..ea10c03a5 100644 --- a/funtests/stress/stress/app.py +++ b/funtests/stress/stress/app.py @@ -63,6 +63,16 @@ def add(x, y): return x + y +@app.task(bind=True) +def ids(self, i): + return (self.request.root_id, self.request.parent_id, i) + + +@app.task(bind=True) +def collect_ids(self, ids, i): + return ids, (self.request.root_id, self.request.parent_id, i) + + @app.task def xsum(x): return sum(x) diff --git a/funtests/stress/stress/suite.py b/funtests/stress/stress/suite.py index 3902c82e4..763c41727 100755 --- a/funtests/stress/stress/suite.py +++ b/funtests/stress/stress/suite.py @@ -10,7 +10,7 @@ from itertools import count from time import sleep -from celery import group, VERSION_BANNER +from celery import VERSION_BANNER, chain, group, uuid from celery.exceptions import TimeoutError from celery.five import items, monotonic, range, values from celery.utils.debug import blockdetection @@ -18,12 +18,13 @@ from celery.utils.timeutils import humanize_seconds from .app import ( - marker, _marker, add, any_, exiting, kill, sleeping, + marker, _marker, add, any_, collect_ids, exiting, ids, kill, sleeping, sleeping_ignore_limits, any_returning, print_unicode, ) from .data import BIG, SMALL from .fbi import FBI + BANNER = """\ Celery stress-suite v{version} @@ -50,6 +51,10 @@ Inf = float('Inf') +def assert_equal(a, b): + assert a == b, '{0!r} != {1!r}'.format(a, b) + + class StopSuite(Exception): pass @@ -163,6 +168,7 @@ def banner(self, tests): ) def runtest(self, fun, n=50, index=0, repeats=1): + n = getattr(fun, '__iterations__', None) or n print('{0}: [[[{1}({2})]]]'.format(repeats, fun.__name__, n)) with blockdetection(self.block_timeout): with self.fbi.investigation(): @@ -185,6 +191,8 @@ def runtest(self, fun, n=50, index=0, repeats=1): raise except Exception as exc: print('-> {0!r}'.format(exc)) + import traceback + print(traceback.format_exc()) print(pstatus(self.progress)) else: print(pstatus(self.progress)) @@ -238,13 +246,14 @@ def dump_progress(self): _creation_counter = count(0) -def testcase(*groups): +def testcase(*groups, **kwargs): if not groups: raise ValueError('@testcase requires at least one group name') def _mark_as_case(fun): fun.__testgroup__ = groups fun.__testsort__ = next(_creation_counter) + fun.__iterations__ = kwargs.get('iterations') return fun return _mark_as_case @@ -262,12 +271,106 @@ def _is_descriptor(obj, attr): class Suite(BaseSuite): + @testcase('all', 'green', iterations=1) + def chain(self): + c = add.s(4, 4) | add.s(8) | add.s(16) + assert_equal(self.join(c()), 32) + + @testcase('all', 'green', iterations=1) + def chaincomplex(self): + c = ( + add.s(2, 2) | ( + add.s(4) | add.s(8) | add.s(16) + ) | + group(add.s(i) for i in range(4)) + ) + res = c() + assert_equal(res.get(), [32, 33, 34, 35]) + + @testcase('all', 'green', iterations=1) + def parentids_chain(self): + c = chain(ids.si(i) for i in range(248)) + c.freeze() + res = c() + res.get(timeout=5) + self.assert_ids(res, len(c.tasks) - 1) + + @testcase('all', 'green', iterations=1) + def parentids_group(self): + g = ids.si(1) | ids.si(2) | group(ids.si(i) for i in range(2, 50)) + res = g() + expected_root_id = res.parent.parent.id + expected_parent_id = res.parent.id + values = res.get(timeout=5) + + for i, r in enumerate(values): + root_id, parent_id, value = r + assert_equal(root_id, expected_root_id) + assert_equal(parent_id, expected_parent_id) + assert_equal(value, i + 2) + + def assert_ids(self, res, len): + i, root = len, res + while root.parent: + root = root.parent + node = res + while node: + root_id, parent_id, value = node.get(timeout=5) + assert_equal(value, i) + assert_equal(root_id, root.id) + if node.parent: + assert_equal(parent_id, node.parent.id) + node = node.parent + i -= 1 + + @testcase('redis', iterations=1) + def parentids_chord(self): + self.assert_parentids_chord() + self.assert_parentids_chord(uuid(), uuid()) + + def assert_parentids_chord(self, base_root=None, base_parent=None): + g = ( + ids.si(1) | + ids.si(2) | + group(ids.si(i) for i in range(3, 50)) | + collect_ids.s(i=50) | + ids.si(51) + ) + g.freeze(root_id=base_root, parent_id=base_parent) + res = g.apply_async(root_id=base_root, parent_id=base_parent) + expected_root_id = base_root or res.parent.parent.parent.id + + root_id, parent_id, value = res.get(timeout=5) + assert_equal(value, 51) + assert_equal(root_id, expected_root_id) + assert_equal(parent_id, res.parent.id) + + prev, (root_id, parent_id, value) = res.parent.get(timeout=5) + assert_equal(value, 50) + assert_equal(root_id, expected_root_id) + assert_equal(parent_id, res.parent.parent.id) + + for i, p in enumerate(prev): + root_id, parent_id, value = p + assert_equal(root_id, expected_root_id) + assert_equal(parent_id, res.parent.parent.id) + + root_id, parent_id, value = res.parent.parent.get(timeout=5) + assert_equal(value, 2) + assert_equal(parent_id, res.parent.parent.parent.id) + assert_equal(root_id, expected_root_id) + + root_id, parent_id, value = res.parent.parent.parent.get(timeout=5) + assert_equal(value, 1) + assert_equal(root_id, expected_root_id) + assert_equal(parent_id, base_parent) + @testcase('all', 'green') def manyshort(self): self.join(group(add.s(i, i) for i in range(1000))(), timeout=10, propagate=True) - @testcase('all', 'green') + @testcase('all', 'green', iterations=1) def unicodetask(self): self.join(group(print_unicode.s() for _ in range(5))(), timeout=1, propagate=True) diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index b36087c82..e04a15f8b 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -57,7 +57,7 @@ class default(object): result_serializer = 'json' result_persistent = True result_expires = 300 - result_cache_max = -1 + result_cache_max = 100 task_default_queue = CSTRESS_QUEUE task_queues = [ Queue(CSTRESS_QUEUE, From aaf9ad2b01bc626fe4becbf7dedafcc4d91864cb Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 4 Nov 2015 18:27:13 -0800 Subject: [PATCH 0785/1103] Fixes tests --- celery/tests/events/test_state.py | 2 ++ celery/tests/tasks/test_canvas.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/celery/tests/events/test_state.py b/celery/tests/events/test_state.py index ad8a041d8..f51dfe74e 100644 --- a/celery/tests/events/test_state.py +++ b/celery/tests/events/test_state.py @@ -244,6 +244,8 @@ def test_info(self): eta=1, runtime=0.0001, expires=1, + parent_id='bdefc', + root_id='dedfef', foo=None, exception=1, received=time() - 10, diff --git a/celery/tests/tasks/test_canvas.py b/celery/tests/tasks/test_canvas.py index 52ed2ccb4..9a22515af 100644 --- a/celery/tests/tasks/test_canvas.py +++ b/celery/tests/tasks/test_canvas.py @@ -303,7 +303,7 @@ def test_reverse(self): def test_maybe_group_sig(self): self.assertListEqual( - _maybe_group(self.add.s(2, 2)), [self.add.s(2, 2)], + _maybe_group(self.add.s(2, 2), self.app), [self.add.s(2, 2)], ) def test_from_dict(self): From 20424c5561bc9a99c624da5c8e98b4fcefc1fdcd Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 4 Nov 2015 18:32:39 -0800 Subject: [PATCH 0786/1103] Worker now stores NotRegisteredError for unknown task, and adds task_rejected + task_unknown signals. Closes #2092 --- celery/signals.py | 6 ++++++ celery/worker/consumer.py | 8 +++++++- docs/userguide/signals.rst | 39 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/celery/signals.py b/celery/signals.py index 2091830cb..bfc8240e3 100644 --- a/celery/signals.py +++ b/celery/signals.py @@ -50,6 +50,12 @@ task_revoked = Signal(providing_args=[ 'request', 'terminated', 'signum', 'expired', ]) +task_rejected = Signal(providing_args=[ + 'message', 'exc', +]) +task_unknown = Signal(providing_args=[ + 'message', 'exc', +]) celeryd_init = Signal(providing_args=['instance', 'conf', 'options']) celeryd_after_setup = Signal(providing_args=['instance', 'conf']) import_modules = Signal(providing_args=[]) diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index bda4d8288..d24710879 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -32,9 +32,10 @@ from kombu.utils.limits import TokenBucket from celery import bootsteps +from celery import signals from celery.app.trace import build_tracer from celery.canvas import signature -from celery.exceptions import InvalidTaskError +from celery.exceptions import InvalidTaskError, NotRegistered from celery.utils.functional import noop from celery.utils.log import get_logger from celery.utils.text import truncate @@ -434,14 +435,19 @@ def _message_report(self, body, message): def on_unknown_message(self, body, message): warn(UNKNOWN_FORMAT, self._message_report(body, message)) message.reject_log_error(logger, self.connection_errors) + signals.task_rejected.send(sender=self, message=message, exc=None) def on_unknown_task(self, body, message, exc): error(UNKNOWN_TASK_ERROR, exc, dump_body(message, body), exc_info=True) message.reject_log_error(logger, self.connection_errors) + self.app.backend.mark_as_failure( + message.headers['id'], NotRegistered(message.headers['task'])) + signals.task_unknown.send(sender=self, message=message, exc=exc) def on_invalid_task(self, body, message, exc): error(INVALID_TASK_ERROR, exc, dump_body(message, body), exc_info=True) message.reject_log_error(logger, self.connection_errors) + signals.task_rejected.send(sender=self, message=message, exc=exc) def update_strategies(self): loader = self.app.loader diff --git a/docs/userguide/signals.rst b/docs/userguide/signals.rst index cae2f7865..9e48e9648 100644 --- a/docs/userguide/signals.rst +++ b/docs/userguide/signals.rst @@ -300,6 +300,45 @@ Provides arguments: * expired Set to :const:`True` if the task expired. +.. signal:: task_unknown + +task_unknown +~~~~~~~~~~~~ + +Dispatched when a worker receives a message for a task that is not registered. + +Sender is the worker :class:`~celery.worker.consumer.Consumer`. + +Provides arguments: + +* message + + Raw message object. + +* exc + + The error that occurred. + +.. signal:: task_rejected + +task_rejected +~~~~~~~~~~~~~ + +Dispatched when a worker receives an unknown type of message to one of its +task queues. + +Sender is the worker :class:`~celery.worker.consumer.Consumer`. + +Provides arguments: + +* message + + Raw message object. + +* exc + + The error that occurred (if any). + App Signals ----------- From 757678a59a72cc79599332b71953d6eec79c33c4 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 4 Nov 2015 18:40:34 -0800 Subject: [PATCH 0787/1103] Worker: Also send task-failed event on unregistered task (Issue #2791) --- celery/signals.py | 2 +- celery/worker/consumer.py | 13 ++++++++++--- docs/userguide/signals.rst | 8 ++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/celery/signals.py b/celery/signals.py index bfc8240e3..c864a1b64 100644 --- a/celery/signals.py +++ b/celery/signals.py @@ -54,7 +54,7 @@ 'message', 'exc', ]) task_unknown = Signal(providing_args=[ - 'message', 'exc', + 'message', 'exc', 'name', 'id', ]) celeryd_init = Signal(providing_args=['instance', 'conf', 'options']) celeryd_after_setup = Signal(providing_args=['instance', 'conf']) diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index d24710879..984826518 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -439,10 +439,17 @@ def on_unknown_message(self, body, message): def on_unknown_task(self, body, message, exc): error(UNKNOWN_TASK_ERROR, exc, dump_body(message, body), exc_info=True) + id_, name = message.headers['id'], message.headers['task'] message.reject_log_error(logger, self.connection_errors) - self.app.backend.mark_as_failure( - message.headers['id'], NotRegistered(message.headers['task'])) - signals.task_unknown.send(sender=self, message=message, exc=exc) + self.app.backend.mark_as_failure(id_, NotRegistered(name)) + if self.event_dispatcher: + self.event_dispatcher.send( + 'task-failed', uuid=id_, + exception='NotRegistered({0!r})'.format(name), + ) + signals.task_unknown.send( + sender=self, message=message, exc=exc, name=name, id=id_, + ) def on_invalid_task(self, body, message, exc): error(INVALID_TASK_ERROR, exc, dump_body(message, body), exc_info=True) diff --git a/docs/userguide/signals.rst b/docs/userguide/signals.rst index 9e48e9648..db5c1eb65 100644 --- a/docs/userguide/signals.rst +++ b/docs/userguide/signals.rst @@ -311,6 +311,14 @@ Sender is the worker :class:`~celery.worker.consumer.Consumer`. Provides arguments: +* name + + Name of task not found in registry. + +* id + + The task id found in the message. + * message Raw message object. From 081c78fffe10081de43b99a9a537d0d6afbbafc8 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 4 Nov 2015 18:57:23 -0800 Subject: [PATCH 0788/1103] new_cassandra is now cassandra --- README.rst | 3 - celery/app/defaults.py | 2 +- celery/backends/__init__.py | 1 - celery/backends/cassandra.py | 300 ++++++++++-------- celery/backends/new_cassandra.py | 226 ------------- celery/tests/backends/test_cassandra.py | 179 ++++------- celery/tests/backends/test_new_cassandra.py | 135 -------- docs/configuration.rst | 22 +- docs/includes/installation.txt | 3 - .../celery.backends.new_cassandra.rst | 11 - docs/internals/reference/index.rst | 1 - docs/whatsnew-4.0.rst | 8 +- requirements/extras/cassandra.txt | 2 +- requirements/extras/new_cassandra.txt | 1 - setup.py | 1 - 15 files changed, 248 insertions(+), 647 deletions(-) delete mode 100644 celery/backends/new_cassandra.py delete mode 100644 celery/tests/backends/test_new_cassandra.py delete mode 100644 docs/internals/reference/celery.backends.new_cassandra.rst delete mode 100644 requirements/extras/new_cassandra.txt diff --git a/README.rst b/README.rst index d79d2e996..ce8a2cf3d 100644 --- a/README.rst +++ b/README.rst @@ -284,9 +284,6 @@ Transports and Backends for using memcached as a result backend. :celery[cassandra]: - for using Apache Cassandra as a result backend with pycassa driver. - -:celery[new_cassandra]: for using Apache Cassandra as a result backend with DataStax driver. :celery[couchdb]: diff --git a/celery/app/defaults.py b/celery/app/defaults.py index a150870a9..85edbcb4e 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -123,12 +123,12 @@ def __repr__(self): backend_options=Option({}, type='dict'), ), cassandra=Namespace( - column_family=Option(type='string'), entry_ttl=Option(type="float"), keyspace=Option(type='string'), port=Option(type="string"), read_consistency=Option(type='string'), servers=Option(type='list'), + table=Option(type='string'), write_consistency=Option(type='string'), ), chord=Namespace( diff --git a/celery/backends/__init__.py b/celery/backends/__init__.py index e214a9129..eec585227 100644 --- a/celery/backends/__init__.py +++ b/celery/backends/__init__.py @@ -30,7 +30,6 @@ 'db': 'celery.backends.database:DatabaseBackend', 'database': 'celery.backends.database:DatabaseBackend', 'cassandra': 'celery.backends.cassandra:CassandraBackend', - 'new_cassandra': 'celery.backends.new_cassandra:CassandraBackend', 'couchbase': 'celery.backends.couchbase:CouchBaseBackend', 'couchdb': 'celery.backends.couchdb:CouchDBBackend', 'riak': 'celery.backends.riak:RiakBackend', diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index 991c73d69..e7ee1dd35 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -3,59 +3,83 @@ celery.backends.cassandra ~~~~~~~~~~~~~~~~~~~~~~~~~ - Apache Cassandra result store backend. + Apache Cassandra result store backend using DataStax driver """ from __future__ import absolute_import +import sys try: # pragma: no cover - import pycassa - from thrift import Thrift - C = pycassa.cassandra.ttypes + import cassandra + import cassandra.cluster except ImportError: # pragma: no cover - pycassa = None # noqa - -import socket -import time + cassandra = None # noqa from celery import states from celery.exceptions import ImproperlyConfigured -from celery.five import monotonic -from celery.utils import deprecated from celery.utils.log import get_logger - from .base import BaseBackend __all__ = ['CassandraBackend'] logger = get_logger(__name__) +E_NO_CASSANDRA = """ +You need to install the cassandra-driver library to +use the Cassandra backend. See https://github.com/datastax/python-driver +""" -class CassandraBackend(BaseBackend): - """Highly fault tolerant Cassandra backend. +Q_INSERT_RESULT = """ +INSERT INTO {table} ( + task_id, status, result, date_done, traceback, children) VALUES ( + %s, %s, %s, %s, %s, %s) {expires}; +""" + +Q_SELECT_RESULT = """ +SELECT status, result, date_done, traceback, children +FROM {table} +WHERE task_id=%s +LIMIT 1 +""" + +Q_CREATE_RESULT_TABLE = """ +CREATE TABLE {table} ( + task_id text, + status text, + result blob, + date_done timestamp, + traceback blob, + children blob, + PRIMARY KEY ((task_id), date_done) +) WITH CLUSTERING ORDER BY (date_done DESC); +""" - .. attribute:: servers +Q_EXPIRES = """ + USING TTL {0} +""" + +if sys.version_info[0] == 3: + def buf_t(x): + return bytes(x, 'utf8') +else: + buf_t = buffer # noqa - List of Cassandra servers with format: ``hostname:port``. + +class CassandraBackend(BaseBackend): + """Cassandra backend utilizing DataStax driver :raises celery.exceptions.ImproperlyConfigured: if - module :mod:`pycassa` is not available. + module :mod:`cassandra` is not available. """ - servers = [] - keyspace = None - column_family = None - detailed_mode = False - _retry_timeout = 300 - _retry_wait = 3 - supports_autoexpire = True - - @deprecated(description='The old cassandra backend', - deprecation='4.0', - removal='5.0', - alternative='Use the `new_cassandra` result backend instead') - def __init__(self, servers=None, keyspace=None, column_family=None, - cassandra_options=None, detailed_mode=False, **kwargs): + + #: List of Cassandra servers with format: ``hostname``. + servers = None + + supports_autoexpire = True # autoexpire supported via entry_ttl + + def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, + port=9042, **kwargs): """Initialize Cassandra backend. Raises :class:`celery.exceptions.ImproperlyConfigured` if @@ -64,129 +88,139 @@ def __init__(self, servers=None, keyspace=None, column_family=None, """ super(CassandraBackend, self).__init__(**kwargs) - if not pycassa: - raise ImproperlyConfigured( - 'You need to install the pycassa library to use the ' - 'Cassandra backend. See https://github.com/pycassa/pycassa') + if not cassandra: + raise ImproperlyConfigured(E_NO_CASSANDRA) conf = self.app.conf self.servers = (servers or - conf.get('cassandra_servers') or - self.servers) + conf.get('cassandra_servers', None)) + self.port = (port or + conf.get('cassandra_port', None)) self.keyspace = (keyspace or - conf.get('cassandra_keyspace') or - self.keyspace) - self.column_family = (column_family or - conf.get('cassandra_column_family') or - self.column_family) - self.cassandra_options = dict(conf.get('cassandra_options') or {}, - **cassandra_options or {}) - self.detailed_mode = (detailed_mode or - conf.get('cassandra_detailed_mode') or - self.detailed_mode) + conf.get('cassandra_keyspace', None)) + self.table = (table or + conf.get('cassandra_table', None)) + + if not self.servers or not self.keyspace or not self.table: + raise ImproperlyConfigured('Cassandra backend not configured.') + + expires = (entry_ttl or conf.get('cassandra_entry_ttl', None)) + + self.cqlexpires = (Q_EXPIRES.format(expires) + if expires is not None else '') + read_cons = conf.get('cassandra_read_consistency') or 'LOCAL_QUORUM' write_cons = conf.get('cassandra_write_consistency') or 'LOCAL_QUORUM' - try: - self.read_consistency = getattr(pycassa.ConsistencyLevel, - read_cons) - except AttributeError: - self.read_consistency = pycassa.ConsistencyLevel.LOCAL_QUORUM - try: - self.write_consistency = getattr(pycassa.ConsistencyLevel, - write_cons) - except AttributeError: - self.write_consistency = pycassa.ConsistencyLevel.LOCAL_QUORUM - - if not self.servers or not self.keyspace or not self.column_family: - raise ImproperlyConfigured( - 'Cassandra backend not configured.') - - self._column_family = None - - def _retry_on_error(self, fun, *args, **kwargs): - ts = monotonic() + self._retry_timeout - while 1: - try: - return fun(*args, **kwargs) - except (pycassa.InvalidRequestException, - pycassa.TimedOutException, - pycassa.UnavailableException, - pycassa.AllServersUnavailable, - socket.error, - socket.timeout, - Thrift.TException) as exc: - if monotonic() > ts: - raise - logger.warning('Cassandra error: %r. Retrying...', exc) - time.sleep(self._retry_wait) - - def _get_column_family(self): - if self._column_family is None: - conn = pycassa.ConnectionPool(self.keyspace, - server_list=self.servers, - **self.cassandra_options) - self._column_family = pycassa.ColumnFamily( - conn, self.column_family, - read_consistency_level=self.read_consistency, - write_consistency_level=self.write_consistency, - ) - return self._column_family + + self.read_consistency = getattr( + cassandra.ConsistencyLevel, read_cons, + cassandra.ConsistencyLevel.LOCAL_QUORUM, + ) + self.write_consistency = getattr( + cassandra.ConsistencyLevel, write_cons, + cassandra.ConsistencyLevel.LOCAL_QUORUM, + ) + + self._connection = None + self._session = None + self._write_stmt = None + self._read_stmt = None def process_cleanup(self): - if self._column_family is not None: - self._column_family = None + if self._connection is not None: + self._connection = None + if self._session is not None: + self._session.shutdown() + self._session = None + + def _get_connection(self, write=False): + """Prepare the connection for action + + :param write: bool - are we a writer? + + """ + if self._connection is None: + try: + self._connection = cassandra.cluster.Cluster(self.servers, + port=self.port) + self._session = self._connection.connect(self.keyspace) + + # We are forced to do concatenation below, as formatting would + # blow up on superficial %s that will be processed by Cassandra + self._write_stmt = cassandra.query.SimpleStatement( + Q_INSERT_RESULT.format( + table=self.table, expires=self.cqlexpires), + ) + self._write_stmt.consistency_level = self.write_consistency + + self._read_stmt = cassandra.query.SimpleStatement( + Q_SELECT_RESULT.format(table=self.table), + ) + self._read_stmt.consistency_level = self.read_consistency + + if write: + # Only possible writers "workers" are allowed to issue + # CREATE TABLE. This is to prevent conflicting situations + # where both task-creator and task-executor would issue it + # at the same time. + + # Anyway; if you're doing anything critical, you should + # have created this table in advance, in which case + # this query will be a no-op (AlreadyExists) + self._make_stmt = cassandra.query.SimpleStatement( + Q_CREATE_RESULT_TABLE.format(table=self.table), + ) + self._make_stmt.consistency_level = self.write_consistency + try: + self._session.execute(self._make_stmt) + except cassandra.AlreadyExists: + pass + + except cassandra.OperationTimedOut: + # a heavily loaded or gone Cassandra cluster failed to respond. + # leave this class in a consistent state + self._connection = None + if self._session is not None: + self._session.shutdown() + + raise # we did fail after all - reraise def _store_result(self, task_id, result, status, traceback=None, request=None, **kwargs): """Store return value and status of an executed task.""" + self._get_connection(write=True) - def _do_store(): - cf = self._get_column_family() - date_done = self.app.now() - meta = {'status': status, - 'date_done': date_done.strftime('%Y-%m-%dT%H:%M:%SZ'), - 'traceback': self.encode(traceback), - 'result': self.encode(result), - 'children': self.encode( - self.current_task_children(request), - )} - if self.detailed_mode: - cf.insert( - task_id, {date_done: self.encode(meta)}, ttl=self.expires, - ) - else: - cf.insert(task_id, meta, ttl=self.expires) - - return self._retry_on_error(_do_store) + self._session.execute(self._write_stmt, ( + task_id, + status, + buf_t(self.encode(result)), + self.app.now(), + buf_t(self.encode(traceback)), + buf_t(self.encode(self.current_task_children(request))) + )) def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" + self._get_connection() - def _do_get(): - cf = self._get_column_family() - try: - if self.detailed_mode: - row = cf.get(task_id, column_reversed=True, column_count=1) - return self.decode(list(row.values())[0]) - else: - obj = cf.get(task_id) - return self.meta_from_decoded({ - 'task_id': task_id, - 'status': obj['status'], - 'result': self.decode(obj['result']), - 'date_done': obj['date_done'], - 'traceback': self.decode(obj['traceback']), - 'children': self.decode(obj['children']), - }) - except (KeyError, pycassa.NotFoundException): - return {'status': states.PENDING, 'result': None} - - return self._retry_on_error(_do_get) + res = self._session.execute(self._read_stmt, (task_id, )) + if not res: + return {'status': states.PENDING, 'result': None} + + status, result, date_done, traceback, children = res[0] + + return self.meta_from_decoded({ + 'task_id': task_id, + 'status': status, + 'result': self.decode(result), + 'date_done': date_done.strftime('%Y-%m-%dT%H:%M:%SZ'), + 'traceback': self.decode(traceback), + 'children': self.decode(children), + }) def __reduce__(self, args=(), kwargs={}): kwargs.update( dict(servers=self.servers, keyspace=self.keyspace, - column_family=self.column_family, - cassandra_options=self.cassandra_options)) + table=self.table)) return super(CassandraBackend, self).__reduce__(args, kwargs) diff --git a/celery/backends/new_cassandra.py b/celery/backends/new_cassandra.py deleted file mode 100644 index b25d234ce..000000000 --- a/celery/backends/new_cassandra.py +++ /dev/null @@ -1,226 +0,0 @@ -# -* coding: utf-8 -*- -""" - celery.backends.new_cassandra - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Apache Cassandra result store backend using DataStax driver - -""" -from __future__ import absolute_import - -import sys -try: # pragma: no cover - import cassandra - import cassandra.cluster -except ImportError: # pragma: no cover - cassandra = None # noqa - -from celery import states -from celery.exceptions import ImproperlyConfigured -from celery.utils.log import get_logger -from .base import BaseBackend - -__all__ = ['CassandraBackend'] - -logger = get_logger(__name__) - -E_NO_CASSANDRA = """ -You need to install the cassandra-driver library to -use the Cassandra backend. See https://github.com/datastax/python-driver -""" - -Q_INSERT_RESULT = """ -INSERT INTO {table} ( - task_id, status, result, date_done, traceback, children) VALUES ( - %s, %s, %s, %s, %s, %s) {expires}; -""" - -Q_SELECT_RESULT = """ -SELECT status, result, date_done, traceback, children -FROM {table} -WHERE task_id=%s -LIMIT 1 -""" - -Q_CREATE_RESULT_TABLE = """ -CREATE TABLE {table} ( - task_id text, - status text, - result blob, - date_done timestamp, - traceback blob, - children blob, - PRIMARY KEY ((task_id), date_done) -) WITH CLUSTERING ORDER BY (date_done DESC); -""" - -Q_EXPIRES = """ - USING TTL {0} -""" - -if sys.version_info[0] == 3: - def buf_t(x): - return bytes(x, 'utf8') -else: - buf_t = buffer # noqa - - -class CassandraBackend(BaseBackend): - """Cassandra backend utilizing DataStax driver - - :raises celery.exceptions.ImproperlyConfigured: if - module :mod:`cassandra` is not available. - - """ - - #: List of Cassandra servers with format: ``hostname``. - servers = None - - supports_autoexpire = True # autoexpire supported via entry_ttl - - def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, - port=9042, **kwargs): - """Initialize Cassandra backend. - - Raises :class:`celery.exceptions.ImproperlyConfigured` if - the :setting:`cassandra_servers` setting is not set. - - """ - super(CassandraBackend, self).__init__(**kwargs) - - if not cassandra: - raise ImproperlyConfigured(E_NO_CASSANDRA) - - conf = self.app.conf - self.servers = (servers or - conf.get('cassandra_servers', None)) - self.port = (port or - conf.get('cassandra_port', None)) - self.keyspace = (keyspace or - conf.get('cassandra_keyspace', None)) - self.table = (table or - conf.get('cassandra_table', None)) - - if not self.servers or not self.keyspace or not self.table: - raise ImproperlyConfigured('Cassandra backend not configured.') - - expires = (entry_ttl or conf.get('cassandra_entry_ttl', None)) - - self.cqlexpires = (Q_EXPIRES.format(expires) - if expires is not None else '') - - read_cons = conf.get('cassandra_read_consistency') or 'LOCAL_QUORUM' - write_cons = conf.get('cassandra_write_consistency') or 'LOCAL_QUORUM' - - self.read_consistency = getattr( - cassandra.ConsistencyLevel, read_cons, - cassandra.ConsistencyLevel.LOCAL_QUORUM, - ) - self.write_consistency = getattr( - cassandra.ConsistencyLevel, write_cons, - cassandra.ConsistencyLevel.LOCAL_QUORUM, - ) - - self._connection = None - self._session = None - self._write_stmt = None - self._read_stmt = None - - def process_cleanup(self): - if self._connection is not None: - self._connection = None - if self._session is not None: - self._session.shutdown() - self._session = None - - def _get_connection(self, write=False): - """Prepare the connection for action - - :param write: bool - are we a writer? - - """ - if self._connection is None: - try: - self._connection = cassandra.cluster.Cluster(self.servers, - port=self.port) - self._session = self._connection.connect(self.keyspace) - - # We are forced to do concatenation below, as formatting would - # blow up on superficial %s that will be processed by Cassandra - self._write_stmt = cassandra.query.SimpleStatement( - Q_INSERT_RESULT.format( - table=self.table, expires=self.cqlexpires), - ) - self._write_stmt.consistency_level = self.write_consistency - - self._read_stmt = cassandra.query.SimpleStatement( - Q_SELECT_RESULT.format(table=self.table), - ) - self._read_stmt.consistency_level = self.read_consistency - - if write: - # Only possible writers "workers" are allowed to issue - # CREATE TABLE. This is to prevent conflicting situations - # where both task-creator and task-executor would issue it - # at the same time. - - # Anyway; if you're doing anything critical, you should - # have created this table in advance, in which case - # this query will be a no-op (AlreadyExists) - self._make_stmt = cassandra.query.SimpleStatement( - Q_CREATE_RESULT_TABLE.format(table=self.table), - ) - self._make_stmt.consistency_level = self.write_consistency - try: - self._session.execute(self._make_stmt) - except cassandra.AlreadyExists: - pass - - except cassandra.OperationTimedOut: - # a heavily loaded or gone Cassandra cluster failed to respond. - # leave this class in a consistent state - self._connection = None - if self._session is not None: - self._session.shutdown() - - raise # we did fail after all - reraise - - def _store_result(self, task_id, result, status, - traceback=None, request=None, **kwargs): - """Store return value and status of an executed task.""" - self._get_connection(write=True) - - self._session.execute(self._write_stmt, ( - task_id, - status, - buf_t(self.encode(result)), - self.app.now(), - buf_t(self.encode(traceback)), - buf_t(self.encode(self.current_task_children(request))) - )) - - def _get_task_meta_for(self, task_id): - """Get task metadata for a task by id.""" - self._get_connection() - - res = self._session.execute(self._read_stmt, (task_id, )) - if not res: - return {'status': states.PENDING, 'result': None} - - status, result, date_done, traceback, children = res[0] - - return self.meta_from_decoded({ - 'task_id': task_id, - 'status': status, - 'result': self.decode(result), - 'date_done': date_done.strftime('%Y-%m-%dT%H:%M:%SZ'), - 'traceback': self.decode(traceback), - 'children': self.decode(children), - }) - - def __reduce__(self, args=(), kwargs={}): - kwargs.update( - dict(servers=self.servers, - keyspace=self.keyspace, - table=self.table)) - return super(CassandraBackend, self).__reduce__(args, kwargs) diff --git a/celery/tests/backends/test_cassandra.py b/celery/tests/backends/test_cassandra.py index bfcbf3c87..161f4b4c8 100644 --- a/celery/tests/backends/test_cassandra.py +++ b/celery/tests/backends/test_cassandra.py @@ -1,74 +1,47 @@ from __future__ import absolute_import -import socket - from pickle import loads, dumps +from datetime import datetime from celery import states from celery.exceptions import ImproperlyConfigured from celery.tests.case import ( - AppCase, Mock, mock_module, depends_on_current_app, + AppCase, Mock, mock_module, depends_on_current_app ) +CASSANDRA_MODULES = ['cassandra', 'cassandra.cluster'] + class Object(object): pass -def install_exceptions(mod): - # py3k: cannot catch exceptions not ineheriting from BaseException. - - class NotFoundException(Exception): - pass - - class TException(Exception): - pass - - class InvalidRequestException(Exception): - pass - - class UnavailableException(Exception): - pass - - class TimedOutException(Exception): - pass - - class AllServersUnavailable(Exception): - pass - - mod.NotFoundException = NotFoundException - mod.TException = TException - mod.InvalidRequestException = InvalidRequestException - mod.TimedOutException = TimedOutException - mod.UnavailableException = UnavailableException - mod.AllServersUnavailable = AllServersUnavailable - - class test_CassandraBackend(AppCase): def setup(self): self.app.conf.update( cassandra_servers=['example.com'], - cassandra_keyspace='keyspace', - cassandra_column_family='columns', + cassandra_keyspace='celery', + cassandra_table='task_results', ) - def test_init_no_pycassa(self): - with mock_module('pycassa'): + def test_init_no_cassandra(self): + """should raise ImproperlyConfigured when no python-driver + installed.""" + with mock_module(*CASSANDRA_MODULES): from celery.backends import cassandra as mod - prev, mod.pycassa = mod.pycassa, None + prev, mod.cassandra = mod.cassandra, None try: with self.assertRaises(ImproperlyConfigured): mod.CassandraBackend(app=self.app) finally: - mod.pycassa = prev + mod.cassandra = prev def test_init_with_and_without_LOCAL_QUROM(self): - with mock_module('pycassa'): + with mock_module(*CASSANDRA_MODULES): from celery.backends import cassandra as mod - mod.pycassa = Mock() - install_exceptions(mod.pycassa) - cons = mod.pycassa.ConsistencyLevel = Object() + mod.cassandra = Mock() + cons = mod.cassandra.ConsistencyLevel = Object() cons.LOCAL_QUORUM = 'foo' self.app.conf.cassandra_read_consistency = 'LOCAL_FOO' @@ -87,104 +60,76 @@ def test_init_with_and_without_LOCAL_QUROM(self): @depends_on_current_app def test_reduce(self): - with mock_module('pycassa'): + with mock_module(*CASSANDRA_MODULES): from celery.backends.cassandra import CassandraBackend self.assertTrue(loads(dumps(CassandraBackend(app=self.app)))) def test_get_task_meta_for(self): - with mock_module('pycassa'): + with mock_module(*CASSANDRA_MODULES): from celery.backends import cassandra as mod - mod.pycassa = Mock() - install_exceptions(mod.pycassa) - mod.Thrift = Mock() - install_exceptions(mod.Thrift) + mod.cassandra = Mock() x = mod.CassandraBackend(app=self.app) - Get_Column = x._get_column_family = Mock() - get_column = Get_Column.return_value = Mock() - get = get_column.get - META = get.return_value = { - 'task_id': 'task_id', - 'status': states.SUCCESS, - 'result': '1', - 'date_done': 'date', - 'traceback': '', - 'children': None, - } + x._connection = True + session = x._session = Mock() + execute = session.execute = Mock() + execute.return_value = [ + [states.SUCCESS, '1', datetime.now(), b'', b''] + ] x.decode = Mock() - x.detailed_mode = False - meta = x._get_task_meta_for('task_id') - self.assertEqual(meta['status'], states.SUCCESS) - - x.detailed_mode = True - row = get.return_value = Mock() - row.values.return_value = [Mock()] - x.decode.return_value = META meta = x._get_task_meta_for('task_id') self.assertEqual(meta['status'], states.SUCCESS) - x.decode.return_value = Mock() - x.detailed_mode = False - get.side_effect = KeyError() + x._session.execute.return_value = [] meta = x._get_task_meta_for('task_id') self.assertEqual(meta['status'], states.PENDING) - calls = [0] - end = [10] - - def work_eventually(*arg): - try: - if calls[0] > end[0]: - return META - raise socket.error() - finally: - calls[0] += 1 - get.side_effect = work_eventually - x._retry_timeout = 10 - x._retry_wait = 0.01 - meta = x._get_task_meta_for('task') - self.assertEqual(meta['status'], states.SUCCESS) - - x._retry_timeout = 0.1 - calls[0], end[0] = 0, 100 - with self.assertRaises(socket.error): - x._get_task_meta_for('task') - def test_store_result(self): - with mock_module('pycassa'): + with mock_module(*CASSANDRA_MODULES): from celery.backends import cassandra as mod - mod.pycassa = Mock() - install_exceptions(mod.pycassa) - mod.Thrift = Mock() - install_exceptions(mod.Thrift) - x = mod.CassandraBackend(app=self.app) - Get_Column = x._get_column_family = Mock() - cf = Get_Column.return_value = Mock() - x.detailed_mode = False - x._store_result('task_id', 'result', states.SUCCESS) - self.assertTrue(cf.insert.called) + mod.cassandra = Mock() - cf.insert.reset() - x.detailed_mode = True + x = mod.CassandraBackend(app=self.app) + x._connection = True + session = x._session = Mock() + session.execute = Mock() x._store_result('task_id', 'result', states.SUCCESS) - self.assertTrue(cf.insert.called) def test_process_cleanup(self): - with mock_module('pycassa'): + with mock_module(*CASSANDRA_MODULES): from celery.backends import cassandra as mod x = mod.CassandraBackend(app=self.app) - x._column_family = None x.process_cleanup() - x._column_family = True - x.process_cleanup() - self.assertIsNone(x._column_family) + self.assertIsNone(x._connection) + self.assertIsNone(x._session) - def test_get_column_family(self): - with mock_module('pycassa'): + def test_timeouting_cluster(self): + """ + Tests behaviour when Cluster.connect raises cassandra.OperationTimedOut + """ + with mock_module(*CASSANDRA_MODULES): from celery.backends import cassandra as mod - mod.pycassa = Mock() - install_exceptions(mod.pycassa) + + class OTOExc(Exception): + pass + + class VeryFaultyCluster(object): + def __init__(self, *args, **kwargs): + pass + + def connect(self, *args, **kwargs): + raise OTOExc() + + mod.cassandra = Mock() + mod.cassandra.OperationTimedOut = OTOExc + mod.cassandra.cluster = Mock() + mod.cassandra.cluster.Cluster = VeryFaultyCluster + x = mod.CassandraBackend(app=self.app) - self.assertTrue(x._get_column_family()) - self.assertIsNotNone(x._column_family) - self.assertIs(x._get_column_family(), x._column_family) + + with self.assertRaises(OTOExc): + x._store_result('task_id', 'result', states.SUCCESS) + self.assertIsNone(x._connection) + self.assertIsNone(x._session) + + x.process_cleanup() # should not raise diff --git a/celery/tests/backends/test_new_cassandra.py b/celery/tests/backends/test_new_cassandra.py deleted file mode 100644 index 6f83db3dc..000000000 --- a/celery/tests/backends/test_new_cassandra.py +++ /dev/null @@ -1,135 +0,0 @@ -from __future__ import absolute_import - -from pickle import loads, dumps -from datetime import datetime - -from celery import states -from celery.exceptions import ImproperlyConfigured -from celery.tests.case import ( - AppCase, Mock, mock_module, depends_on_current_app -) - -CASSANDRA_MODULES = ['cassandra', 'cassandra.cluster'] - - -class Object(object): - pass - - -class test_CassandraBackend(AppCase): - - def setup(self): - self.app.conf.update( - cassandra_servers=['example.com'], - cassandra_keyspace='celery', - cassandra_table='task_results', - ) - - def test_init_no_cassandra(self): - """should raise ImproperlyConfigured when no python-driver - installed.""" - with mock_module(*CASSANDRA_MODULES): - from celery.backends import new_cassandra as mod - prev, mod.cassandra = mod.cassandra, None - try: - with self.assertRaises(ImproperlyConfigured): - mod.CassandraBackend(app=self.app) - finally: - mod.cassandra = prev - - def test_init_with_and_without_LOCAL_QUROM(self): - with mock_module(*CASSANDRA_MODULES): - from celery.backends import new_cassandra as mod - mod.cassandra = Mock() - cons = mod.cassandra.ConsistencyLevel = Object() - cons.LOCAL_QUORUM = 'foo' - - self.app.conf.cassandra_read_consistency = 'LOCAL_FOO' - self.app.conf.cassandra_write_consistency = 'LOCAL_FOO' - - mod.CassandraBackend(app=self.app) - cons.LOCAL_FOO = 'bar' - mod.CassandraBackend(app=self.app) - - # no servers raises ImproperlyConfigured - with self.assertRaises(ImproperlyConfigured): - self.app.conf.cassandra_servers = None - mod.CassandraBackend( - app=self.app, keyspace='b', column_family='c', - ) - - @depends_on_current_app - def test_reduce(self): - with mock_module(*CASSANDRA_MODULES): - from celery.backends.new_cassandra import CassandraBackend - self.assertTrue(loads(dumps(CassandraBackend(app=self.app)))) - - def test_get_task_meta_for(self): - with mock_module(*CASSANDRA_MODULES): - from celery.backends import new_cassandra as mod - mod.cassandra = Mock() - x = mod.CassandraBackend(app=self.app) - x._connection = True - session = x._session = Mock() - execute = session.execute = Mock() - execute.return_value = [ - [states.SUCCESS, '1', datetime.now(), b'', b''] - ] - x.decode = Mock() - meta = x._get_task_meta_for('task_id') - self.assertEqual(meta['status'], states.SUCCESS) - - x._session.execute.return_value = [] - meta = x._get_task_meta_for('task_id') - self.assertEqual(meta['status'], states.PENDING) - - def test_store_result(self): - with mock_module(*CASSANDRA_MODULES): - from celery.backends import new_cassandra as mod - mod.cassandra = Mock() - - x = mod.CassandraBackend(app=self.app) - x._connection = True - session = x._session = Mock() - session.execute = Mock() - x._store_result('task_id', 'result', states.SUCCESS) - - def test_process_cleanup(self): - with mock_module(*CASSANDRA_MODULES): - from celery.backends import new_cassandra as mod - x = mod.CassandraBackend(app=self.app) - x.process_cleanup() - - self.assertIsNone(x._connection) - self.assertIsNone(x._session) - - def test_timeouting_cluster(self): - """ - Tests behaviour when Cluster.connect raises cassandra.OperationTimedOut - """ - with mock_module(*CASSANDRA_MODULES): - from celery.backends import new_cassandra as mod - - class OTOExc(Exception): - pass - - class VeryFaultyCluster(object): - def __init__(self, *args, **kwargs): - pass - - def connect(self, *args, **kwargs): - raise OTOExc() - - mod.cassandra = Mock() - mod.cassandra.OperationTimedOut = OTOExc - mod.cassandra.cluster = Mock() - mod.cassandra.cluster.Cluster = VeryFaultyCluster - - x = mod.CassandraBackend(app=self.app) - - with self.assertRaises(OTOExc): - x._store_result('task_id', 'result', states.SUCCESS) - self.assertIsNone(x._connection) - self.assertIsNone(x._session) - - x.process_cleanup() # should not raise diff --git a/docs/configuration.rst b/docs/configuration.rst index 301f3eba4..e6c603b1b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -374,9 +374,9 @@ Can be one of the following: Use `MongoDB`_ to store the results. See :ref:`conf-mongodb-result-backend`. -* new_cassandra - Use `Cassandra`_ to store the results, using newer database driver than _cassandra_. - See :ref:`conf-new_cassandra-result-backend`. +* cassandra + Use `Cassandra`_ to store the results. + See :ref:`conf-cassandra-result-backend`. * ironcache Use `IronCache`_ to store the results. @@ -742,10 +742,10 @@ Example configuration 'taskmeta_collection': 'my_taskmeta_collection', } -.. _conf-new_cassandra-result-backend: +.. _conf-cassandra-result-backend: -new_cassandra backend settings ------------------------------- +cassandra backend settings +-------------------------- .. note:: @@ -786,14 +786,14 @@ The keyspace in which to store the results. e.g.:: cassandra_keyspace = 'tasks_keyspace' -.. setting:: cassandra_column_family +.. setting:: cassandra_table -cassandra_column_family -~~~~~~~~~~~~~~~~~~~~~~~ +cassandra_table +~~~~~~~~~~~~~~~ The table (column family) in which to store the results. e.g.:: - cassandra_column_family = 'tasks' + cassandra_table = 'tasks' .. setting:: cassandra_read_consistency @@ -826,7 +826,7 @@ Example configuration cassandra_servers = ['localhost'] cassandra_keyspace = 'celery' - cassandra_column_family = 'task_results' + cassandra_table = 'tasks' cassandra_read_consistency = 'ONE' cassandra_write_consistency = 'ONE' cassandra_entry_ttl = 86400 diff --git a/docs/includes/installation.txt b/docs/includes/installation.txt index 4e6b9195f..3b4a669d7 100644 --- a/docs/includes/installation.txt +++ b/docs/includes/installation.txt @@ -78,9 +78,6 @@ Transports and Backends for using memcached as a result backend. :celery[cassandra]: - for using Apache Cassandra as a result backend with pycassa driver. - -:celery[new_cassandra]: for using Apache Cassandra as a result backend with DataStax driver. :celery[couchdb]: diff --git a/docs/internals/reference/celery.backends.new_cassandra.rst b/docs/internals/reference/celery.backends.new_cassandra.rst deleted file mode 100644 index e7696fa62..000000000 --- a/docs/internals/reference/celery.backends.new_cassandra.rst +++ /dev/null @@ -1,11 +0,0 @@ -================================================ - celery.backends.new_cassandra -================================================ - -.. contents:: - :local: -.. currentmodule:: celery.backends.new_cassandra - -.. automodule:: celery.backends.new_cassandra - :members: - :undoc-members: diff --git a/docs/internals/reference/index.rst b/docs/internals/reference/index.rst index 52611b186..16897b9d0 100644 --- a/docs/internals/reference/index.rst +++ b/docs/internals/reference/index.rst @@ -32,7 +32,6 @@ celery.backends.mongodb celery.backends.redis celery.backends.riak - celery.backends.new_cassandra celery.backends.cassandra celery.backends.couchbase celery.app.trace diff --git a/docs/whatsnew-4.0.rst b/docs/whatsnew-4.0.rst index 7a8e808e5..86b725486 100644 --- a/docs/whatsnew-4.0.rst +++ b/docs/whatsnew-4.0.rst @@ -101,8 +101,12 @@ Bla bla New Cassandra Backend ===================== -New Cassandra backend will be called new_cassandra and utilize python-driver. -Old backend is now deprecated. + +The new Cassandra backend utilizes the python-driver library. +Old backend is deprecated and everyone using cassandra is required to upgrade +to be using the new driver. + +# XXX What changed? Event Batching diff --git a/requirements/extras/cassandra.txt b/requirements/extras/cassandra.txt index a58d089a5..a94062dad 100644 --- a/requirements/extras/cassandra.txt +++ b/requirements/extras/cassandra.txt @@ -1 +1 @@ -pycassa +cassandra-driver \ No newline at end of file diff --git a/requirements/extras/new_cassandra.txt b/requirements/extras/new_cassandra.txt deleted file mode 100644 index a94062dad..000000000 --- a/requirements/extras/new_cassandra.txt +++ /dev/null @@ -1 +0,0 @@ -cassandra-driver \ No newline at end of file diff --git a/setup.py b/setup.py index 4a9d9679b..8af1a1e25 100644 --- a/setup.py +++ b/setup.py @@ -200,7 +200,6 @@ def extras(*p): 'eventlet', 'gevent', 'msgpack', 'yaml', 'redis', 'mongodb', 'sqs', 'couchdb', 'riak', 'beanstalk', 'zookeeper', 'zeromq', 'sqlalchemy', 'librabbitmq', 'pyro', 'slmq', - 'new_cassandra', ]) extras_require = dict((x, extras(x + '.txt')) for x in features) extra['extras_require'] = extras_require From cf247b28f5ee43f0a95e60e0add779ad5cc0ec49 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 4 Nov 2015 18:58:24 -0800 Subject: [PATCH 0789/1103] 4.0 release candidate 1 --- CONTRIBUTING.rst | 6 +++--- README.rst | 2 +- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index f7a02bd83..1b5dde68d 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -538,7 +538,7 @@ If you only want to test specific Python versions use the ``-e`` option: :: - $ tox -e py26 + $ tox -e 2.7 Building the documentation -------------------------- @@ -586,7 +586,7 @@ To not return a negative exit code when this command fails use the ``flakes`` target instead: :: - $ make flakes + $ make flakes§ API reference ~~~~~~~~~~~~~ @@ -619,7 +619,7 @@ Edit the file using your favorite editor: $ vim celery.worker.awesome.rst - # change every occurance of ``celery.schedules`` to + # change every occurrence of ``celery.schedules`` to # ``celery.worker.awesome`` diff --git a/README.rst b/README.rst index ce8a2cf3d..8622d7141 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ .. image:: http://cloud.github.com/downloads/celery/celery/celery_128.png -:Version: 4.0.0b1 (0today8) +:Version: 4.0.0rc1 (0today8) :Web: http://celeryproject.org/ :Download: http://pypi.python.org/pypi/celery/ :Source: http://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index 260a78738..e6d0b214a 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -18,7 +18,7 @@ ) SERIES = '0today8' -VERSION = version_info_t(4, 0, 0, 'b1', '') +VERSION = version_info_t(4, 0, 0, 'rc1', '') __version__ = '{0.major}.{0.minor}.{0.micro}{0.releaselevel}'.format(VERSION) __author__ = 'Ask Solem' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 7986c52a4..69ea7a113 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 4.0.0b1 (0today8) +:Version: 4.0.0rc1 (0today8) :Web: http://celeryproject.org/ :Download: http://pypi.python.org/pypi/celery/ :Source: http://github.com/celery/celery/ From cea5140e9ee4bc1ee5164dc40766c6f85784d3ec Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 4 Nov 2015 18:58:50 -0800 Subject: [PATCH 0790/1103] Forgot to commit this --- celery/worker/strategy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index b135ace1a..a753e78dc 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -101,6 +101,7 @@ def task_message_handler(message, body, ack, reject, callbacks, 'task-received', uuid=req.id, name=req.name, args=req.argsrepr, kwargs=req.kwargsrepr, + root_id=req.root_id, parent_id=req.parent_id, retries=req.request_dict.get('retries', 0), eta=req.eta and req.eta.isoformat(), expires=req.expires and req.expires.isoformat(), From f3b1f7fd54012f30105d44dd343d814fad26d22d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 5 Nov 2015 12:29:45 -0800 Subject: [PATCH 0791/1103] Updates whatsnew-4.0 document --- docs/internals/deprecation.rst | 100 ------- docs/whatsnew-4.0.rst | 513 +++++++++++++++++++++++++++++++-- 2 files changed, 483 insertions(+), 130 deletions(-) diff --git a/docs/internals/deprecation.rst b/docs/internals/deprecation.rst index 817aa9aa6..23df5be0b 100644 --- a/docs/internals/deprecation.rst +++ b/docs/internals/deprecation.rst @@ -7,106 +7,6 @@ .. contents:: :local: -.. _deprecations-v4.0: - -Removals for version 4.0 -======================== - -- Module ``celery.task.trace`` has been renamed to ``celery.app.trace`` - as the ``celery.task`` package is being phased out. The compat module - will be removed in version 4.0 so please change any import from:: - - from celery.task.trace import … - - to:: - - from celery.app.trace import … - -- ``AsyncResult.serializable()`` and ``celery.result.from_serializable`` - will be removed. - - Use instead:: - - >>> tup = result.as_tuple() - >>> from celery.result import result_from_tuple - >>> result = result_from_tuple(tup) - -TaskSet -~~~~~~~ - -TaskSet has been renamed to group and TaskSet will be removed in version 4.0. - -Old:: - - >>> from celery.task import TaskSet - - >>> TaskSet(add.subtask((i, i)) for i in xrange(10)).apply_async() - -New:: - - >>> from celery import group - >>> group(add.s(i, i) for i in xrange(10))() - - -Magic keyword arguments -~~~~~~~~~~~~~~~~~~~~~~~ - -The magic keyword arguments accepted by tasks will be removed -in 4.0, so you should start rewriting any tasks -using the ``celery.decorators`` module and depending -on keyword arguments being passed to the task, -for example:: - - from celery.decorators import task - - @task() - def add(x, y, task_id=None): - print("My task id is %r" % (task_id,)) - -should be rewritten into:: - - from celery import task - - @task(bind=True) - def add(self, x, y): - print("My task id is {0.request.id}".format(self)) - -:mod:`celery.result` --------------------- - -- ``BaseAsyncResult`` -> ``AsyncResult``. - -- ``TaskSetResult`` -> ``GroupResult``. - -- ``TaskSetResult.total`` -> ``len(GroupResult)`` - -- ``TaskSetResult.taskset_id`` -> ``GroupResult.id`` - -:mod:`celery.loader` --------------------- - -- ``current_loader()`` -> ``current_app.loader`` - -- ``load_settings()`` -> ``current_app.conf`` - - -Settings --------- - -Logging Settings -~~~~~~~~~~~~~~~~ - -===================================== ===================================== -**Setting name** **Replace with** -===================================== ===================================== -``CELERYD_LOG_LEVEL`` :option:`--loglevel` -``CELERYD_LOG_FILE`` :option:`--logfile`` -``CELERYBEAT_LOG_LEVEL`` :option:`--loglevel` -``CELERYBEAT_LOG_FILE`` :option:`--loglevel`` -``CELERYMON_LOG_LEVEL`` :option:`--loglevel` -``CELERYMON_LOG_FILE`` :option:`--loglevel`` -===================================== ===================================== - .. _deprecations-v5.0: Removals for version 5.0 diff --git a/docs/whatsnew-4.0.rst b/docs/whatsnew-4.0.rst index 86b725486..1f24fbd14 100644 --- a/docs/whatsnew-4.0.rst +++ b/docs/whatsnew-4.0.rst @@ -55,12 +55,14 @@ Dropped support for Python 2.6 Celery now requires Python 2.7 or later. +Dropped support for Python 3.3 +------------------------------ + +Celery now requires Python3 3.4 or later. + JSON is now the default serializer ---------------------------------- -Using one logfile per process by default ----------------------------------------- - The Task base class no longer automatically register tasks ---------------------------------------------------------- @@ -70,6 +72,51 @@ The metaclass has been removed blah blah Arguments now verified when calling a task ------------------------------------------ +Redis Events not backward compatible +------------------------------------ + +The Redis ``fanout_patterns`` and ``fanout_prefix`` transport +options are now enabled by default, which means that workers +running 4.0 cannot see workers running 3.1 and vice versa. + +They should still execute tasks as normally, so this is only +related to monitoring events. + +To avoid this situation you can reconfigure the 3.1 workers (and clients) +to enable these settings before you mix them with workers and clients +running 4.x: + +.. code-block:: python + + BROKER_TRANSPORT_OPTIONS = { + 'fanout_patterns': True, + 'fanout_prefix': True, + } + +Django: Autodiscover no longer takes arguments. +----------------------------------------------- + +# e436454d02dcbba4f4410868ad109c54047c2c15 + +Old command-line programs removed +--------------------------------- + +Installing Celery will no longer install the ``celeryd``, +``celerybeat`` and ``celeryd-multi`` programs. + +This was announced with the release of Celery 3.1, but you may still +have scripts pointing to the old names, so make sure you update them +to use the new umbrella command. + ++-------------------+--------------+-------------------------------------+ +| Program | New Status | Replacement | ++===================+==============+=====================================+ +| ``celeryd`` | **REMOVED** | :program:`celery worker` | ++-------------------+--------------+-------------------------------------+ +| ``celerybeat`` | **REMOVED** | :program:`celery beat` | ++-------------------+--------------+-------------------------------------+ +| ``celeryd-multi`` | **REMOVED** | :program:`celery multi` | ++-------------------+--------------+-------------------------------------+ .. _v320-news: @@ -79,6 +126,8 @@ News New Task Message Protocol ========================= +# e71652d384b1b5df2a4e6145df9f0efb456bc71c + ``TaskProducer`` replaced by ``app.amqp.create_task_message`` and ``app.amqp.send_task_message``. @@ -86,10 +135,109 @@ New Task Message Protocol - Worker stores results for internal errors like ``ContentDisallowed``, and exceptions occurring outside of the task function. +- Worker stores results and sends monitoring events for unknown task names + +- shadow + +- argsrepr + +- Support for very long chains + +- parent_id / root_id + + +Prefork: Tasks now log from the child process +============================================= + +Logging of task success/failure now happens from the child process +actually executing the task, which means that logging utilities +like Sentry can get full information about tasks that fail, including +variables in the traceback. + +Prefork: One logfile per child process +====================================== + +Init scrips and :program:`celery multi` now uses the `%I` logfile format +option (e.g. :file:`/var/log/celery/%n%I.log`) to ensure each child +process has a separate log file to avoid race conditions. + +You are encouraged to upgrade your init scripts and multi arguments +to do so also. Canvas Refactor =============== +# BLALBLABLA +d79dcd8e82c5e41f39abd07ffed81ca58052bcd2 +1e9dd26592eb2b93f1cb16deb771cfc65ab79612 +e442df61b2ff1fe855881c1e2ff9acc970090f54 +0673da5c09ac22bdd49ba811c470b73a036ee776 + +- Now unrolls groups within groups into a single group (Issue #1509). +- chunks/map/starmap tasks now routes based on the target task +- chords and chains can now be immutable. +- Fixed bug where serialized signature were not converted back into + signatures (Issue #2078) + + Fix contributed by Ross Deane. + +- Fixed problem where chains and groups did not work when using JSON + serialization (Issue #2076). + + Fix contributed by Ross Deane. + +- Creating a chord no longer results in multiple values for keyword + argument 'task_id'" (Issue #2225). + + Fix contributed by Aneil Mallavarapu + +- Fixed issue where the wrong result is returned when a chain + contains a chord as the penultimate task. + + Fix contributed by Aneil Mallavarapu + +- Special case of ``group(A.s() | group(B.s() | C.s()))`` now works. + +- Chain: Fixed bug with incorrect id set when a subtask is also a chain. + +Schedule tasks based on sunrise, sunset, dawn and dusk. +======================================================= + +See :ref:`beat-solar` for more information. + +Contributed by Mark Parncutt. + +App can now configure periodic tasks +==================================== + +# bc18d0859c1570f5eb59f5a969d1d32c63af764b +# 132d8d94d38f4050db876f56a841d5a5e487b25b + +RabbitMQ Priority queue support +=============================== + +# 1d4cbbcc921aa34975bde4b503b8df9c2f1816e0 + +Contributed by Gerald Manipon. + +Prefork: Limits for child process resident memory size. +======================================================= + +This version introduces the new :setting:`worker_max_memory_per_child` setting, +which BLA BLA BLA + +# 5cae0e754128750a893524dcba4ae030c414de33 + +Contributed by Dave Smith. + +Redis: New optimized chord join implementation. +=============================================== + +This was an experimental feature introduced in Celery 3.1, +but is now enabled by default. + +?new_join BLABLABLA + Riak Result Backend =================== @@ -99,6 +247,11 @@ Bla bla - blah blah +CouchDB Result Backend +====================== + +Contributed by Nathan Van Gheem + New Cassandra Backend ===================== @@ -112,35 +265,53 @@ to be using the new driver. Event Batching ============== -Events are now buffered in the worker and sent as a list +Events are now buffered in the worker and sent as a list, and +events are sent as transient messages by default so that they are not written +to disk by RabbitMQ. + +03399b4d7c26fb593e61acf34f111b66b340ba4e Task.replace ============ - Task.replace changed, removes Task.replace_in_chord. - The two methods had almost the same functionality, but the old Task.replace - would force the new task to inherit the callbacks/errbacks of the existing - task. +Task.replace changed, removes Task.replace_in_chord. + +The two methods had almost the same functionality, but the old Task.replace +would force the new task to inherit the callbacks/errbacks of the existing +task. - If you replace a node in a tree, then you would not expect the new node to - inherit the children of the old node, so this seems like unexpected - behavior. +If you replace a node in a tree, then you would not expect the new node to +inherit the children of the old node, so this seems like unexpected +behavior. - So self.replace(sig) now works for any task, in addition sig can now - be a group. +So self.replace(sig) now works for any task, in addition sig can now +be a group. - Groups are automatically converted to a chord, where the callback - will "accumulate" the results of the group tasks. +Groups are automatically converted to a chord, where the callback +will "accumulate" the results of the group tasks. - A new builtin task (`celery.accumulate` was added for this purpose) +A new builtin task (`celery.accumulate` was added for this purpose) - Closes #817 +Closes #817 Optimized Beat implementation ============================= +heapq +20340d79b55137643d5ac0df063614075385daaa + +Contributed by Ask Solem and Alexander Koshelev. + + +Task Autoretry Decorator +======================== + +75246714dd11e6c463b9dc67f4311690643bff24 + +Contributed by Dmitry Malinovsky. + In Other News ------------- @@ -155,21 +326,182 @@ In Other News - **Programs**: ``%n`` format for :program:`celery multi` is now synonym with ``%N`` to be consistent with :program:`celery worker`. -- **Programs**: celery inspect/control now supports --json argument +- **Programs**: celery inspect/control now supports ``--json`` argument to + give output in json format. + +- **Programs**: :program:`celery inspect registered` now ignores built-in + tasks. + +- **Programs**: New :program:`celery logtool`: Utility for filtering and parsing + celery worker logfiles + +- **Redis Transport**: The Redis transport now supports the + :setting:`broker_use_ssl` option. -- **Programs**: :program:`celery logtool`: Utility for filtering and parsing celery worker logfiles +- **Worker**: Worker now only starts the remote control command consumer if the + broker transport used actually supports them. - **Worker**: Gossip now sets ``x-message-ttl`` for event queue to heartbeat_interval s. - (Iss ue #2005). + (Issue #2005). -- **App**: New signals +- **Worker**: Now preserves exit code (Issue #2024). + +- **Worker**: Loglevel for unrecoverable errors changed from ``error`` to + ``critical``. + +- **Worker**: Improved rate limiting accuracy. + +- **Worker**: Account for missing timezone information in task expires field. + + Fix contributed by Albert Wang. + +- **Worker**: The worker no longer has a ``Queues`` bootsteps, as it is now + superfluous. + +- **Tasks**: New :setting:`task_reject_on_worker_lost` setting, and + :attr:`~@Task.reject_on_worker_lost` task attribute decides what happens + when the child worker process executing a late ack task is terminated. + + Contributed by Michael Permana. + +- **App**: New signals for app configuration/finalization: - :data:`app.on_configure <@on_configure>` - :data:`app.on_after_configure <@on_after_configure>` - :data:`app.on_after_finalize <@on_after_finalize>` +- **Task**: New task signals for rejected task messages: + + - :data:`celery.signals.task_rejected`. + - :data:`celery.signals.task_unknown`. + +- **Events**: Event messages now uses the RabbitMQ ``x-message-ttl`` option + to ensure older event messages are discarded. + + The default is 5 seconds, but can be changed using the + :setting:`event_queue_ttl` setting. + +- **Events**: Event monitors now sets the :setting:`event_queue_expires` + setting by default. + + The queues will now expire after 60 seconds after the monitor stops + consuming from it. + - **Canvas**: ``chunks``/``map``/``starmap`` are now routed based on the target task. +- **Canvas**: ``Signature.link`` now works when argument is scalar (not a list) + (Issue #2019). + +- **App**: The application can now change how task names are generated using + the :meth:`~@gen_task_name` method. + + Contributed by Dmitry Malinovsky. + +- **Tasks**: ``Task.subtask`` renamed to ``Task.signature`` with alias. + +- **Tasks**: ``Task.subtask_from_request`` renamed to + ``Task.signature_from_request`` with alias. + +- **Tasks**: The ``delivery_mode`` attribute for :class:`kombu.Queue` is now + respected (Issue #1953). + +- **Tasks**: Routes in :setting:`task-routes` can now specify a + :class:`~kombu.Queue` instance directly. + + Example: + + .. code-block:: python + + task_routes = {'proj.tasks.add': {'queue': Queue('add')}} + +- **Tasks**: ``AsyncResult`` now raises :exc:`ValueError` if task_id is None. + (Issue #1996). + +- **Tasks**: ``result.get()`` now supports an ``on_message`` argument to set a + callback to be called for every message received. + +- **Tasks**: New abstract classes added: + + - :class:`~celery.utils.abstract.CallableTask` + + Looks like a task. + + - :class:`~celery.utils.abstract.CallableSignature` + + Looks like a task signature. + +- **Programs**: :program:`celery multi` now passes through `%i` and `%I` log + file formats. + +- **Programs**: A new command line option :option:``--executable`` is now + available for daemonizing programs. + + Contributed by Bert Vanderbauwhede. + +- **Programs**: :program:`celery worker` supports new + :option:`--prefetch-multiplier` option. + + Contributed by Mickaël Penhard. + +- **Prefork**: Prefork pool now uses ``poll`` instead of ``select`` where + available (Issue #2373). + +- **Tasks**: New :setting:`email_charset` setting allows for changing + the charset used for outgoing error emails. + + Contributed by Vladimir Gorbunov. + +- **Worker**: Now respects :setting:`broker_connection_retry` setting. + + Fix contributed by Nat Williams. + +- **Worker**: Autoscale did not always update keepalive when scaling down. + + Fix contributed by Philip Garnero. + +- **General**: Dates are now always timezone aware even if + :setting:`enable_utc` is disabled (Issue #943). + + Fix contributed by Omer Katz. + +- **Result Backends**: The redis result backend now has a default socket + timeout of 5 seconds. + + The default can be changed using the new :setting:`redis_socket_timeout` + setting. + + Contributed by Raghuram Srinivasan. + +- **Result Backends**: RPC Backend result queues are now auto delete by + default (Issue #2001). + +- **Result Backends**: MongoDB now supports setting the + :setting:`result_serialzier` setting to ``bson`` to use the MongoDB + libraries own serializer. + + Contributed by Davide Quarta. + +- **Result Backends**: SQLAlchemy result backend now ignores all result + engine options when using NullPool (Issue #1930). + +- **Result Backends**: MongoDB URI handling has been improved to use + database name, user and password from the URI if provided. + + Contributed by Samuel Jaillet. + +- **Result Backends**: Fix problem with rpc/amqp backends where exception + was not deserialized properly with the json serializer (Issue #2518). + + Fix contributed by Allard Hoeve. + +- **General**: All Celery exceptions/warnings now inherit from common + :class:`~celery.exceptions.CeleryException`/:class:`~celery.exceptions.CeleryWarning`. + (Issue #2643). + +- **Tasks**: Task retry now also throws in eager mode. + + Fix contributed by Feanil Patel. + - Apps can now define how tasks are named (:meth:`@gen_task_name`). Contributed by Dmitry Malinovsky @@ -179,16 +511,143 @@ In Other News - Beat: ``Scheduler.Publisher``/``.publisher`` renamed to ``.Producer``/``.producer``. +Unscheduled Removals +==================== + +- The experimental :mod:`celery.contrib.methods` feature has been removed, + as there were far many bugs in the implementation to be useful. + +- The CentOS init scripts have been removed. + + These did not really add any features over the generic init scripts, + so you are encouraged to use them instead, or something like + ``supervisord``. + .. _v320-removals: Scheduled Removals ================== -- The module ``celery.task.trace`` has been removed as scheduled for this - version. +Modules +------- + +- Module ``celery.worker.job`` has been renamed to :mod:`celery.worker.request`. + + This was an internal module so should not have any effect. + It is now part of the public API so should not change again. + +- Module ``celery.task.trace`` has been renamed to ``celery.app.trace`` + as the ``celery.task`` package is being phased out. The compat module + will be removed in version 4.0 so please change any import from:: + + from celery.task.trace import … + + to:: + + from celery.app.trace import … + +- Old compatibility aliases in the :mod:`celery.loaders` module + has been removed. + + - Removed ``celery.loaders.current_loader()``, use: ``current_app.loader`` + + - Removed ``celery.loaders.load_settings()``, use: ``current_app.conf`` + +Result +------ + +- ``AsyncResult.serializable()`` and ``celery.result.from_serializable`` + has been removed: + + Use instead: + + .. code-block:: pycon + + >>> tup = result.as_tuple() + >>> from celery.result import result_from_tuple + >>> result = result_from_tuple(tup) + +- Removed ``BaseAsyncResult``, use ``AsyncResult`` for instance checks + instead. + +- Removed ``TaskSetResult``, use ``GroupResult`` instead. + + - ``TaskSetResult.total`` -> ``len(GroupResult)`` + + - ``TaskSetResult.taskset_id`` -> ``GroupResult.id`` + + +TaskSet +------- + +TaskSet has been renamed to group and TaskSet will be removed in version 4.0. + +Old:: -- Magic keyword arguments no longer supported. + >>> from celery.task import TaskSet + + >>> TaskSet(add.subtask((i, i)) for i in xrange(10)).apply_async() + +New:: + + >>> from celery import group + >>> group(add.s(i, i) for i in xrange(10))() + + +Magic keyword arguments +----------------------- + +Support for the very old magic keyword arguments accepted by tasks has finally +been in 4.0. + +If you are still using these you have to rewrite any task still +using the old ``celery.decorators`` module and depending +on keyword arguments being passed to the task, +for example:: + + from celery.decorators import task + + @task() + def add(x, y, task_id=None): + print("My task id is %r" % (task_id,)) + +should be rewritten into:: + + from celery import task + + @task(bind=True) + def add(self, x, y): + print("My task id is {0.request.id}".format(self)) + +Settings +-------- + +The following settings have been removed, and is no longer supported: + +Logging Settings +~~~~~~~~~~~~~~~~ + +===================================== ===================================== +**Setting name** **Replace with** +===================================== ===================================== +``CELERYD_LOG_LEVEL`` :option:`--loglevel` +``CELERYD_LOG_FILE`` :option:`--logfile`` +``CELERYBEAT_LOG_LEVEL`` :option:`--loglevel` +``CELERYBEAT_LOG_FILE`` :option:`--loglevel`` +``CELERYMON_LOG_LEVEL`` celerymon is deprecated, use flower. +``CELERYMON_LOG_FILE`` celerymon is deprecated, use flower. +``CELERYMON_LOG_FORMAT`` celerymon is deprecated, use flower. +===================================== ===================================== + +Task Settings +~~~~~~~~~~~~~~ + +===================================== ===================================== +**Setting name** **Replace with** +===================================== ===================================== +``CELERY_CHORD_PROPAGATES`` N/a +===================================== ===================================== .. _v320-deprecations: @@ -202,9 +661,3 @@ See the :ref:`deprecation-timeline`. Fixes ===== -.. _v320-internal: - -Internal changes -================ - -- Module ``celery.worker.job`` has been renamed to :mod:`celery.worker.request`. From e7b01149f17d14c2ff137655cc117b69229e1d17 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 5 Nov 2015 15:50:16 -0800 Subject: [PATCH 0792/1103] Make the redis_socket_timeout setting official (from 75ab5c3656c5fd04e6d86506cd4995a363813edd) --- celery/app/defaults.py | 1 + docs/configuration.rst | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 85edbcb4e..179a9ffbf 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -171,6 +171,7 @@ def __repr__(self): max_connections=Option(type='int'), password=Option(type='string'), port=Option(type='int'), + socket_timeout=Option(5.0, type='float'), ), result=Namespace( __old__=old_ns('celery_result'), diff --git a/docs/configuration.rst b/docs/configuration.rst index e6c603b1b..cdbb4ea57 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -692,6 +692,16 @@ redis_max_connections Maximum number of connections available in the Redis connection pool used for sending and retrieving results. +.. setting:: redis_socket_timeout + +redis_socket_timeout +~~~~~~~~~~~~~~~~~~~~ + +Socket timeout for connections to Redis from the result backend +in seconds (int/float) + +Default is 5 seconds. + .. _conf-mongodb-result-backend: MongoDB backend settings From 5f019dfa6f7d74978edb2cd4eed2b08537edba57 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 5 Nov 2015 18:03:55 -0800 Subject: [PATCH 0793/1103] [settings] Removes the CHORD_PROPAGATES setting --- celery/app/builtins.py | 19 +++++-------------- celery/app/defaults.py | 5 ----- celery/backends/base.py | 8 +++----- celery/canvas.py | 6 ++---- celery/tests/backends/test_base.py | 13 +++---------- 5 files changed, 13 insertions(+), 38 deletions(-) diff --git a/celery/app/builtins.py b/celery/app/builtins.py index cfe6cc884..53cf11925 100644 --- a/celery/app/builtins.py +++ b/celery/app/builtins.py @@ -54,20 +54,12 @@ def add_unlock_chord_task(app): from celery.exceptions import ChordError from celery.result import allow_join_result, result_from_tuple - default_propagate = app.conf.chord_propagates - @app.task(name='celery.chord_unlock', max_retries=None, shared=False, default_retry_delay=1, ignore_result=True, lazy=False, bind=True) - def unlock_chord(self, group_id, callback, interval=None, propagate=None, + def unlock_chord(self, group_id, callback, interval=None, max_retries=None, result=None, Result=app.AsyncResult, GroupResult=app.GroupResult, - result_from_tuple=result_from_tuple): - # if propagate is disabled exceptions raised by chord tasks - # will be sent as part of the result list to the chord callback. - # Since 3.1 propagate will be enabled by default, and instead - # the chord callback changes state to FAILURE with the - # exception set to ChordError. - propagate = default_propagate if propagate is None else propagate + result_from_tuple=result_from_tuple, **kwargs): if interval is None: interval = self.default_retry_delay @@ -93,7 +85,7 @@ def unlock_chord(self, group_id, callback, interval=None, propagate=None, callback = maybe_signature(callback, app=app) try: with allow_join_result(): - ret = j(timeout=3.0, propagate=propagate) + ret = j(timeout=3.0, propagate=True) except Exception as exc: try: culprit = next(deps._failed_join_report()) @@ -191,8 +183,7 @@ def add_chord_task(app): @app.task(name='celery.chord', bind=True, ignore_result=False, shared=False, lazy=False) def chord(self, header, body, partial_args=(), interval=None, - countdown=1, max_retries=None, propagate=None, - eager=False, **kwargs): + countdown=1, max_retries=None, eager=False, **kwargs): app = self.app # - convert back to group if serialized tasks = header.tasks if isinstance(header, group) else header @@ -202,5 +193,5 @@ def chord(self, header, body, partial_args=(), interval=None, body = maybe_signature(body, app=app) ch = _chord(header, body) return ch.run(header, body, partial_args, app, interval, - countdown, max_retries, propagate, **kwargs) + countdown, max_retries, **kwargs) return chord diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 179a9ffbf..7b08e7a9f 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -131,11 +131,6 @@ def __repr__(self): table=Option(type='string'), write_consistency=Option(type='string'), ), - chord=Namespace( - __old__=old_ns('celery_chord'), - - propagates=Option(True, type='bool'), - ), couchbase=Namespace( __old__=old_ns('celery_couchbase'), diff --git a/celery/backends/base.py b/celery/backends/base.py index 05cd82a9f..3f96fc5b2 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -359,7 +359,7 @@ def on_task_call(self, producer, task_id): def add_to_chord(self, chord_id, result): raise NotImplementedError('Backend does not support add_to_chord') - def on_chord_part_return(self, request, state, result, propagate=False): + def on_chord_part_return(self, request, state, result, **kwargs): pass def fallback_chord_unlock(self, group_id, body, result=None, @@ -553,12 +553,10 @@ def _apply_chord_incr(self, header, partial_args, group_id, body, return header(*partial_args, task_id=group_id, **fixed_options or {}) - def on_chord_part_return(self, request, state, result, propagate=None): + def on_chord_part_return(self, request, state, result, **kwargs): if not self.implements_incr: return app = self.app - if propagate is None: - propagate = app.conf.chord_propagates gid = request.group if not gid: return @@ -593,7 +591,7 @@ def on_chord_part_return(self, request, state, result, propagate=None): j = deps.join_native if deps.supports_native_join else deps.join try: with allow_join_result(): - ret = j(timeout=3.0, propagate=propagate) + ret = j(timeout=3.0, propagate=True) except Exception as exc: try: culprit = next(deps._failed_join_report()) diff --git a/celery/canvas.py b/celery/canvas.py index ff43f05d6..779fe715f 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -939,11 +939,9 @@ def __length_hint__(self): return sum(self._traverse_tasks(self.tasks, 1)) def run(self, header, body, partial_args, app=None, interval=None, - countdown=1, max_retries=None, propagate=None, eager=False, + countdown=1, max_retries=None, eager=False, task_id=None, **options): app = app or self._get_app(body) - propagate = (app.conf.chord_propagates - if propagate is None else propagate) group_id = uuid() root_id = body.options.get('root_id') body.chord_size = self.__length_hint__() @@ -960,7 +958,7 @@ def run(self, header, body, partial_args, app=None, interval=None, header, partial_args, group_id, body, interval=interval, countdown=countdown, options=options, max_retries=max_retries, - propagate=propagate, result=results) + result=results) bodyres.parent = parent return bodyres diff --git a/celery/tests/backends/test_base.py b/celery/tests/backends/test_base.py index 60f7a800d..d811bae59 100644 --- a/celery/tests/backends/test_base.py +++ b/celery/tests/backends/test_base.py @@ -328,24 +328,17 @@ def callback(result): def test_chord_part_return_propagate_set(self): with self._chord_part_context(self.b) as (task, deps, _): - self.b.on_chord_part_return( - task.request, 'SUCCESS', 10, propagate=True, - ) + self.b.on_chord_part_return(task.request, 'SUCCESS', 10) self.assertFalse(self.b.expire.called) deps.delete.assert_called_with() deps.join_native.assert_called_with(propagate=True, timeout=3.0) def test_chord_part_return_propagate_default(self): with self._chord_part_context(self.b) as (task, deps, _): - self.b.on_chord_part_return( - task.request, 'SUCCESS', 10, propagate=None, - ) + self.b.on_chord_part_return(task.request, 'SUCCESS', 10) self.assertFalse(self.b.expire.called) deps.delete.assert_called_with() - deps.join_native.assert_called_with( - propagate=self.b.app.conf.chord_propagates, - timeout=3.0, - ) + deps.join_native.assert_called_with(propagate=True, timeout=3.0) def test_chord_part_return_join_raises_internal(self): with self._chord_part_context(self.b) as (task, deps, callback): From 1faa01e04debfc4a154c37b9ed9bfdcd4ff1b62e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 5 Nov 2015 18:04:22 -0800 Subject: [PATCH 0794/1103] [docs] Document lower case setting changes --- celery/app/defaults.py | 10 ++- celery/app/task.py | 5 +- celery/bin/worker.py | 2 +- celery/tests/tasks/test_tasks.py | 4 +- celery/worker/__init__.py | 2 +- docs/configuration.rst | 145 +++++++++++++++++++++++++++++-- docs/whatsnew-4.0.rst | 69 ++++++++++++++- 7 files changed, 220 insertions(+), 17 deletions(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 7b08e7a9f..9f44884e6 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -221,7 +221,9 @@ def __repr__(self): default_queue=Option('celery'), default_rate_limit=Option(type='string'), default_routing_key=Option('celery'), - eager_propagates_exceptions=Option(False, type='bool'), + eager_propagates=Option( + False, type='bool', old={'celery_eager_propagates_exceptions'}, + ), ignore_result=Option(False, type='bool'), protocol=Option(1, type='int', old={'celery_task_protocol'}), publish_retry=Option( @@ -273,7 +275,7 @@ def __repr__(self): hijack_root_logger=Option(True, type='bool'), log_color=Option(type='bool'), log_format=Option(DEFAULT_PROCESS_LOG_FMT), - lost_wait=Option(10.0, type='float'), + lost_wait=Option(10.0, type='float', old={'celeryd_worker_lost_wait'}), max_memory_per_child=Option(type='int'), max_tasks_per_child=Option(type='int'), pool=Option(DEFAULT_POOL), @@ -286,7 +288,9 @@ def __repr__(self): redirect_stdouts_level=Option( 'WARNING', old={'celery_redirect_stdouts_level'}, ), - send_events=Option(False, type='bool'), + send_task_events=Option( + False, type='bool', old={'celeryd_send_events'}, + ), state_db=Option(), task_log_format=Option(DEFAULT_TASK_LOG_FMT), timer=Option(type='string'), diff --git a/celery/app/task.py b/celery/app/task.py index 23617d48c..bf2bd449f 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -667,7 +667,7 @@ def apply(self, args=None, kwargs=None, :param args: positional arguments passed on to the task. :param kwargs: keyword arguments passed on to the task. :keyword throw: Re-raise task exceptions. Defaults to - the :setting:`task_eager_propagates_exceptions` + the :setting:`task_eager_propagates` setting. :rtype :class:`celery.result.EagerResult`: @@ -684,8 +684,7 @@ def apply(self, args=None, kwargs=None, kwargs = kwargs or {} task_id = options.get('task_id') or uuid() retries = options.get('retries', 0) - throw = app.either('task_eager_propagates_exceptions', - options.pop('throw', None)) + throw = app.either('task_eager_propagates', options.pop('throw', None)) # Make sure we get the task instance, not class. task = app._tasks[self.name] diff --git a/celery/bin/worker.py b/celery/bin/worker.py index b1648c98d..914957dcd 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -242,7 +242,7 @@ def get_options(self): Option('--scheduler', dest='scheduler_cls'), Option('-S', '--statedb', default=conf.worker_state_db, dest='state_db'), - Option('-E', '--events', default=conf.worker_send_events, + Option('-E', '--events', default=conf.worker_send_task_events, action='store_true', dest='send_events'), Option('--time-limit', type='float', dest='task_time_limit', default=conf.task_time_limit), diff --git a/celery/tests/tasks/test_tasks.py b/celery/tests/tasks/test_tasks.py index 38ca84cba..eef8d118a 100644 --- a/celery/tests/tasks/test_tasks.py +++ b/celery/tests/tasks/test_tasks.py @@ -462,8 +462,8 @@ def test_apply_throw(self): with self.assertRaises(KeyError): self.raising.apply(throw=True) - def test_apply_with_task_eager_propagates_exceptions(self): - self.app.conf.task_eager_propagates_exceptions = True + def test_apply_with_task_eager_propagates(self): + self.app.conf.task_eager_propagates = True with self.assertRaises(KeyError): self.raising.apply() diff --git a/celery/worker/__init__.py b/celery/worker/__init__.py index 1ff4cb107..f038c01c1 100644 --- a/celery/worker/__init__.py +++ b/celery/worker/__init__.py @@ -358,7 +358,7 @@ def setup_defaults(self, concurrency=None, loglevel='WARN', logfile=None, self.logfile = logfile self.concurrency = either('worker_concurrency', concurrency) - self.send_events = either('worker_send_events', send_events) + self.send_events = either('worker_send_task_events', send_events) self.pool_cls = either('worker_pool', pool_cls) self.consumer_cls = either('worker_consumer', consumer_cls) self.timer_cls = either('worker_timer', timer_cls) diff --git a/docs/configuration.rst b/docs/configuration.rst index cdbb4ea57..31c80beae 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -34,6 +34,139 @@ It should contain all you need to run a basic Celery set-up. task_annotations = {'tasks.add': {'rate_limit': '10/s'}} + +.. _conf-old-settings-map: + +New lowercase settings +====================== + +Version 4.0 introduced new lower case settings and setting organization. + +The major difference between previous versions, apart from the lower case +names, are the renaming of some prefixes, like ``celerybeat_`` to ``beat_``, +``celeryd_`` to ``worker_``, and most of the top level ``celery_`` settings +have been moved into a new ``task_`` prefix. + +Celery will still be able to read old configuration files, so there is no +rush in moving to the new settings format. + +===================================== ============================================== +**Setting name** **Replace with** +===================================== ============================================== +``CELERY_ACCEPT_CONTENT`` :setting:`accept_content` +``ADMINS`` :setting:`admins` +``CELERY_ENABLE_UTC`` :setting:`enable_utc` +``CELERY_IMPORTS`` :setting:`imports` +``CELERY_INCLUDE`` :setting:`include` +``SERVER_EMAIL`` :setting:`server_email` +``CELERY_TIMEZONE`` :setting:`timezone` +``CELERYBEAT_MAX_LOOP_INTERVAL`` :setting:`beat_max_loop_interval` +``CELERYBEAT_SCHEDULE`` :setting:`beat_schedule` +``CELERYBEAT_SCHEDULER`` :setting:`beat_scheduler` +``CELERYBEAT_SCHEDULE_FILENAME`` :setting:`beat_schedule_filename` +``CELERYBEAT_SYNC_EVERY`` :setting:`beat_sync_every` +``BROKER_URL`` :setting:`broker_url` +``BROKER_TRANSPORT`` :setting:`broker_transport` +``BROKER_TRANSPORT_OPTIONS`` :setting:`broker_transport_options` +``BROKER_CONNECTION_TIMEOUT`` :setting:`broker_connection_timeout` +``BROKER_CONNECTION_RETRY`` :setting:`broker_connection_retry` +``BROKER_CONNECTION_MAX_RETRIES`` :setting:`broker_connection_max_retries` +``BROKER_FAILOVER_STRATEGY`` :setting:`broker_failover_strategy` +``BROKER_HEARTBEAT`` :setting:`broker_heartbeat` +``BROKER_LOGIN_METHOD`` :setting:`broker_login_method` +``BROKER_POOL_LIMIT`` :setting:`broker_pool_limit` +``BROKER_USE_SSL`` :setting:`broker_use_ssl` +``CELERY_CACHE_BACKEND`` :setting:`cache_backend` +``CELERY_CACHE_BACKEND_OPTIONS`` :setting:`cache_backend_options` +``CASSANDRA_COLUMN_FAMILY`` :setting:`cassandra_table` +``CASSANDRA_ENTRY_TTL`` :setting:`cassandra_entry_ttl` +``CASSANDRA_KEYSPACE`` :setting:`cassandra_keyspace` +``CASSANDRA_PORT`` :setting:`cassandra_port` +``CASSANDRA_READ_CONSISTENCY`` :setting:`cassandra_read_consistency` +``CASSANDRA_SERVERS`` :setting:`cassandra_servers` +``CASSANDRA_WRITE_CONSISTENCY`` :setting:`cassandra_write_consistency` +``CELERY_COUCHBASE_BACKEND_SETTINGS`` :setting:`couchbase_backend_settings` +``EMAIL_HOST`` :setting:`email_host` +``EMAIL_HOST_USER`` :setting:`email_host_user` +``EMAIL_HOST_PASSWORD`` :setting:`email_host_password` +``EMAIL_PORT`` :setting:`email_port` +``EMAIL_TIMEOUT`` :setting:`email_timeout` +``EMAIL_USE_SSL`` :setting:`email_use_ssl` +``EMAIL_USE_TLS`` :setting:`email_use_tls` +``CELERY_MONGODB_BACKEND_SETTINGS`` :setting:`mongodb_backend_settings` +``CELERY_EVENT_QUEUE_EXPIRES`` :setting:`event_queue_expires` +``CELERY_EVENT_QUEUE_TTL`` :setting:`event_queue_ttl` +``CELERY_EVENT_SERIALIZER`` :setting:`event_serializer` +``CELERY_REDIS_DB`` :setting:`redis_db` +``CELERY_REDIS_HOST`` :setting:`redis_host` +``CELERY_REDIS_MAX_CONNECTIONS`` :setting:`redis_max_connections` +``CELERY_REDIS_PASSWORD`` :setting:`redis_password` +``CELERY_REDIS_PORT`` :setting:`redis_port` +``CELERY_RESULT_BACKEND`` :setting:`result_backend` +``CELERY_MAX_CACHED_RESULTS`` :setting:`result_cache_max` +``CELERY_MESSAGE_COMPRESSION`` :setting:`result_compression` +``CELERY_RESULT_EXCHANGE`` :setting:`result_exchange` +``CELERY_RESULT_EXCHANGE_TYPE`` :setting:`result_exchange_type` +``CELERY_TASK_RESULT_EXPIRES`` :setting:`result_expires` +``CELERY_RESULT_PERSISTENT`` :setting:`result_persistent` +``CELERY_RESULT_SERIALIZER`` :setting:`result_serializer` +``CELERY_RESULT_DBURI`` :setting:`sqlalchemy_dburi` +``CELERY_RESULT_ENGINE_OPTIONS`` :setting:`sqlalchemy_engine_options` +``-*-_DB_SHORT_LIVED_SESSIONS`` :setting:`sqlalchemy_short_lived_sessions` +``CELERY_RESULT_DB_TABLE_NAMES`` :setting:`sqlalchemy_db_names` +``CELERY_SECURITY_CERTIFICATE`` :setting:`security_certificate` +``CELERY_SECURITY_CERT_STORE`` :setting:`security_cert_store` +``CELERY_SECURITY_KEY`` :setting:`security_key` +``CELERY_ACKS_LATE`` :setting:`task_acks_late` +``CELERY_ALWAYS_EAGER`` :setting:`task_always_eager` +``CELERY_ANNOTATIONS`` :setting:`task_annotations` +``CELERY_MESSAGE_COMPRESSION`` :setting:`task_compression` +``CELERY_CREATE_MISSING_QUEUES`` :setting:`task_create_missing_queues` +``CELERY_DEFAULT_DELIVERY_MODE`` :setting:`task_default_delivery_mode` +``CELERY_DEFAULT_EXCHANGE`` :setting:`task_default_exchange` +``CELERY_DEFAULT_EXCHANGE_TYPE`` :setting:`task_default_exchange_type` +``CELERY_DEFAULT_QUEUE`` :setting:`task_default_queue` +``CELERY_DEFAULT_RATE_LIMIT`` :setting:`task_default_rate_limit` +``CELERY_DEFAULT_ROUTING_KEY`` :setting:`task_default_routing_key` +``-"-_EAGER_PROPAGATES_EXCEPTIONS`` :setting:`task_eager_propagates` +``CELERY_IGNORE_RESULT`` :setting:`task_ignore_result` +``CELERY_TASK_PUBLISH_RETRY`` :setting:`task_publish_retry` +``CELERY_TASK_PUBLISH_RETRY_POLICY`` :setting:`task_publish_retry_policy` +``CELERY_QUEUES`` :setting:`task_queues` +``CELERY_ROUTES`` :setting:`task_routes` +``CELERY_SEND_TASK_ERROR_EMAILS`` :setting:`task_send_error_emails` +``CELERY_SEND_TASK_SENT_EVENT`` :setting:`task_send_sent_event` +``CELERY_TASK_SERIALIZER`` :setting:`task_serializer` +``CELERYD_TASK_SOFT_TIME_LIMIT`` :setting:`task_soft_time_limit` +``CELERYD_TASK_TIME_LIMIT`` :setting:`task_time_limit` +``CELERY_TRACK_STARTED`` :setting:`task_track_started` +``CELERYD_AGENT`` :setting:`worker_agent` +``CELERYD_AUTOSCALER`` :setting:`worker_autoscaler` +``CELERYD_AUTORELAODER`` :setting:`worker_autoreloader` +``CELERYD_CONCURRENCY`` :setting:`worker_concurrency` +``CELERYD_CONSUMER`` :setting:`worker_consumer` +``CELERY_WORKER_DIRECT`` :setting:`worker_direct` +``CELERY_DISABLE_RATE_LIMITS`` :setting:`worker_disable_rate_limits` +``CELERY_ENABLE_REMOTE_CONTROL`` :setting:`worker_enable_remote_control` +``CELERYD_FORCE_EXECV`` :setting:`worker_force_execv` +``CELERYD_HIJACK_ROOT_LOGGER`` :setting:`worker_hijack_root_logger` +``CELERYD_LOG_COLOR`` :setting:`worker_log_color` +``CELERYD_LOG_FORMAT`` :setting:`worker_log_format` +``CELERYD_WORKER_LOST_WAIT`` :setting:`worker_lost_wait` +``CELERYD_MAX_TASKS_PER_CHILD`` :setting:`worker_max_tasks_per_child` +``CELERYD_POOL`` :setting:`worker_pool` +``CELERYD_POOL_PUTLOCKS`` :setting:`worker_pool_putlocks` +``CELERYD_POOL_RESTARTS`` :setting:`worker_pool_restarts` +``CELERYD_PREFETCH_MULTIPLIER`` :setting:`worker_prefetch_multiplier` +``CELERYD_REDIRECT_STDOUTS`` :setting:`worker_redirect_stdouts` +``CELERYD_REDIRECT_STDOUTS_LEVEL`` :setting:`worker_redirect_stdouts_level` +``CELERYD_SEND_EVENTS`` :setting:`worker_send_task_events` +``CELERYD_STATE_DB`` :setting:`worker_state_db` +``CELERYD_TASK_LOG_FORMAT`` :setting:`worker_task_log_format` +``CELERYD_TIMER`` :setting:`worker_timer` +``CELERYD_TIMER_PRECISION`` :setting:`worker_timer_precision` +===================================== ============================================== + Configuration Directives ======================== @@ -223,10 +356,10 @@ is already evaluated. That is, tasks will be executed locally instead of being sent to the queue. -.. setting:: task_eager_propagates_exceptions +.. setting:: task_eager_propagates -task_eager_propagates_exceptions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +task_eager_propagates +~~~~~~~~~~~~~~~~~~~~~ If this is :const:`True`, eagerly executed tasks (applied by `task.apply()`, or when the :setting:`task_always_eager` setting is enabled), will @@ -1785,10 +1918,10 @@ george@vandelay.com and kramer@vandelay.com: Events ------ -.. setting:: worker_send_events +.. setting:: worker_send_task_events -worker_send_events -~~~~~~~~~~~~~~~~~~ +worker_send_task_events +~~~~~~~~~~~~~~~~~~~~~~~ Send task-related events so that tasks can be monitored using tools like `flower`. Sets the default value for the workers :option:`-E` argument. diff --git a/docs/whatsnew-4.0.rst b/docs/whatsnew-4.0.rst index 1f24fbd14..e88d31df7 100644 --- a/docs/whatsnew-4.0.rst +++ b/docs/whatsnew-4.0.rst @@ -68,7 +68,6 @@ The Task base class no longer automatically register tasks The metaclass has been removed blah blah - Arguments now verified when calling a task ------------------------------------------ @@ -93,9 +92,77 @@ running 4.x: 'fanout_prefix': True, } +Lowercase setting names +----------------------- + +In the pursuit of beauty all settings have been renamed to be in all +lowercase, in a consistent naming scheme. + +This change is fully backwards compatible so you can still use the uppercase +setting names. + +The loader will try to detect if your configuration is using the new format, +and act accordingly, but this also means that you are not allowed to mix and +match new and old setting names, that is unless you provide a value for both +alternatives. + +The major difference between previous versions, apart from the lower case +names, are the renaming of some prefixes, like ``celerybeat_`` to ``beat_``, +``celeryd_`` to ``worker_``. + +The ``celery_`` prefix has also been removed, and task related settings +from this namespace is now prefixed by ``task_``, worker related settings +with ``worker_``. + +Apart from this most of the settings will be the same in lowercase, apart from +a few special ones: + +===================================== ========================================================== +**Setting name** **Replace with** +===================================== ========================================================== +``CELERY_MAX_CACHED_RESULTS`` :setting:`result_cache_max` +``CELERY_MESSAGE_COMPRESSION`` :setting:`result_compression`/:setting:`task_compression`. +``CELERY_TASK_RESULT_EXPIRES`` :setting:`result_expires` +``CELERY_RESULT_DBURI`` :setting:`sqlalchemy_dburi` +``CELERY_RESULT_ENGINE_OPTIONS`` :setting:`sqlalchemy_engine_options` +``-*-_DB_SHORT_LIVED_SESSIONS`` :setting:`sqlalchemy_short_lived_sessions` +``CELERY_RESULT_DB_TABLE_NAMES`` :setting:`sqlalchemy_db_names` +``CELERY_ACKS_LATE`` :setting:`task_acks_late` +``CELERY_ALWAYS_EAGER`` :setting:`task_always_eager` +``CELERY_ANNOTATIONS`` :setting:`task_annotations` +``CELERY_MESSAGE_COMPRESSION`` :setting:`task_compression` +``CELERY_CREATE_MISSING_QUEUES`` :setting:`task_create_missing_queues` +``CELERY_DEFAULT_DELIVERY_MODE`` :setting:`task_default_delivery_mode` +``CELERY_DEFAULT_EXCHANGE`` :setting:`task_default_exchange` +``CELERY_DEFAULT_EXCHANGE_TYPE`` :setting:`task_default_exchange_type` +``CELERY_DEFAULT_QUEUE`` :setting:`task_default_queue` +``CELERY_DEFAULT_RATE_LIMIT`` :setting:`task_default_rate_limit` +``CELERY_DEFAULT_ROUTING_KEY`` :setting:`task_default_routing_key` +``-"-_EAGER_PROPAGATES_EXCEPTIONS`` :setting:`task_eager_propagates` +``CELERY_IGNORE_RESULT`` :setting:`task_ignore_result` +``CELERY_TASK_PUBLISH_RETRY`` :setting:`task_publish_retry` +``CELERY_TASK_PUBLISH_RETRY_POLICY`` :setting:`task_publish_retry_policy` +``CELERY_QUEUES`` :setting:`task_queues` +``CELERY_ROUTES`` :setting:`task_routes` +``CELERY_SEND_TASK_ERROR_EMAILS`` :setting:`task_send_error_emails` +``CELERY_SEND_TASK_SENT_EVENT`` :setting:`task_send_sent_event` +``CELERY_TASK_SERIALIZER`` :setting:`task_serializer` +``CELERYD_TASK_SOFT_TIME_LIMIT`` :setting:`task_soft_time_limit` +``CELERYD_TASK_TIME_LIMIT`` :setting:`task_time_limit` +``CELERY_TRACK_STARTED`` :setting:`task_track_started` +``CELERY_DISABLE_RATE_LIMITS`` :setting:`worker_disable_rate_limits` +``CELERY_ENABLE_REMOTE_CONTROL`` :setting:`worker_enable_remote_control` +``CELERYD_SEND_EVENTS`` :setting:`worker_send_task_events` +===================================== ========================================================== + +You can see a full table of the changes in :ref:`conf-old-settings-map`. + Django: Autodiscover no longer takes arguments. ----------------------------------------------- +Celery's Django support will instead automatically find your installed apps, +which means app configurations will work. + # e436454d02dcbba4f4410868ad109c54047c2c15 Old command-line programs removed From 3976de202f468595692af4352a4357fc4fc4b46e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 5 Nov 2015 18:14:28 -0800 Subject: [PATCH 0795/1103] [docs] Updates worker_graph_full.png now that Queues bootstep gone --- docs/images/worker_graph_full.png | Bin 107927 -> 99783 bytes docs/userguide/extending.rst | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/images/worker_graph_full.png b/docs/images/worker_graph_full.png index 38cb75c902b5f9076ba54adc12d5ff387e1bd66c..ea104a53ece82a9b5babe4b1c044ceb8e24efd4d 100644 GIT binary patch literal 99783 zcmZU*1yGz_(*+0wg1ZEFcL@;Of;$A4!AbDo?v~*05G=U669_iAySu~SzLPKV?(SbT zRTNa+>F2hdKHU#rloTY95%3WpARv&XrQWMRK)f=AfPkigdky{z*B8?&@DE4_6-iNu z@)5#)2nb;a>GvWZTp^FzVABa_opwB^chfJBs7cLzL*KvVC^Hb3v9C83s%X8|ZduV2 zI=09?s%z6Xuc)oRR>q-HR;Q$+Hi06Ik@D?N4jtgxy*+UWK^{S(c2v}G(opkB60-kZ zQetzvdSaNtyX#5w(ZIq{W=_f%`v3RwR3Me|#}Dulq=%)1#323ueq4opCFWmYJqG=s zQ;NP4DW@q8@Bi-!(#s#VzYeTH`e7EZNXuIG1l}Oh&()KEkvAZDXzXJX4 z;KyjwvJj(Q)B5%9aNV5`7Em%&C{n0yg6t5 zcQ^!&wcv=1k{_)QI#^O-TW8f<+{kE?im!rJiOp`Aa*fMx;h=?`hm9KGdO`<}vh$*_ z^tumI`nUEBP#~WHjwopDNV-rjkKM)-LadD9`Ih3i^|*~B(B(@^P{+d&6XO$%Sx=Xk zvC(L|Go1Wi{%X*`8|p1^XSXXCQetgsQDeKe41tbOha!7@9kSk?ISHDdg7w`PixcO~ z)|lUMb#d)jO_&=Od^E)WKAmFFAL9iA17V>eguTSVT0sjsydjLtWOQSRqgS!MlF-y2 zh_!~>2vnkzsG^8gw#V!EQ~s}DQG<`J?=yYYvA;wp#ttp{Luhd(Nka|N*mZ2wLl4ID z-=xr}#j{LHCn?5i#QqT+0v7Dm@81DK>UZk(f{cfU7uTgSqk+(Qyc;V(IX3fwWtpgK zfix^C;{41aQ{j(sDJ#(+@l6q9)<5@QzPp++ZPL+af{~e{cpL6!61i%}JjMX^>Zwfy zph_mG)^9ZMmA~SCqvxM%)cmsWa6koApz!dMgdQm`U&s3(W@_mE;wCu^O=7<<=bsZK zomIbTYyq>Mq8KYM^~L|mCH*BgXPrQSYE0-I<;!J8mb9lzvB}oimk!mNltcDN;RqlF zA|ia^sOJH^dKfDTg9rI<-}i(4rDNeA04TrCsrY^ufTw&fHFbVvpqq(b<*sh73v}wC zWqw16p5V*UBnAP|ww`%w>OUpx|FUjRSAYVQ-O&_GQE72fPPG#T2L8d6sl6Vh0i56S z3h}4nl6V@DMTVd|t>DW6g?|A_Of8&yWJt}wp#+7k=4Ai)lP^~F1NnH>XB0iEByp%- zG+YvbX*UxF*)WsU6{cQCp#Hz3J5<8CR{D(mdvBrIo?d~vTmT7qQ%KUILrx2zMq&)s zc!(}fOL`mW4<^Z;_i<5p3*3AZe?#3h86u)OB;_o$8&0l>PY%~44ukTpr-ZPcmpKGF zoMucH!1N@r41!6J#KFche1tfP`FC3!u$F7=&Vn%ETfL83r)D^1$AMQE7#JHrsqEz` z4RHNdP81R()tN>G>j3r+>_~@6L@~b|3Id){4xC%MtZpM;!7VjuP|zp>5|^+|F|dg? zeX&2oQyLKaosTOw0lal)%IFgG__&g}(LAXCK6A1-_%I`(=Q<^nfPfc>h{`7`Hj}Ps z|3jq1#ELx=HJXxP`}8(96&II^;Q2WPHBu+@pWyX{#vupmij1A##u6$nE)G=o#+&*_ zwqb0CQGVHM*Fc;v3*6f&^+92uiV=#iaJZYEXu^QM7hZb>Ht(&n9i?9qz(W>4Z85*l zF$0<-YAD8INE}b{TS18!*g3XxM~S14hTd-SnkoKsW*hWx$~!pJnI<$)+_oCPrVEXP z38R$v5|WFb4$>LmZ__*X`YVAHEmUdTWr>pYB>vesm=A_VboDdEF} ztGn3Zalx=dZ@hojYqC2J#;VX}L61=`?@oM~pU|YyU`ZdpNLPv7B6$QEtf;bkS#95! zRp(e82W&du=F2W$cAV$h4%WAYT(vWD?IyL~piUG{y*yvOw71TE@*E|FqUaV=(Q%U` zo^RK$x6(CLsyGV%t2u)v@D-Sn2rFuk{OYfn?{-N@k}i+~-~~1wPp{vJliC zqr)Sl_Do$Bf8VHIYV+=Kme2yY6trG<+e(`$JjY7Rmj+z~6+u(SNIIW>&!eR2m&{zT zJg!H{I5PzRH+hj0D+61_X%ZW!KxE zE8H@}H%fmiFV^jq2-$w){bY=RkH4{_#vk$T`{cmiHCloo0S<~o++qR=bdK% zcAg`OQA$^^>h5!sDDJL@is`KD*0r)6~q*CXSs zi~dE}qU%|;TjQQG%Zd3F^7%UklgUU))7c*dW@WEfmU!N*#|S-*aBuBN3R0BGBirvE zv3;^Eo_(@e_EI9AznK$z;yo5l+L&8eBF%a^QU$6e?$vI87Wm_h20CD!ShYB%-{!m6 z0uyd*qz6)wF^KU#$&xxCGvjhhyVwqC%lZJKxOk&IxVU)fJD#^+7`2*$CKe~Dd_bq( zstZ%{T>RE_c=Pt_Lj+J_M>o$G03gswhfU{?Ny)@&K>&Ff)8P7Ve=`8FSe^53@1FWbg3J%Q;J4Zk<;9 zy#~u(E0Zp39@00V=Y|ftmxrhqr7*M3y{{39A9{zV%PaqSaFx!2eLc}MGQlur?IHH+8wsL5dYtNWpW z0qMJXb_~T-6PzbudlHO~h?5QfA-) z`Q1?=7Y`5FG!SiAMlq%!%XLcGa?|KfLG6TxKEgZQh6d?&SHp)#Mh;xCw33grRV~QY z97QsEtZ%G`Cv$~Y)Y!J?g%@V|7(}nX3vY_@TIbF$$yF=wc&G>BA;e|CW=4Z_G^VS# z(sLHK9r~yQ>BMKO=ZgSY?pLtvoIF%nkJ~H)to>)NMPb5ZX8ZB5{omD)L64b&zpaZf zcgowbmUEQfTC1a*<94J-!?cFISNAm@FM_%$ir`hriwYR9!srXURUDaqQ}}Wu_;~p$ z0I9EYuZhK5D@pm?IVh}rlJKfV9Udqb83BBg=^^9Be`mg`oHp3=8%@6RhY&E0oa^n> z{(0l$D0hD!Y?8nZt=pvJOwHExcc1O#MVf{>inq{Wn-vS>S&uAd6>%-*vRP&D zh~&==Pd){jCWtv*Qe-83`SRKA^mwxRX0HZ=ke%srOi8sUBJ<^Hde!Tr6s|TH<<&%* z?_dwRNAQFc`4a8Mz8$vWWMp+*r>hd(v^VN(X5Q5Vofzrce~&Y2BxH#ZJenDlxuUbyp=YZK%+j zkVs^3M5jSUMYVWk-QKR-r`tKOrIXL9CV^WMV+l4a`9(I|-5!%S!KvVfh zSHk%_0Af_A5^?{2)vm{CWZ#-{SxFNO?6q~=G=n+8gtIQ{uG}ABssP4mLq&Xtolcx@ z>w&Tr>FBPXHZbD|YeeBp9G&Vvp=zn>MYMu8xT)S+EvyCly>=4o-x;;iatpfOo>>mr zDpGvj7;5-wS{fMQL+cG?J0B+V9-2NlX@q#zi16%P6MmrBnpPU!w{>^apx~G|XvJg} zZ3*Xg;_uRQVk5KPaN%1cK`;mX8KxxZ|p48Q+7M1r%VbQYm@y+OBuP3U&$(|4yeL6cav^#S<`x35e)6M3ra%87hA zZ@Zvz$|}D(a|l3i2rT8dZ!+WRKYBR?c&k6}bUghS>Lz{?+#a7Nr`y+BsBuv*Lq>?> zSoAnePsE{;saBTW%yqLbFCme->>f*>A7K!|Bn(K+Q`NSfCPFur^Lge_q<`gHE;K_| zvc>z?W$>W)=p1?ldA+e&p4a5@aPgJ>F$m%%2Y3d9Or92o`G^_efc}>8ANcGG5-rf$ zZZBTW_&{|&PnMuLNuh#+=={A@lZxky3zd533r1{_P8z3yr%9udJMKI*0H(R)&?Z54eUB0Nx&^{loWL9^T38}#AD=LJ-O zFcY7iSY*-KMkD05e&}wal;tQYmJ&Epk8?K-xKIAs=T*41A6WMp`A;2Pdk?Oo>5U~Q zI+H05r@&)AHI-rnik7Z>!up?DgMPK1c6R->Ykjb<3z=EVN|mK{PI(@~1$~}omOxxy#+ApR+ZUgD{h zOXW;o#{K)_4#}g7>6qJzh>Y^<(e^%}V#|Q#Hg9q72h^9JOu0y#T}n?bK99(5&$jK< ztG6{TzGB)`=)v18Pa`i3Y{1+-i_G*eOM@Z(4o3yqr%flRkA?cH6^|!AXIi1uy__Y; zD3{V=PC1_SJY11^#<7sHNUpk-wMwDKfZa=IivheOMDYPjtkUGy-v8-II>n$4);%Z$Oo>G|_uIh)z+1-E`w<42DGDA^WcKLH zlArHx>MXf)j5j_k*T0*Ut?*Vr{J7;d9TM|K%o<&k@&sH2#Fecd5A&P>2?%g%Tn`Re zZvL-R;ty zjrv^EeTn#Ty~zKL^)Kn*y9>^*ZX@9ux4xZUCX@17s}&Fu?hQT`a)=Ik`bLN`ijpe0 z_0*pT(n`xTO?U_ung_dGUmpBI_x2+xV~XWpT1paKSOE`Qu@+PwSiajD zC_V4D2*th6N0AB`dVUzdPFDOTu^l5G9Mo?vbQgFR@cAEA1gQ!)M#}-iz^8`-IVJAB z&Q}4Fwv6YF_ zDS|%AYB_ddmkU5&pa`Xpxu+WYEblZ8x{QmG@wWLs`&9MnFUbV?@~h<&x+Zjc?dO4g zv;DIuELP)#S#s?|_{kju{JXE6b904D9WJUfrOb(KEnGYn*1AZ4lnzY}?yUV94{>xH z6m2_N&)Q}wd6-mHII5Yr_!L5*OT{4!+iwPKr5zmuJr8!Hv@Pw^k^kDOvl@I|9ULxw z6D&XJ1mWxB(?>b%X5@s&jRXm&L2s$fD(KXM;=GOq$@XucesYQ~q80__@IRdg)E6)r zAiQ9WSbjClJu2g+T7p%k%kbj|8BRJ`!F#9~G6?0UoqPPdehyel8ir+F%uTs}Bt$E? zu^wXZY@H*3YB>EckkYWC24*X$*m--?%3Yb@-(_~r?S-P~ELFD4Ia>jCZ?EkA{x+IY zU^T-`*ywyL(_3x>EuVIfeM*^+jaArwhiHvlU?c*iN*`0&{P_K6Ur#@cp!-jMj;;9r z}Q*Yndc94@98iHM`xm3^<9UdA3+~TUSza3)%+%BnN5EJ7_$?j_Z zF)%49SXqdn*7Y;w0Qalf+IlCMm}1FZes;@aWOHKysAaw5i3vM^;IjiP6^FpIJLXpS ze{#Wm&5&i_Q70zU>1rd5ZO28sx^{Tz*Xu2nOj#&MNdrd{&Fz@wL~ETH;LNZkl+Baw z-=Km<_+{VfNgPdc56K44mykY6CI*dI6%IyL3I#vtQdt9;gs?(&n4Gw{jqO|^m(FI) z?V5L_ulZq{otK}s;}+b~GroOWX=r!#H~HIv zlRshm4|T1TD1abUhF+VBX1%(+XD=ae&na&Vm2<=Ja2L z@x{$t1%uV6<7krStw58l9a$sxgw4lhJi;Xq^ml5S{O^y!c?Hyi9E!&^kDfdt{{qSQ zSNsN3c%+1M?#(6hzjQHxX(9%myPk{mA7bE(allWhFUNfuN5AWGP?Y{C)j!m5%l9`N zd;`EVqQlwFy~ih^$8-4oeWK(M3PdB~l#ZK#O`SAhbITeq3<4Gio+qp5%2i&Q@bLW; zECxQxfkngTfZkYW-@nr0EQuhiA?xZIc*yDlE-gc11|L5PgH0Ug<!w)*OQ{0oqE`z}{Ze;$$WMsPuyys7inZnl8XgRZU5TNEp~BE{@%NINQ$6h0 z#>=X0m#Lc;opo$D0kbbS1&P4oOHK?ToEptHo>Bjp3^Hi{*WV=y2&-|JyQ82F@U^#W zUv9$aqe`XN9S|SiVWP-G&Csy2u`#=>b}**w4}f|APmT8UiY0N8@{!Ww4+2e-?y|vs z!FjB1L8T*_3}0fTX8>5{Qh$|I$p|HaF(s92rM2feE(aAL)(z@-zK6lX$4^+w!3bs= z3FNrAA-#Gumhn2C8l}668{UByoEzOP&8evq?+t#~zVY6@z<5LOr1a(hlYBZO!MZ$_n! zZ3C?OR~RihX3_-1W}b1Z(>brWWbEa>ma1mfECqT__H9G_Q|S(dud+lossmol4Fu_! z;28BA%hkm882#fO&A2KB{{a8|oXt$#qLxCjkyvAuZ70f;#ctKVV-WMP|LtR2zF~jVo+3elFYXnT+D7YWsuK^ zYsdf;3RKwMXX##*Hs}pEKRa%JhDU~v69{@RBLeKfh+Zpb5GK*63sDRx&oB$4*zPqQ zaIo_VlJg~myaf|+(LaM`jt9;&*KL}n?Zqs3ODMU>6+A{hrn`wo@BV?gzMOEZp5yor zomz9z5btr<$oyQOGAsP^_C@E<5;MvKnO@~Sfux?9i>z4QxMrc1@0FxoZEw5}-9}w~ z$wcW3kR0v5_MY@mS#$zqYF#Kdw0Iql3}RrSGeqGK**)Oz5k5bIakq)bML1C(>xw06*RC?IX!pf}s`g_b`Dpmp` zT$9*K_K%aNd99>6k&rznR2MCM3}VX#tx8Gu%^GNZl`WrRwcvMxh-Td3epNpZd*_25 zcGdQF9~R!}#He&tO4M-sKE?f>KKMdH&o{!moAy9U3;I{~0?3DT|svM5yWf6;Crr zS~^VHrR61S`MHSlJykQ&^`3Jdb>^JVLP3w7L1(unQ1;S#S+KI#yskP|H>ca!#A`;v zeMD}_)>-~fB3r@mEnwV2?YK{2Ppn;^p+{3j>Ql;-GYA;adGO?ZkejPVtb02`a z^LTjHYMOPHtlvI~Udr?I`;1uzP4bQF;^7LJPef^ zRwIXmA8R%NNu@H|G7)m&QuR^LZ1dYW7Dfma%!FKkT}f`Iac6Jwudh z3O;6yc1@Yd1H<01VpRq=9bV=Py0OSdD#V;*GpD4drr$gTp&042+M|H+tf3I$ zWiE#-MQ3dW*paSMAz8!6_!B2e^*R|<>VuiH&a->?YCT2yAgPt|rh>%{PLm~cYPj5i zp+u|M!@4Dvd3+dx{eY!kS`xiYf&GvRY|rq{2WewJq|%%?tt0+3uQ zB^#M!IKPdo9J=VowB|u&DDENFNwpjk?aKZMf_73c_+X)RK$6A8Rq4q>jU9ptw*Ew)*h z`cslTjo*LM zmj*CBbirgBX~qUc+vt)1oP)6)8cmU}DCKU|E5`9N;sVdK)`^Y`X^Lbpd~kxZZA2{# z9oOW{b*?VE9)Ct7zP0O;ic|=R%yEznNG?WEx9(bSl~|5yes3aANGepNWHh*O8Q{x( z@+?On{(Pyl4hj7n6!?7g-O~S2bnCoKS-jATPR@wj{$FxPqyg(?yCD?wZ2&|(W(;ib zBU>mNf7=el1-?l%rzEk`i&RH>6-)q9@X-9hByZlx$ZNbtaz{1>=0)>~CpbghNaf(B za&XG_>Ta8uTl7t;w}Hd<5{3U67#xUz#1V*b-$LecI$uW;hdiH_-}+SE>Ua@yPxIdD zKLkJYi@MZpKU(&OF6L#Ztv0HlKM6Zy!d1W6rn^FN89Y1)#v{(`(-5RWv+^}s)|oDV z<}oRhHQNnl<-~ngShQUd%&RrX30yNB3?2O7PtxGUsi)c}0uST)JTaY0+UUos;=}EU zs$Ni$kfrl*AtU1xq+8xXO1ri^rgM5)^x5+-#C;1H%);yL(bSj%y+$N9%!W(OpcXmt zJ-zG#-?;I0CZg+v0h}-vq$u(`=kK6EEK2Not`IBgsE3Sc?P)YO$94-=AQ(kXK6aE2Yj%w73-U#dJ|w0e^M1Ir z*le-a##S8tI+{4#w0PMKpCupo>d#PH*ca`aG7c#L(ca+ucBFW0ite0R7btRAL{3+j z5%mle?ra|I4Px#QrNIE!SCPkzb2Vf6^}Lauytv61Zt%m}aN+E4T?cbE__=}64_KJQ zCEMr6#Js$5Y)GWIDQ9xXwB1AWGMUMk;l1^j>KPvJ6NxJZ4`+QMY~34?K38I?M8hTuft@IK(6Puze}08w*j+LkKNGMN8coA|>O z8v&!&I9*qNpxr<{|7x&-1g%H{ttoCdtr+v7J^oKM8(I;BEkPdX(44nU`R_}jCAJtM z!=w5$yS%>RTwDhV5F7DVf$;pue9F!Rh%B0zEpwk0$Xb*B3o1USA-ao5l8Sm(zvhz% zP>qgI3^CPB#1fG!1rA3)J@3gr8vD}aY5!_j1PfN2U=hEG(#v9jMG{O2zL~;DwA;|I z@T4C8V6(Exv{v>Zcz>*DP{n>JyX9B!!#A0oV);?hvVgesLRYa~>b^&=q`TL_v?xD? z&wMw%Ow-$P+5!DFCjWLG&LVv|PFUNFJ4=WpAh@fQ)I4Q_NO!1k`c+GOY@|0g%Q2!F zH6Q9rN0)AeH@uB3M0XR`cl}~1BF}-s@~Ffja6GrUZkF+6tMGmgpbecbx*#+1p5qB~_qFzaRwkcybb zw|}@Udsisn@dWgJf4*-ig9VMs_Ft6qQ-GtKL!W;!2ilGzlza^CPUeerLcxdSs`T3> zDszvL?ci1oAFcWwft>tKy*IkH&GdD8ob(|_zCR@?8&UW+%oBi!z|i3-f4pZ+wD4;1 zaL)RS+2wI6`K@%QO#IhtkHO|)rgM zzoUrXj26bA=XDjTXGo^xXZ%~b>?!E+h>tNW& z8g*~x=>)O5Pex+)Qv?bWQ1fyE;A zD^yM|TL6@ALroCA_cA)tZbxAJi@Gi{&O&seLbmWwNDS3n_~~8A840>J`cUg z*Xm!ml7JU#uE`Le%?zRXul$o?kr_{iXxe<0dEfHx)uVl#x8#=b zL%#@&HCeT*Fvxn9pY6-VZ$|cmtQhHMT0J~`pvV_kdP$jrFk>+|Xr9c!DnQ%&Mb_Ta zbaov|p1@~*?*O;cYsuxwm6!D7+AC z;Zt*Ozl4Q0{46GY3uTC7i;Y`_8(1m&juH(V1UC96AM-F8d5RI`vWnlCmE;WXSK-}5 zB7%680cY}RA;ZApSdOL(mp`t^`it7&D6CejfUiZZ-M8y0&t~S)zRtrzI;))or(d2T z8}%5;x`K}{VjD!RqN{U@q#g)2@R@6TbrE1@$;+jzREoK33Ih{a$-t2b4s;7=ouTpq zt)~#k%3ur-w%MKM(PDL$-$=+NNP; zN^vAECl=a>IJ)qrRs?J7!#11Oyx#@|s&uEA3L$Z$Qwl-ibVtF(`R*OFa*+QPxsSqT zb`^Tp9LJVk8M{Wo9etjYtCeP0U1KB+aP&45ng|G&or(y&uxv=gN+?CH;Y>MnDnf7{ zKB^B$XZfEW_@aZYfhQ8LUG7@9{xz|572v6<6lTG5>^ZE%&N@+HqaFp<)w;ul z6&gGTeL&08rsncN#>S|9yVZ{s7>=MNrfHVLzOl}P`17@2YMVGLG(s9$S(Q$J0pbVc zea@Q2kpct5XF)?4y z3=zdo5-1dB2^^oCS+&6Pi%LmeTgOYo#SAUO9S6m)YZY|8eN|`Lx7&Z+nql8|Ud%Oe z)Nfcmd!%`1Ss(q>Kw&xZdv&!_8+<`6OCczGE{Q;f=FT6gb~Sg+I48czol7%93pA_? zGerd;Fe~V=>b!J_PZfYVEi-jdtt}>E_?DIie;gExP}@a`^eAi%E{oOF4jG-8;dT;y zTuHM>=W2?B{kNf{vq-C!Tz}yuJ2=59tmuVRAVXmllqiIvuYc8~Zw1gYxA7Vu`{&>! zS*(t))YQ@YP@!ojG^Yf?y|!^Io5=z5PH!ncM6a5^4~Rh)V^I>r)kESowVhrsHZM_} zUl@c|lMm!5m!gYlIL%6^nJ%!VcsgA`eEvJ!a`{D4Ct;0p#6x@)>2TgP-6#B*1ZF}) zx!1oYvOsPe#wWz>r@BSOoYj1yzgzIy; zG*LxkRvI5Th}3TaRW#(nq;D){q9H>xATSnN!mfuc?bdrQWuB<$6zHzp`75XoyTi!N zAR*!4QZ?&-)J&IvR>Cdj;hPrv#gZ*Zxu^UV8#cZnMIc+S#4T%?^q3A#mI&o^w9u$k zSxs@JrKi?DRKNRMSpEf)6%=bG^DH2`!E!`KYoU1GXpBB^crlo;3T6AVhhvW)%rw3& zR|g9wkX5HYZ0DG;)L9^0$@O*F5!I_2*ZkB_d>si+SD9|nuDqh8mH6Z{O}#8s&V%TN zPd7Nk80A>hH;Apiv{;d_d=f=23mM3}ivMdp4Sc{FIO+(g*M#gsj7sV#xBY%yKF0!P zxP}kML5^(5ZCCbS>;vZW$dqi-Dv-mk5wYHoomA<5dAhr=1F>zX$;&B(PFutco8l0l zo9{(%+V8S&ROeL{c%LXi-tkg!sdAawj=QD7y*P+6$S}_|nQ0D|xajA=g<^6gSpHs? zM8h|IL`zoqW?ZSa%8wk00yJjGihcYkD%ML#ND%E3KWi;+{{=->w{Wrq7>7edOaVq1 zERJ6{BfI0Gb3rAgHi0?n^IKba*M=rP1`!{BK5lCIt5U=hat|qY!t-wksZLZdI|Dlh(N=d*}#m<@e$5 z>6={OTo|W#LwL8mHR?b^)|@{%8phtK^P(7Ilaut|;N04scMWr9?1nK&)|^8J84tUjU&6)2LJ>=>wV#1d{# zYI0EuBsGI@l(6xM+cF@lX&N;e4&nC>hBH!LvJR`A_MM-*Krq5r$mz~;%G5HIyc=q` z`#UI_^9Pp`9jmxDY93xT`U3jM0Wgq~K|WU&S*D~Q2(KcLpLNtD zqOZXnaHfC*ZZLZ}K04~-UBmj5FnPTNnt7|M=BScvC1roliqno~??*`xq;`dH%)V}Y zb}_Q(l9t`l{!I=MDuD0Z%{@0e7=VxeVZ1q{W&7iNoJw+S*BITJ#349+$6{wOF-+o& zq6e%e5+{*GPol)n2#$SkQxK=?sn=o;ulA>Qr-FdrSXMIyd{c(g$l;w}e(EK-px{6tCPUVYVSoh7xK#4$4#frByHfDv>2743r@8lYS4(BxI-bg5fE==6lv(P}r zQYBnl*h%iDKbS!tNN`fWJmd5gt-MyVW!_$oAn3$>;G|(%N~ht8syi(tm6D!Pn>h)f zixgFKnonaU<*n_vX#6cRahw?W5)YfWOboY;N#W#L)XJXR{-l1(W|qwRww*Ty^_t`$ z986mGrvT#))`#wW5sm8y`eG)3XWm<1ur9o@TB&_$PPo%52AfNftH~^DW}k*Jrs%Xw*I$#Y%Ys>O&7)ea)2CrsA2Oy4rkcp}9uS-frFs9$ZGw=?gHa$$q+&Rt z#k|P3pM}A!ZO$Q6JM3S-lw@tht89xD$GMzjO_JkeE9+|&7cq$5A_iE#MJYH-&kLDf^p4c!< zzR}S6NoWwAEuk%Kk&>cnwE7yu*MgWo&5Ih>l=YhSy6Vg<;d}Yiy4RbNEWj;gNGm2b zem=J0MI6F5cpUTet&dE0ll@O)S|Uzyreu^g#`clj%#>`F z6I;ZTG>O+|8PnD-^>*)n3F;&Un|a@R6|PCY2OxPih|1pJvpIYf9N&QF<*Y zFMgjb2_La1$0N8opE5$fI$u-k_-W2gtz^hR5pO0GO~M_BvucfH5W-6a?GfF82Zy{q z3@Z~U1xC2P=n%BXVTnKV>hrIi;H*pQx6W$jL9vN~WI+QAA95ba;E}li6tA0@H!+Q? z$rc5~C0z%=gElL*+(8KvNYw&-Fs6`r6Wtql4v}qHIXTtDm+dfhfruJ0hANU?N$fJ? zYmXyVqafspS&9=oRc1FOu?EKMTgvNA)N?@rBU=xQ4*90>T z)1T7&e_}SLrsCJv9i){9Ffa(a&?LA8F$y7*Goo`5nZ+1sl<>PM(IYd6lZwRfO86LF zneOY-{07^9mU)5CU;ghA?wlW3hkQq|QM;i^!cr34*HeQ(Vsm4&H1{U)bXAK#4k**9#Oof-G;YRi zYG;qf8I9XG4aIp%KycQO0)*a~l`03rP>R!VR16JggolOBuI%5WtsqLBhYj;ALI>}A zASZ1v2g};1Y<~LF;I0XPQNRj^;|EmQcF&wgWF`vl#>>x>meIpOv*1}V^0HI4pvUA= z1CHnN(kv}2CAxQynIhELNAX~Byc4=t$bqcRs5~w2s$*K(nH_Skl>oNQ7A6h#`-m&m z;8CaJ7vBp`+x_N{`V@W|ncv9k&su-4qNl5;X5^}eUdb9+AC&z@yG=h;z&kMo&+q1 zvfuql#B;|@vprqWMt|ygNc5@1>SERriJ;-cBI84P32SLbKCeIH^Vr^+N&(ZM7ucK= zsypa>g(+se<^wHPeoU^A0ncn|CirBPw>n=6q5k$wtf|6<<3!a$4`{;gW(U4<*S%R6 z9zcgjM7dbpvTazJqIc8!JWo8Ft%H}qhF;~J_OShiWiajKLOumIEqbh92_6L{ar}oe z&vi%Sz(U_Fu;%4l3qFGN#?tw?U^cIzPJ{TVPEx?AVQRqqWxcdkqhT;f|9L{C03RgC zZ+hmbpdfEiQAU>GJq9m|%fJVWIASNg_is3w=3|kzc8M5O(=3C=&90M7Fno=k$DLs4 z+<3`$uJPV4q0x@{x99hwv)k}OIX7%L^uyyvUM7OZwP>+0 z(OB1yY6&UM90Bi1%bc>fJ}K%a$*#6h!Ym?e^`8>Qgij#pK{Q>rX*YqM80;^u(%1!g znX?`z72SpA$OV=JoHW4Q3%*S}6MG(=?RxJU&M>kt5&~lLJ>2LXl8;BTJaOw}>%rH@ zTO(_RjB?s!rwovptv{i#rwU^)plZp!LVT@tjoxM-S`#<^OJ-C28v4V1mUne$>K_i@F>N#1?|UTWxw=&G-^`9m&m1tqqaY_n zSEA9m=Sc~I0D|OS=D*nAZ0UV!Ufvp3f^@L{;99GHI5ro(ZmFQ&3xl73E+E+=Xql^S zSh`;ddNc~rL&w7}Lgoncpom9bU;W!q8bCw31H-?4%oY6hA+&2Og`;hb!_yu?b~&c^1KgGTsG_{<33(XLXp|Ll4x=|N_3oZlv2@; zJ-Y8=9$1X0L7pYp+Hhf(nJlBmJ0VCUd3qp`_B3lHe9q|J0L=(oZ}&lO0$%2w;1S?s z`#K&QGlTgY?1pDGgg$p8N`~X{@X;y|fx(Dv&ZIGiXC0+v_$%A7ukjd7D~|FAaT<(= z9tUt}B#cm*Gh+0f=qh)vjWDNLq9!?UiT!oIbNH_GDFso~Zrou_H{0ouVavM@wsGd0FPm;|S5bh3u`?xUy%>5mTaAH> zj_1}|_1c56{^x7&CkJ~Ym9qzf*;+UeMb)TwV#A2l=VRMLHi-vGr<;?JADnLhPwzhx zivAori#J4uSN?Ko#pu8>YicuQnD6S_Ma>&>WiX(e5f#Q?BV4xII|}u?EPe*hOF1I* z^F1sMEaN#6XdLz7>>Em>3E1;7Lct}{A^@p+d+-ycDyHCQ2*CkS?2i+Xfz{a4@p!V- z&yN+6vd`Z=?=AtJ+T=#$msjteeSGd-fP*jFrgs}R?SdPGTCOi8P^C$>_$3yzU$ zS3MSY*S20d9t+WZ*FhCNBKnUSEC=cAkAe+PmnA|8dL5Or&MzA_$KyI9gge)tb`GVm zabAlphX0zA3y2|c>!9d_YOnnmxZ1H?unY!Itya#s5uv16fsrLXQS|$sUwxqO3;Yj% z^~HYMm}@UuNuwiN8qG-6k9nf=dbfpCNbH@KdqtDk7WtnTl6o-cz0iU+%S9SWMW)4H z5xu&f2%YNllLJdl}wDMPk=Y(~hTa@&25IqFCDej?Z>CW|SW!{&hqD z@BaemB<*erT114|yncJLwjIml7&Z`wfSrvf4LTZ_A(yA~?8vIS#P4th2A`FPZ15@<23w3Fn3Q_FY{$ zU5;~`ONbX-JMvkoU}Cu}J||wQqRLF_C>P+eP%KtrmQkP1Gj!0o^``%c{je8jJ&g4> zy03qS2l7b%vqjO8MN@WWP+Jed@4q@$0Ip+hGD8x(I`gE5MEmGl zjR}mCvccBEaMTC>;Cao1qefS57Zn$$`kPP*=2=nXY=6Mw9cKn+u!F>FiQ39O&)twd zxxw1p-ua=|;*z%7y7yL}?zV@1q@4@2&B!Tg`GexK$8~o0%ROD6Q^TDWK9(O8-ky>A z#Dzw~F+D?K!I|f+kD|i-R#yRZYs=<^<9*;5mHfD=1Mrdcm{J-q~fA;t{~XF+$h-ygL0s@+uRQSSHx29rCt|R7|fr;AP_**9L!v><7_!BuA7zwz!K} zbLq3u5T?iW>?6X%DTKVDoJ_7d(8 znDg|l(TN3+hTuuy@ta1pArGeDSEPs-Sns&s%^zo3z^F|Ln54;*^6-jJ`|}qbif;TE z?>%CwQCwsALvJVl?>(4!c5R1mbLXbJ1|h`>7tMKnra(a_4uc?GlZ(&q3)iD;A8U^2 z7HGf|%ZrSS@hb4lV>{8^=%wl^3$)kVMDy~@?JeoA;!l}xSHG~nbb;e807f}w+hyQ| z*8B%Mw>}(R9}5eaW`brIC=xblHpC3GgU=j05KetXk#u)8fR{njVWPWtEc#FN;6eA1 zQ~SCnrf<@9I0D1N<@eEt*7jx3i>iyN$%H;rD|@+ZWdXxo(_~)oViST}!A`o>=qg=) z(ICT<;Q{nK&+mApf-8Ku2KT{?RoE-?ISz$l3Ljd@K2`05K8`~U;+triCIG1d0v*AV zG?%dFH49fkUG`li;R}s`ZxYuF^ZHdiJ5Zk<@|N;?6xf~wz;SRwPR2ltK}b$PUojLO zJjNF&DeA{Y2h>!?=Gd{^7Su2;P1wBAfAjzFbd_OIeotEvBqgP!yIToqkOt`n$)&qH zq#I$0r6rY+?i8e!?k?#@x_KYu_kX|d#o^2|dCxs__%Xhig&i2eGRDiEEf>nV97zEk ztaa+tG|ro2RQI5N9O!lyIJz^#ZpUCb)`LcbtGGD_K?y za8P++E)4<59p9h-_viCP!_&<6qNw-U!I+kT)+pFBnRO}vb+($}r_h8EUM9ive?I7G zW$&O*?QyBT*}wKXZu$Y&-sABICS(jxrB~8b2o{-ui}GetQE*aJ)nxk10?|HUMNs& zdQK{JFREy+cJmhCxK8Jg22{H$Oh>eh->Q9lNrI?b@3%DG?;-fCGo3)orE2z7tX$M~ zoVeVr!H&}3!unv^?KoL>pl5d4$KNoN;e?4B)M!Tha$wp%GjyA`|i{`jZm% zDGIte)I;|C= zb+T-hk(F(KuHKx^JC%9M20F{8BiV0IA@<^L_%7IyuxY{5h4-S{olMf+OmMs)bDz#P zw3uL!>i4|$)3zOc@hws;H!wcUT+XKQ`qhwp?>KUl=_bV>H%lymf^aBn5u0As z+X*~11~6*WMcVt1?dOqWZ+Z{%>IUvhLn$b85QgYGqdG_roz@^vHQI_Q0RLH+S4kn; z9om3jc1cpPkF~*vCI=R`HmMi0G+IV57slrQ>{HFGjLXKb*#4ojtJ-Sk0E?&}WQuY^ zJ7^x)6^k61Y)SM})-`=<>pxQhz>ygqGz1w@u$kgMlvCf%l` z-OcOsdQ4MV;qXBTDha5D?=q?zUS_wmq(XMXj=`_#Mv1tLkxHcKM^_`G1ypg)g7!Dl zZN4bK?j<($h0w0Mvph+O=Sj^_4naVHty|)y!)0QZG&YfloBr)qrReBK;{KKHIjSB# zW6FYJ)ob!7I^{uAeb}GJUF>Q%ze;eie;tc+&UA>L@0OEwTJ^3@J&U4zaCSe^DI zoD82nIKY>E3yN(2+oKZ)7SumY66WyG+kE9G_p6+Jb;o?qE-^hwxAR7~C>P+8GDq7L zB!hnN9{U_2w)>+ap^d`-YT2_`EjHwrll#;=ay8OH`!@rG=|9284|;QVj7oaa@49}J z6jAF{qQ6P-A@d&(JBADZh(zaS-cQh*F(^oVHVUvF^!Y8ysk5Z2eUs{Jiv-5>nLNQp zIZtsZVtJe9gVe5LYWhhlih(O8q|wj18>@Ypb)VF+e}z?NF>hS(KosMLqi4H}(9U!O2LbzBSv=?zCd!`J)L zWD}X=u2UsS$5$V_(i&aIav9N(Z7tJBQo&e8N+-x!tnkJ;WQ?NYK<9da#LvXw!;}hjNFCq%$8$u(=5a?^jF=3}G~P#P{tv7$ z{F-0n9{NtW)h0f+w%-mvL(MDRfT`X!_r%!=sd>m16cm{F^%?#e{Eq!LF9Sj1re+#C zz1OGoepED2EMFbTmyCNjBMCo^3g`94*!hO64t=s47gYzOlk$n+A?W{gZmzF&82SEuopQ=XgX$HeWreP`}R5tQ*!0own zP$C21!=!GeG9aNAH5cV&d=h4_F{Q_~;AB+psRuguXwfk`O2ygFhqX!AMW}k`2TjU99MSDU& z71BIgc8XNp)B)W3<%fYDg>g2Unc4?jy|UhFn51nUAUnaz!GagB&zPX-q&#;Fk<4J; zKx56mUX45kEBTC;97zH>PLWayC8D8;4oQGBPNQbLGR@JHK1WT;&53DG@v{O(Jsl>M zYI^~%R=FoXZQ}Lt$!Gk}Ca9M8_1FXD8s!&4WWQRVRWx_(5?}3X@*6>7wzphpG*z(n z0!+^u31R-guT547tbnlCWP@4c7RASe4tU!4KjK>t%;pH?BEo~7@h^K8z()u!>Gb~A zFY(ctGP9Xb5cc73RtT|D`&@ZEVlN#bbJ6FH@)|(NOA?uiiBwtpa9Vlzcdc_#c|onm z;2$4FJ^OKQUVxPL7aG`n6d0Yw-^s<#zN~P6tUQj3H5Sso({1wd>Se#yg zV2Lcpfk#{Ae_%lGw4*_?OoLr`ld-lkjHbgA6l@_*uRhDpk65!IY+M z1sdLSe7v+z9-;+0%a{FEP6*LxkzYmC6IUt?phcmq-OJ!nBi+~;sz_e`A>LNfQ#ky5K@Xnt*JDK6AzZ(Do^3h({HjP zHOKB9U6UAKU?%5K!b|YgPb9Y&4TQ}dcY^A6g~+2DsxJ3^V>jE{(pu$=O$F%%W{mNT z83*soxT5ve1R1I?`D$Qb{m;V@0Ul0np?`01+L>iMGw0q{gW{b7q^ePZ-MTo4Whfng zf@PK2NfdKR7)k&4gZ!sd!j3R)?as9)et^cnl>op95dcRRA?Cu^%PRM)uAXW(NQ8U8 zG_OQ?`ZDEDX(P=6{6x1RnWxSYn&cEk8am+#(M^a~qKxL8x83czruWA$>McQasO0&s zwgH%}_&IgG)(QkC{5$lggLUMrKm5s~US&(X>FC}c)E zNT9s^^hCG^qBc#G!5|A}V_1@Ty}>q#z~;3kgKiIhr)r`rHr^0i`)S`K$}tZ z&-$jxfguLZkVb+I^A|Sda==guFbjve9j*5acxpa1tpeS?+2v^rsb*rlkzY)&8eCZ_ zUDONZfc~MxhU@xNI)iai#bHzPDV{WqsOry~#Z;A#?m=slczV1bU@My~yVjvJfQYM{ zj|^}V=k@erKC5Qk<1SSX?U|tf${dSTbjD$qZw=zYJ`NMJ5?XPwQ5^2JBPklSBZY%A zYjNF2_||3wuz}^G;h4bmA}yO!X({Sf7w z|GZ^6H=tIp_LqTjQhC%t;o{993jWS*!^{e`m<0zTPIM+;52Wl{M7S<0pUnE+2N2pN z_*!QBWw-uEb2tGIXB!dC1KQFoH0=9ctKA>EfXVX@YKOTkZP5*&fSLncW}td|`Q*7A zRS1FJTT4j4_QcC;tbjQKu!yVRJDv$##B2<$=V&q3;9r|^hL8}2s*$Ogzk9TfA(32Y z4P8&?APZ&^bb@+czV21?qt&_9vA%3MB_Y8*1ci`;B>&YLPJo>xg?pQOOMPP8@#4>4$ZT_h zwD4?kETGN|(sd4AO*KpA0rPy?H)%Fi1eloP)CeJxM7C0NO#{~F;?|xY&8b?rfW=Id zg-ilmVn936ho`726dE)9_4jwG4d$ku{a{CzeV`3Dk+aufRxGc6B)5h!!DLhLh4U0?~+Qv)OKW=E22+SRIGWyU926VFgwP; zyxK1hE@Q+uG^Has?{N+;qjut2{?lQgzTrT35VNQ#suI0;bC4V)<@J`FgvFz8+$*Fh zQByR}Mi!=Beqmr&%^cZtZu|HiNUPbwQF@T&8K_G=&C>MKUBs=6Pf+{rs4#pk2Z<|dK zRY8}p$+9G57fI;M9C>t{&Jo0V16ARYQGg6TrvK9EZsstqNX{=E^J~*|Y}fp`|F2!_ z{lqCq1FydRpzk9g6t0`t+C5kdyY?>3o2+l)@O4f97q!e4V8oQE4J?)4zI}EquI~Zu zP`mo`dp!tXTk7~OXS@?xPh1T^LmsXo5F=w?(w+`5M$BdBpGA4{Bv(M?%)xCAzz^p6 z;`k)JLPlxwFRooau|czu>wwW@3sqP+w|U#)&v{*+#Y2kY_(DTjZ;*LAG)CzzRvG_{ zubKQdwXDc1Sdv&q+dm$aQAMPjFu9eyC2Cb>`8R!^{o)xOhtp(X&AZE7nVC+t z3D@v`*v*50QQrfQ{THPWdD%$m^vV83Ikc$*3vrko8=GFA z7aHCumND= zT)?6?=To^2B&649lkhoP&4bBKh-XFV9JHQL-F{t2tu}Ik`nDsYx-usUoV2%oFso4+ zr6>isFiHZl=XBJc<$g6OIAS_`C$C@0=2&tEbfDw;7zPJsIk6aT{{dU$fa6UIya2ds zuwp=){M(Lh0`Jl3MkpNF7xs5tR1RNswADVn?n|rCE%OWRMbOs$m_F^J)es0EJ~nw`h_LX#OGhX=C{;5!wnA7x$;q1! zv@(bU9E)#)ueTD2ao$n`O-)t9cQAgRl>;U&CsIc^o1T#@wp0L*K@St1S#!LH*UD;8 za>P1`+T8Dbcm>p}mU`CpRnZ*QB)bpUpNJj%mu%|SFYkj2t-ZZXF$p9bYdm4FHoc%<;(z z4qu*ZCAt!rESByOZrW!Q{BDIq@Oq~2p4^+ArDK+Y^z>jE=))S-&wJ`N=`sXn&5Y}It?-?*XK|!u(RDttILRyZ|K1DS*K6?sUmSM z?&l+s?F)I6D3>pc%|*s1sL}X}&*ftn(qfhDb7%-egsWCkA6KJx)3NIx`XUgI4>7?Q zYu_ob%Tp-EdNmXwq`OnwOdL;;#SGAX!)1@xJKdrmOUa5ABTi-3PX;4T7PEYGC^`gD z_3xIfSf!l25=xk~sU1d#dXLgl&T2*^@u_E{A7?x#C{Li;K=vI5;iC<-QdGnUR$U0_ za{A}OV&Pz?==R?3f`RQA{Q%hhCGH*9Fe2H$lkf(ZSI@(NoPo}=xz4<8XqrdOV}aIj zcXiR?{D_bH@UwYiEuWwuFSEfH&wy5Q!(WCmIP!nt zg($mw70IKnnyeEmu_a3SkbPG8g1j}6e`p$V^B_D4Lo-EK_$?^6NNQh4t&obEMvZIB zS|r+yj#m%6}XVcaatqOJH7T=@P2Q4MO z3^80{0_I{N_hF<3AYmsLRMtnmZokSES!q*Hu#{{3ZYFS+1Sy9<-mMC&MlzVC{-+O8 zWMt7m>%t7qCGG;3Sn(l-?4`r2PZ}?CUwT%s!&rl1URW{_v-rquS&@N9)%5#I6gm=a zGe`A0%|)cJ9)ed-dT%mflz%yUKy|z0wfOb!MSU=Byp0B2!i0R}$T;391Eob?hIrm< z+E@&U^%v;Tuiq|l^ZjStA;RqhAYk{&E^WxI>LC3BTsd=W`^t}Uo&`*zK^leZDUFe zl+m{SAONEAMpMpFIbXKg+=skK)_WS#@)v=Fh0*~ufdrsNG|@Rk`}LSNlcew>v$D4w z-~tK>hj|Jk$j0tcl$xo9!>=zuCNpPl!E*ASF3}aVY)vywsQAQOM!T$Gc|p+dXbdI& zcgcl-$+c#;9OJV=fiW97pYn?1?I9lobmI7&gnXOq$%e>v=s#`SgoEv-3lv=fR%_%@ z<;%$^_glBxJY+p&b3kDQoE;8Xs3h}yfqB-kJWJRXr^khby4PMAsVOw`QvwvfNY{0E zrB^!Pkw_Tz2~6-gRgrwMqZQ;(qPFqwVuz8)2GBC~^%GN%uVL?b$bJhdJ_JAW>)KP{ zNaDFV<@;_YHHgpius%oLzMQe3lWGsV;W2`b^aJt(wBw%o2}8$t(6q=byj&?iCVyCG5+nh-zakC5G^RDtCP%8 zz<0Zf#EV~HM+iJk%^EZ-Kl zGnN`d%~{NUswlBSp1ZGo?sbwZ^x)L_s$51Az5#x0Y$+$=0L@Dv-#~5TBCYLF;qxV@ zKEjq=M&eeik5g+MlM9oeZKJufoqk#V=0@s({{6+WtT*^?aOvW`Z(e@E&ak$=UzU|j z$YWHJIgi-F3wwxt%CpK1MsqGs4~TS<+Jti^%C6jQ~q34MYg zw{XBlw;&^ot!bW_TW(KCkxycY5v={SD)~(!(RciM+b2&^&J^I^N6%@AQ1E2{+kV6; zPqa-*r_1+Fxc&Q4PfC1&TkxA3OM-7QxXU2tTT8i^Q#KfUP^WQcBhP*3(Mhi(5}65R zh}N+ErDM~|zZypvmIg>nOeu7qmKv1v!?HmQsIDu?gTBUJwNeQ_Y9dwz7Mb4)&)?A= zZ&xBweFT!7Fp1epm=n>be~0ZWkS$}cR$4tVaxjJQzm;|3Va~n(;b=Q^C;OtCy<-oU zf@?!R2D^-_nDT30{96z83K+*X1-85Z=%)b_%sRLM`2*qnX-CU1y4*r*L%~*WT56g^ z2?$a}j+Oan9<+MTGDXh+4qN36%-r3p zaYSBLV)cFe-7gaT(oV~@-x9xf=+9$Q3z4mA-a85YOZf1BvTnap1z52NhY|2K91{c< zzj5w59uT<)4?Y4bN(@;IQYr`}g$Y42az%bY5)cQ~S?qrbDhA{Q>_ktMf`0&m88*N(;Ap}@kM0g|_)D604tz`UTNE9oedj*y5r#V4-p$z}M%-8~JXEZ6yC1L97R zHZbwlY>PttEB0x2{c7OHv6~Q0Lx*>=vUlFLXVQ7H$vujxyi6@VXGm1i*^RT6`6Ca7z}J5?weyu**;A=7dD; z3!LgZ@jzz1`s#d_g)fusN*_}lb$ZhRmVNs#!?cwRLNIcj9 zur?NnO-)UU!Ga&*zLli!MtWYez~2V7k9Q{ALQD2B`h+9n(Eb&4asI7{Y;@zvLgufL z>iJ>%z_U7N`q@oQ4V-l2vvsTktmz_gD@Fb35k*Ek;C>>tiuTS*BfS|>Ril-Z##XK!W=gG`3v z=}*U#G-sRY+3BjdPb7MqTxLaRcwWX@l# z;6Nvl5wA(Dzejvqx{Mw2cvIR~TXfmlz#TO(tArjo0EXNiRO$=NYPfzl&x7vPi}0PN zAVxoHU1dwsBF{02w7_IP<^$@SqD;Zf1AmsEwGnMKX6<*pfH zGCx}Ml_Gj+*DoCP5IY=!j)wq9Lz~?Y4O=U z(m<{3@}PxrPuf(f($MEKC-=&h6TiV)PG4}wicbU!=XGYI)%%W}C*J1$UW(4L z{n+Mp^UbEMR!Sf(Vo=|@+XY=WH0n1hk(M9aPV!hHv7UsRNFDp4++x#j1*g{KP5Ynd zQvrzx_BVCoQus-VJH}4l0Aa4SKC>|6+xphI8HQCP`|hqv>+}G4QoP$?+N94YO#eF+ z%sYg<^|rzBngU(Mjitwt3-DGUwAqo_a=(Q(`QJ+lfZg8CB7I~3##4ESc)>Iqh#=%% zuKaF&VoF8xYD^R^1fj_4VfBHh<@}xc-83>dG4ThP96yDKNZOJOc&4kB^x}s`1* z3C9Vz6WIy*YeMQvB)I6$RP{`(%#zD|w_H4ldxvdkCxw=MGD98BW}83S>3pS{k>gqR zTe4@e<{XjT?(WfW{4;l<PRkJvH{}I+t^^NUQvf{8U{x%yoYq zFT6wgDsG3i@8XCd_ha#UxJ}oZ^Spq9N(tPPf6z z)%(|$*NbbXBxMU$Q~~4rg{H zAFZx$CM1dgQytk9Y#bO0OzyDEA8Hj{5@Kjq`=#}ZO1?6!IfYPsK~tuf*U&$t4oHBr zdLNnAbkSsX5c>u*D^6~*5BQjE`NsI4xw>zq4dNvk>=CoRQgd}etMm66Aq!^LUv^mcbj7savXd%Z)*+h zc5_aIbuHxuxUZ8T*?D$+elwaHQpuN4*v6O&Woeai|`6(1Z?Wd<~Roh8I7^0*Q`{nrS zMpTA`OA%^sUL~A0cdL6Y^mCu`N3CviWdBR%<mSv&jmDL2u<>P}#>*^nb9zwG*(6|z1}AD zdo@ltwOkWR1c#8*ds_yx*+P0hg*2oz&iJXLq@)dvnQ52Pe%l_!AXRh>Rt zvD~|^eC$RFvNR%9<~ZdXp%}S-l($O{bFn-=Hs7;1Vf~l2GQ)BqK~s&=nS{U;2%AvH z&Q$S<8&Pew-ln6JwwFSF$4(oo6OL0Uh#yP5yz~Z55iOM}nk%MW(>57=@Yg|$Olr|A zpy($kX0do<;jKkt#t_^Pyjx1drl;{W)DIIJw-ME$KQ#Q|e4d^fi~M%zlZ{|-me3sm zB?)N&rb3LOo|JmY>GqIi*=z&xmc9MC$o_*bHRS?y^b zvqsay%O?8tan=GxwV~y$yO`zHYk81(ab}?DPrHLIUJVo|5%~%EDJQ&9yvVV=)m1X$ z#h+ge)lAK`Z6ovJgb;XC=#|eud%?2*4C^90*R65C^x0-xXBFSHvq_@Sp z1Y0GEjvX5Q>Dqf zyI;cOLqqNyt0Rwp(5D|2l_)Z#@05u8G0eBWb(9RV^M}j?^EW(B>LX4EazMGc=5fCynkD+pGbVC%yNai|Cow!rAWLovQj~A zeSNyQLITNg!h~FlgZEvNC6Wyea$hBqn--RKO|P;zCI&mPJO6Fp?3LxNhFURMqwzb> zg()#T#$Egq}n0OX2*-)6L= z+}`6-_ZQe-yuMJ|rkE2qSWlo#T#10|D=><&lXp4z%UQ@q7_b!6=a|L;a=DN1A#vKm zKgW*q0abc`{oi|q>52XKxYSQZuv4B5r&SD4ONKJgfJ(8BBRx@b>0BAgqB?N{aYA;{=n|wh-kX z_5#(*IYo3FNZGHh@3Ru$8fEd_e9tey`WSYSL;pRgB(+-WUZ^-ROv-u797i!cEYig7 z1KQpUnJH!{!I39yto>3OWVVK83M|Yt>KTyIY&q2HB_M0{Ue; zS>8yQA57%;j?iSy2W}gJwS}I6lhT%B88#|WQb2XIT6^UnE&PjT4kp(*>H5iKKb$^T z4jYb_n*O6;08W7|%U{RVL+SnNZDLyAo}PXo#Wrc~Hp0EhH`L<1Gg_+f;_&6BUteZ{ zvc8(M6hYg~bo&%TWu|HIm8wW(h57^fbFg?%Cq0r}M-7^?E`E09Fr$q4n%}X-`TPc41 z-KRBO`}x>_ve@mcg5cdBFt%E;I+^riYPxmPg+Qm$=Sb)jN%X!n3%=%jIkv#NEv=)0 zrZ6>Y@Z4t5^}P`H)=}s_t&XcXyX_7K8s@{~@M1A<46~sApmfJD?bM&cC3K#T_oKK@ zj!A;zXnubm;PDgfv5mgy@by;{am9LgnHp(^ehovh*zBXw=Vz6|xZWD@MnNoR0GX1^_t;bU5^V%>I zDA?Jgp4`{cFgp2I$$p-l^W~t2L4$%Jo817Yigj~MFG+RfaZO%KmLLN0T@aOhE}BcE zvLD`r;2;=kJVmS#{DA!qCTgF-W=@XL|XML*+vNRZ&$;-1?W za*TcKFZ*AZdQgXy{B7r~kJQVP z#0OcSV$xdSg0J(`p04+FLsS5hNQH@&2hJ}M%Pwg~&1uCi0f`amF>sp3a0RVtP?A9a z1J_W0jUx0^`U2!!B$!vAPk*F71;Rzi2`8~c?0%!<1gYvxTyjgb5>_6?-a^j~XtgE( z{e1%h;KxZ0#*BbRV8rTW>QzKvaFk+0L7EU3=12(;6wxk-G9q>PyZZjwWf)@{D4JS7 zj(TF*Bf~#-m;u$XEEoSj?eXsyOJOJJq=dOz%=ehI;J`y${_n~$9pI3Gh>U0{uFT&XhWR55VVBRkRJYO_x`H?>iIiuQLJb!J6Eb)k+w9>yfMLH6xF^${hWlR9+yxixe)+ zX8J9ps#9|m@;fkmaCZjsHk1r5c&=F33wVzJv10a26jn@8Sy%MoJ>qQ#9<^Vy@BFK-Y$zXVmeel?L*j|6~=?-4jzxk1J7aO&S%|eV}t0XNujxubb89W`>SDv2V!otw}Gqo66y%X zg$nYkS*dAXdj09nBgyz68b@q2=iD*P?_37Wml=e}8ngnSOn$KHME;xBNS=LIsmF&0u^d-p{2ucHx?5 z%33S@XlkgZD-WgqG{kG=@UWiwtLKG+oT`vOvL$xPp1@c-D>7 z@H~lBNnM>Bb%icX*Fx7rvDIeAU9RaKbDy^oC5!3qg;s;s&!G1GhVE+f=zjVeeP3Ng zWq-icF{?~<>^IC-=9uWGP*+hK?wuWM?Gz?8Z@zI@u5tzWHgo{$7`2Mz|-w`*mAVLpnu%lNH1iO>FT6BDp1l0wUxSLbiPXpnfm z-T-a5yG`lLIBZ{9US9vIh0CGT{?kQ2J}S|#?b~=PW=2hQg#%CL$5wZi4wN?Wwijv;|cnFmL(P&GGJ%8o(v2n@R?b69Iq}( ziY|Buwg0|8LB5VNARnUP#t!b1Jai}GMr$UBoc0S9-^0z$tIq4Y=P`bXq%9yI;J~Z3 zEQ>Eb)hupMZL3QrN(y+wZsON(|FDW^DcJ_Pxc_VY170ZVqrZq2XEU|((wgL_-wBt3 zkxt=e?8cjEJU%s&bL2STlQq^$0w5MBcLgN~hDSp)I_NakhYMl!$~ zXMJ7zMQ1OhZBB(|k6z1Jp?$bN7Q(UU75?hkqrMA%5m;F;QdDT$Q~xRD;dsv+n>cb9 z_kkQXhdJ1&0zJU?ufT+3cg`P*=HdfRKBt{(pI1|V=}L8~M_mf`Wz=(&hojIee-K7s zf!S3G2;JV9hka++bhu==R|Lk z-%M*P{wRxtHMafCQt8Oy!aGdGOlL(vl*>5(aa$~js%ksSzb(0qBYu|S{x$;lSwtyM ztpbnqRMlc}_iE@I>aGAB%Je|zK53h{uc2?kjRB}1DeS5z*G(v#vJicB?fQsPfN}6- z2z5^R4Sv3^Olw{ffo{L7xS=$;k#lvH99d#w;x2i3eb6|cK z349OJ)#A2_yKbM3UcTesEyF7%wE$rnVrO5ba$jHKTPWM~)7@gXm~N7!dO7Ws>(gf* z!*lktg>PB0q#dFi4%8c!hXc(i4kSQiM*RDFGSLPodc2^{8}XjefBzmUd13u%*JPzw zdUjH5_<%-pxoQ^H52XkmlwqvG^Qu@4CMCv!l$mO2F)*>hw`g0UVp7osA0M#q8m$5k z5GF?vE6bRaLLrrr1b7&PZwo1vaSEhf932k%VYzIjc%ouepqoN>13LSk4K#$Zofdy;ulOhXdHGW08NP+c;2K~z?V1Ep;bRKB0xY(p-2)@(}R zR@{Gy14ID7)63E;s;kMj=)*gi$w7=1yNwn%gsNP5#a}FE_W2v{^EpXCP&TFKMVadp z8K=R-;2pP~UM0po>!3-(z<y6@AO@cYmb3QQ(*a##V>WNbv>7V`$G3CT^gK~yqUb9XU?{%N^NczmGeF0c<|Kl7jI+M zp7krvUml5#u8MvsginKl4883WW?Trn*8R`qLOhaY8)TZ8Run5gonT0Rn}ngmx5=JS zu~5$sA8)+pv}&A{&;U<}c`lx*QXn}zVUtjX3&!D~D<(uMdo~pO7$EOLXku!w6NOs` z+iVREWFv=UMY_yWZ3?x3{ql6jxrnjl&h!dwaAg= z1g$TWt2_JwN-3-LuE?u~qXtpe9R7XYBLQ%RqJh7k;cl&1MxIDPb~NOP!iR1*9S$0@ zkqG7b8`1RrU8~R*vCB#|=b$Fq*CYYZ#$5kbYHDtRhG1TEyro}XP2cfw4bxj;#2(t- zc-+vZA!J_A)I$wt^-3>Fo^$sVRp86-3}jRmg6?|5-Ty3^*|YpD!GhuS*#Qrg zI14)Q3-CKEGou6H^RV+~dAew&B)j*D0+yLlhrXt-zb_okDfRo?h~1pbDic`AAVGeC zW}@!!m)z0*W%Mc%@ZALg#{WJ4qz70>fo$Ci8q7GZ(TXeURY}d34awe4=13bPsiYIg zt$urc9*QOJ+GI7=Jev9167>uib${bc4?HLwf~TaZ{T&{3v#3C0Yfv!ItuMjo#(MkM zTut^Ns~@jaE@|;Oj|QHN4djqiO9M_>z(R*dT^=)zIMlbUoR*6-S=e5fEwlB}<7|&hSb9^oU~JTg zU~}$sRi?6mvtj{L9UVe)71mdb*&hX3jkin?jw06GA?DX_sVcAD+?Zov+Wn`;LBJ&f zd5VeP4(XX^LxTxO~_4#!gzi2V>2$AF9SL?64KgaK zOJHK?G%9Phl^k%QAR#Mx-Chtc8~3l4%(=z+VSIu8vfo;^@3=|5j<-cFSedJ~WgIy~ zB})?=e^r}p)g;&=(4wTOY|;cX!}(v*KNAj!iw}4}oei)`=-OxYpm^ z!`~qM`Ei@sc`krpI1~Z7a#-U4C-Yq>ZoZ|Nx>c$KT?~OUWCQ||L`pHXiB-$=JR8U< zNRJH53!5-1l;PE0MGX=jKp7GEe2A~X=+IqOpgh6=Djh$)El2I zy4&x+xvRC~dMHag_l9mol%>(;Mx_q)iKlEvhXrBa;i7 zBY+pGyx%b&}jJz4-_!u<&eW$a64cj;&o4iY(2{@tDQwbY)v>%#G&M zt6?#|N03%yAa|aYtbTr;vYj6mX#V%~#nt>gVn;0FKYP2dun0!%Jwm8`>qsKO-l(wU z>uz|$C^h`4r>SbW1E{bXIrJ_P^=d0-Lc_dF@JzlrscPsAwy+yhsn7%zD<$c0agk~? zlqIoRCTzfr{ruxF^e;D0PX_dT>%25o%f$Rmrr$9rYFh|7((eXsndKIWVHqIdB3s;x z-~8@et$M6d8OY4_L5$}d$y3a8GrG8sGn1X>I7ERM%Ze>>^Gtj)GO{Zhs~~kf^)Yl9 zp!~p*!*f!A_@@64%VMHv&mN$#?zW_&Agg;#7+ZMghnD>F^KZW8)&Ok07(y$oSC#5X zv4WnL3HZDQ?^<>ENP)7WFw(~dNT}oV<6_TBs%26skP{#YO4kj%b^unfYUzsAShGKy zZc8+#{#%;A2EiM54fA;Pw_v^ZsJm}&*K;(Sd~#Gnoj-)S>ZIGxEXs0y($KEFM`<-H z{vtF<=kdLt`9MS4ze0tP)Z*h6v)@6BA|$MCs}6{T7c$dmCtJE#TTg}@RS7;{55_LK zMYVx!6fXM&2}ru>fa*In&?y##V1ZP>bSz7NkDi{{v|u2!y_6>jZX6X_f(X|8FP=QP z;!p#(g=}9z-;@szh6{dsRCld@jDjhgVj21yJdx>M>zTw}#XhEh6Q1)hwOd`-sG6kl zmgZZw?z<7^;%&_?DM2M2^3gzPbp65Z)9j5w@+Kx~KVo2J?0h-5z8(<>NE%gr@#wtb zv-K$spZnv_%kvsK=X=Ejl3712yv8LU^G(Q3{eKrE2UzUamTA=avOPhJGC8lO&hqZ@ zn$4}Bm4u!397e!)SdzN~)LA}*9W9IH4ns{#Qu1SQ0V6ky-FKA8!@f#2Qak+t9bO&g zZ{VtWqnvp<_-;xww6)_Mv19v7;Sy5~r+y_HR;_F|3?s35t~$ohYP@gg3GrECs4LmH z*l{o0K-cvDLx}JpfOsgq${@9pK_CA#Am}T5WhLD;BkHr>6Y*a24(=xo?oq51!%N04 z{N=GFatFG_n?F?7s&yIh0730~T?SE}d<=UHjX-xZTx(9Fm;y&cktnFnV zID#QBtW_%9YEnIVIb8%<26@78R4=!QP`yQw1GJzrm2MZXHm7a^l_4^FA^ALK2Oxw% z*>4ITCUMxYI7nQFMjUE(=m(>rTxJkL1M9fIphT%4JzX11;FyLy6qChUFhKx7RVJF? zN{`6`9Q8T@9x}Mg7!TodFJ5X)KD39Js1j!rtK0dHu&u`Pp9V=zHBSs{&dS6-idNk^ zsv=b0MMF3wN9~sCvE(>aruajw|4r&st+)e_DACbNJ5}eFX;4tQr9nw)=?>}c?(XjX_u=r}d)L2~ zi?xJv-gno`o|)&F8ICH3-j2X^?72sMGVVx1lkRjJeUcLNf1X9)Z@caQcc=jdDw**FGxiob6*axWT62ecLA6B{4zFgN zsCXqDDk^GvV1>z02-TS8C>@hxiidWoh0u?`m83*%5g}RHxw?95CT@5;KHG1O4)V+KIpV8ruxEjb*cuY^{qv2mvNPt`mc`Pp zR^j4LqRoVgH?~o6N#1U4rzw^{Z|z5mkAH8I$EQ7>PBz9k&lu#)eqO^(%NH+EBmrKY zZQHTjINmPwhyE~uNuVm!rSsVa>nrGL%lg1D>$7<7>pu1>oVVh>Y6z%!{~|;Ac1ylA zjggB!-YM=qc!PU|`MDFBk%S=6t@2x?$^_Z2lvw}HXk!WFJPo~RP#iW}KQR2y)_A`2 zRY!0p{--VyE+PBBWq%y@Le?TD2r5UOEGIGL{L;Hmzk?*KYj0~=a-k(qKcj_hifJtW z4gBJmVjGIti<s8LqipHNV|_2DhBPfjKsqg|LN}Xzo0hCxo7+8-FU|zI>w9l9Z^= zICbv0EjrG9M>AOnL$DZEK9`e^R1#?X&2tI1y|+U+Rt?u7aavBPRJfpnshhbn6K$0s z_$Kp=RkfZ2Yf`9OM9=EmOL5yvXv|rYT7w|se|#MYRBG*_ZuLv6wXO`86Tv399)h+`;}hNBcs~* z{NjRyiUG0XO-EI+)U(ZzU#J;#Cvo{^@#TgTzV2lSG{#;BX9oxEYrIy6IyYe&i?}dK^ zUZ9eHK&dU$e=~a8D)u=e_jBhWf{cs|S)iy<#c!Wo-`CUcUyt!^N2My2>=l@M7|Y$Q z*9br{2r{^71p_k6jfPR)#H%+O*D&=k>Ah5oSTtnXdT=e1xcc&ebc#B9MEwpAp*kyj zpaRry@U$DR1$_SPAK&x|pnT8-X3lNpx~|@efolJY_)ySw{N6&t-$Pb~I%NEC`Qkvu z=X|M)J&NX$Wi#mv?BhExqL2*B|KVBVC6I~ou2_kc687pvD3888HxCUTVZ*JWt>D93 ziD3;PH+F2zhG%7SSzr@2YOOCJfy=@aQQg0@E$rV}6*?iokGdB^$@=x+gvh^%PtFCb zMY6;a;63=~AdA=!<{NKr(?Gh*!RcXZtu7(|&r^+0vJRK>-w;LP*Oh;+{_DW=@x;gP zNe^h<$I62u0N};s05vN9G75XNIq{bbV)lRV`EUR%@pI=ZnVxWU#|BLHG-M2_VJ3ZG z7}BxZN<({SfDk$WF(4NA3uzq@D}F@amZ${cxpR%0YXtRSyMGPbwxf(C$Xp17_ZKfs z>28|B&1~g(z)gTx4)vBgjSPbE8D7y#IzQ%c=5R7x#T$2Y4H?MbchUIDFDHiX%HyV? zSC&BW;53AK5&{i5l&X}_oG~hAzCSpuG*E8p(XGPx0$H(72*e&Dx`T(Tu;IRizgapI zGPI^#yDw7#Y!iTvrF4f{Fb5B4CTUh|wYnU%Ysk!xSH6C0>Tui1u_Wmk&B+5Rr*-@r zzi#L%_Qike!voNl48uM4F!LEsyS8Iv4Cv~N4K?`?eA7QN@g4pr7jJei45)t4#8%LE?)4$dY_1# zyz-(f7d!jI3bFfxyImyA@xWS5xI}(4@R7@NML6POk8A9UZx?jZ>8Hml??2t9C7H9Q z#w5oUY>LfHmrQpC)9qG&SEx_SzBUVBJAK)N?I4j1q9|0<)Vdknik>~|XMlwX;g{$h zmG5i;aTqcevi*txgKBT>ob2d#v<QzT+OfFRy8+Cz|(fx{EY`E)PbZ2J;O@!KUF~mE8`d3+bA}V z3I|&U>jVN9m_8i zQRBuRL6lb4KbI-Pr{>x6zc7i3;=M+u(0fe0e{bkNnbVEau55S*1E2#}Xx8|}jS|4k?95(%Kjxgf9 z2m2XT^kp1OhwlhhB!RLuj5~?@I(v<7UW(aPcV4@6Z!4`wrBKgc0>VHxJ3!a+CxwBf zA*)^-^qMyS%$|sla0l@^G(7-)hqOQ2l$yo8E1%YSzU|S=n{J~D7e%f1zZU^LeJLP7O+6{G0SNxo^jY7%3hBMXno!)R|72= zq7&7!{T5?GdOpSmnE0@t`%uw$HV#-!2DH?{hX<3tzZH${{`h5f^hM>EjkV2@*bw=N z!l{@z|LDnTzLU=F3kdh-VBQZ^=ieMfmS?WX%JV%-enoj~!sIjd-RrBwmuEyv`@?54 zf95R#Irba8QU<9VS*T5><+tGpDtN-D8nBgA)hEr z6lI?VL^{b?pnsX0t=?RDGRJmG{5B~55KY~-FoPR&%*=GJuPC#-ZD?$^$%Z?qp-%q_ z6G4U7=q(_z_{3MotNt^pqTmy17w!zApH-&yp{Ni1zKfr?+i{1rikV&i8U#)yca%5bRpAdQLi!F~VDQ)3*9?Nn&P0n)?`Qy}7z#PgZFwp$!Jw2^cR+eu_4>sL!XzdFSQnB^4#AF7E%a3)C>I@+j5RK*KiH z*YjID0l@9}ForTysCPpOLP;1Atx`ggdZzI~f7*iek(PQNbCFCi%4J9JuoyZfwMfy= z4wKpTIHR$gkyNZUNV!qvPt49HBl{!IZ><>aebzAGMyhk5oqo(Q`|GTEHuNS;ewm8~ zwkG!#mUT$B(B_A$0$sq;oi>?hutq< zx9T5Xv(}E=wu^`si#hsvsM%iEmX@Lp-X|4N;mUl>e7V5sbh9UD_Qxvb+iPOYxUX~W=b-8~zql_YtNDnv=SVVs*$Mw2@`XcrU zNCAE-4^)tKA)x8FuxWGJt+tHtp}Xf_)IpCAw$39G*1CA+b9Y5I?P zwz*d@iq>J@|CXO06_g+Sg+Sl$3-Ge^n)oNdjYkoh!m0*w6mW-6P!c)& z(|7|_RRqzOY0^L4Sz-;+rwe$$Q^Q2Z4(s;jhF79xA@a_djX3*6f$p zLsDhn;^F~d^bbU3sdQb4`bdlYBEF<)`Yj;m-ld(4DyKk{LbY{&Xeyl+)%BgjnGX znm!|y2D6&8tOR3Wi3nen5>zOd&JI(?jRh&=V_u^+hA*Ju9wwM_eBU8Jo}j02nOr70 zKU~-(eflXV)Nk>EzFP^kGQ*BMg?@qM{=IJd1%h zvJtg)OP}#{Kr}&zN8fJmYF8j7Qs7PWKH1At8grnZ3@RLFnC8AQu>bg#=JPw+uBh#U zWf=+q@ACA!suCSU^F7rw#u@7_Q>?X<5jG3Av6awGkn@!#8J`d_3OJ%CpGI!%OATd; z!`t}RXFt6Q1=Oh)kYZ-b=*Kl>M-BslDa7IR1|}b~yB$PgJPz~hQ3=p%1L;OAtyR4g z#jQ2^L?M-^jVx`&Edtg(jJXGu6}d0<_4P3?3!g|bUxhYoq?gjOsH*85XsO@mlzy=> zI?5}U%f4-ob=GvpdYU#Tb%#R2ND{i2`=_M7jweaw@Y&NMI>wEof6o2y&FnD<3ELb)%)P&3?(6vS4D_1ojZs=H|u1$Kqk_lu|%K6w+)ckoJPv7fZiI$pK?+w8RrU-flLlHL0@{kFl); zk443VU)0?2U4Z=vHfIs1O<1EtzQwQ36_IOgulY}2rj^Z-7`za|ud%P1FK^ea&iyW? zjP%yF%5rlTWzA68P}kz|4Fw@?^b>$^b~p5Zh$zx4zXv zJTth5FohezFJ}CMX}YWS164~*amon9I-7Tecar^abP>s%{XWHZiVl!cv<4_Rv6J1- z&=oB-^FN-Md|VR2c@3yDm={)znYLBu%nGe3fm-)94EiCQxi5!L%>Uwg!OX#Mb05e} zGq1W|mpL-?#&A9AFjg{WCFC0A0~wgse6hDTg2;IiDQU+cD*?OeRJs~P(DFWUU{h3% z&i|G+B0k&ZP$pofT_djLx}t?JhZ+s|ZS#au_@@+vSAZ{&W z8fqlWC`{`K3-af6_r}T%h@xf?Y4u}pVW73ts`&*s6{FUCPcs#Ba#_8bZ;x$qDDf2F zi1XZjliKf>THA22BgG%=Jzu2ZqOj%!S;l^Lun8{+ZqG){rZ;6)>Bqj;iqQ}79hbJFyR@E1Lnk9lLuJ7**O{6&^B zU|y>yFflj^EI9z$S<(1B^~tm+PDYJ)!Zm))%?@AT z*|-I=i!L({2HJ`c2fHe~ma=F=6yh@0iV3a`-IP6fvTZ(CPT*5E*XEQHlHi?5C1S3(L7)z zVSPA_M4mWD%3URP)Fs_vA@8P?uP!gE6S2{iRD@P98-Kjhof!S|r_Y?$5s$ijmYat? znH^Ou6Ci0ixj4o1A&ovzQjx&{bZG2;pYWq0B}pI@ZAxy3(Z%spEr#I`<=F;v?!cQT zSAIqqq!yywG2=!WwMwDRPNoGPcJ#L}Mj5*|1*~LLhQfT|zy2;M`S^2g4hBAj-zCa~ zzBZ~b#%d;p?7d2gOsK{7FA~UU=ZEvaum$&*0Ez(}M>myVnKZ~=i$vHEF=#P;w{gc3WRCzgQ!9M9*TJA6= zJ(o;aU+>|3MJ`!+^Z9spSj6$#{NSe4E?lk8eDo`?KLgXPutc>s7c_`~ht_|qW*zI& zX$BLmQS}5iOrc(XU77oAUZNt-0c+1_@IpkI$i(jOFx22!sq4F9MgA3k?nIHizKj`1 zvalEH-w*C$yn48VBRKJhyCIup$EE3R0mc|rYLm0kJGTuP0YnquQ~=)j@5MWBh*X&X zH%u`esUVa;00>yKZxe`zNM#>nw(p-O+As-Dl}bF-m8}>YuO@HBUNi!u`QmGKCmTM4 z?k_KJBIwvz0_Gdgp6#(=@ZHt*%C42K;H{H`FA(mB0R6m+}D68w)?Ce1n>L1*XF6cKmlUNn@G#ET8CCIbuF9dF$1E{S5xfSGiQi z0a`X$n+ME0w=J9}j6V^cdzy#Ay;+dke$KYS?&P;a-58(M_VfVf)~gd8;7Fmp7lD5y5B&&Pw$53l+P z?V8R^jCGsx0&N|q+owuxCsCHV(}${9t8`>=nf0q2FoKHTdJP`RFJrHOB+B9Jn<8eY z6l_7bxIEF@Kj2@H&#x`{L7*=XKKtAkrUCf9Qx%b}e+L|dv-EdIqV z_xq>4f^3fu=@qe;0mkiU{dKzbviv7Ac31Muh9JKa_4Ud~H}p=xk^YaNgD#D7J*$f{mv zq|m($%5HeV$4LQWsTjTJ5k?`MSFm*=!_C6jz%ME8y7OCwd;Umdw6Y@_-Wm2hp69^W z+}EIL_xBE(WkDj$^)B{SB{(blJ(R6+!j!~U9m~1Rvh@t9GT%<`?GFW@<5q`vSC#|C zAtVxoR+%OUj=I&4QUi=JK)hiZwgWM{$3#S0+|8p`Ns#4{fi{4FS&~mo3n)4v-)%aBc9)h{k;* zkM+3vO%2H|9~l z(vAuM`GodgYLsAIYHheyc){r?paOADVg}2=IgrSzD+V54zXSK;ub%gi$AHuqaFvG; zsYQ5dAhr*t%8eNhUMOUXt`Uva+tq=h2CXPWAwHC+IRfcBa5+iuB@sjr0OisTjh+k7 z3smI@W!{K=W6RoS11Txf3n6_me~GfSQH5ia`;>Mx*4@&*qV>9*SZVBN2f*C~I!k|b zmQYbj6_)q))q?oWA~}Xp@u%IvVkr!t&4etnp8E8L-B0fW&^0vhZ3blXRB#j<*O|L+ z()!7>=o{Czm(jgMd^)7Pnw&m>?#%HN&nCcz1T_&A}dzV zn>iXHx=1W#B(MZ%qTjc&1o=6eDdq!Ol65wG_eLhPIT`Yzk*xD3e;a1bDpxu`?(8Z( z`G`UV?3Y0y1kbfl6cS=T;ml>SRKS=QXkANzK2_>9AZr@eD96Lw2{rV!p(?aWH7#4; zYZ`IRyTOrws-gkCz-?LyF^Se-f%^aHi~qj4P=RY7R@`#_2#s^n)|%@ z`k)sY=Ud$p+ped`$c4sd1rFtJGa2GznW`1mJcCI?6B5&QD8c+i4%RB}NB0pjI`P00 zG5$4p;Zve}W$^nqX?M&8gaVg=7vOX4!(h{3bp4ya+gxU({omk+(n#tCKf}n4)h>Pk zGjU>}7Z=}A%#4Q^#~<3_oPcOSF`>k%AY2tmKT{DFL|yq@J!=6lw5D8ZQ~lyT)14HKx_-={X9SsLI!GfTPI{Akm;%0_+vH%YX&dsiJzrQ zMAvAWc&pkH?A-TIm}e!Bkyr`;6VZkg_{AwQfI&&`RLyUZCof$kLlD*WU&Xk?E{OlU zoBje#?eO=tNJP_5=FaJvbD1O!kOJn1A=+6L7~RN-QtIUcjorNqf=QN*YYiwBYY2sR zt#vz?H&%Gs3B$=4A<1HUUB23cG>T7!FSq#K_1f zkRn$4%q0DhsnJ85yZ;>}67X||l?d1g&@sGF*Xjf8f`Cy^OF)Tc3JJh%%srFSb ze=0=J_sYkOcRsW8J9aUi{nI{Oipy5dE&hqa=Shi~Jvw9%3MHBS)mN zAACZLXLU7~1py~YAI~|4{Gj}PFWX&1W5m@C>rym&o|VR#mvXH>IbhAbZL5v3gT-^a zlfRf4WGk|hd+Ng^fmD|RjA~t#dEfu7HGn}HVu%`z!iE`2{=}AVg#-^u2ooWo3$l& zhf~DaSkU4|om0cK+x*HMv}|;K>|DC5Vnf2t055R$tPNOz)Q=h?;?70tur7c4q{fmL zH&dJGlEJm;w}^sVua~q`x=_oK0b*IU9^ukbz{bCTP?!p&uarvs{Tnlp`92n%r0b_g zEkW|NYOcA>JAR#@c+@$!=e2is3>WG0{c$tjP}wk{kI`F+Q?Pl88_D8FH1*2IGm8$m zbFV4Vf*7A=Jmgf30Ns$|>q7+em?S*h&7*^Mf{>*o#TOuyzk3pYX4X8LT1|b`+0~clYT1DY(yDz@ zX-V2xg-NZ%)T&RqqIhaZUSt@Hp~Z0t@`JCeYd_L*fj;G?#|9%J_90|YQNG!pMu5vBkv-7 zsrA?6Y-uFSP9gBmr1)F}A(0chX?eap z?9jlFA384fx16!?&=^;wDO3v|cuOhT7s9i@qDARctI^iz66HKhPD#4RXlaGgg{WxW z0EUtTY*8W!BALF{dpWT%?8p|}f0_`_A2_M9@qi{(8oU1#)Zrz7*Q@+3x>nC{fwjE6 zI^W;%=&b_9Cnci^Itd}ZCwA3@^nAx}Nk6;3H8#@b7+?A{X9`&vt6}HT5(tYRfzW@{S>S_<4l1yfExNBAJ1@ z;D?WfKFSiz&>h+Y)?+T++Zy1M$chnj?7WT6zawlRZ;2W+se3aNi7Rw05Y15bBRJ1_ zDz}gHm{I@w9dIt@{gQD~b893^yE^(9bKEqdiUgMhy21~pGpg42R-TUux!3+r)Eh&??FG!er;ej@mH*4H(7$yM}yLFSgrmSo)U6?)_m{a>GKow zj#}+IG&_@?Kwt9h@eB(%oWZ^~B=Kl-{Z-WaZ=qKY36KNkukl7?NbJ#`@ERBi9m(>2 z@-$P`g1F1Ds7@yIAxx-F#sxXJN(Tc)XVgE5SldiGEFK#d@MJnJl@F=WY>Hp*A0L}B zbQ#LE^%x!Z9CKLR38)m|&J?}@stBj#Mh^*{K+FfSeWDHXKuP0Hd%a7B-n~&D0JxA4Y)6A2JVt*7(goN zN=^kAT?Ha6I{s%8SW^u4WOr zjR_jsNXxz_BK#MGk&31e$lLhggdwb*@0zJS(Erj&q?FWPVnmjrhB&G`Nw_Dyu;YOp z?_HJa^@A?QA0+k83eWH&4D$`rvj~pTR1OYsHU5NnFX_`g9S_MGi47AaLy)8qEg7x< zJO>yg(DeO1Zv9i}%gD6R;6S=}1xPO3c?~XcV!BjF=$%X(8@$lqvkp1b$HcJW_Bt{) zT-ex!Tu;GlzSvMsmNF#o=zocaWZG(QMRttR5shb7M}<55+lkQO=7VLJz+yxAuk~^l zV%mz9z!%L|oPuP^qp)gF#?(3_Ien(e*Y|-y1!^QnXC?_2S|a|2Q@wT$djKF3Ac-`Y zR<9|czRP?!{IXfm(S{*9(ZoP(W=AFD+n!w*`*sttwu@jc4zEL9nQ+JG)J?5w=CK16`euUj*&Ot(j3oNEr%1u9xNm4o`h=k?dkG#SId}WZ9`s*> zr?@>6EZH6`>gBkAzZ`a+(ZsPGviUUsk`SjirCYO~BUlCA#=pp#g%auYlI6P`UTWwP zybJ+TUAhRwj_~}9^tB8{=%XK1)1*w3SOYq|bOyd(@=rb;@5-7n^lOfL_SiVYb$5|n zeF=tOfDVAQrb)K3-%@+56}z`v*`*y730_*fmyaRznBXSrW)uh`s#fBRbi)bb-{jX{ zGxj_Ece}S%XSw|hRW`@|o$f*I?(gV`Y64$xh0IS@nj!`~Z<_z(gFcnH2s3-KS0K{a zfyt6|?C3t`{49YRjd4a-R$VqQ5!rEZIU^#}EDqGG;C!~}xkaggI!8Qb!g}}!N%=nf z*sIRZ5^mU>fTk;3>w)#J{8pS6HSA)d#8G_MFJV(<6qDGk6_w4o%DK()9%>>IJj1vXYH* z7Xoi4H2j25(;8#xWR7XP3l1|kX; z9MtX)1twd$pE(2Q&rN?Obt@#h`;*`lVw#4ig8k*(5K(!J1M>9rTP8#VC4%OnyV7S@ z(R%{*)t8^jcq&)Ol9c!1y?=WTmlkvN>g*2LLU=(H8|m>@S01zZMdn)4myqI**=F&v zk$>PSV9%a60O(P5VlKnOLf%WI^ABdpdOLIK+s6H%5v2CP7S>3b!C7W#2i!dXK0@+18G z#5;vS4^W=$;5a;!#AO+P2a51a$V{Q(5rJRZrORG<-$2#W`d4r_b6Ugq5W^}FTg?St zp-olm5GlDGJx4_4{}&36KV|i-{Psk#!lhzqysvGT_wR1Gol=jybY{cM3<0v1s&Y!Fqms*hWg^JEJ{Ep5x~NxWF!_Q4OY$F4>Dzeg_eDC1B95>ER=x z!M5xbm&6Vp^=e1~lDBqDA9Q1AnIFB*c^AfSt-{!wp7_2`(JgK~d<1Gp1VX|U=(WP} zVhtTgcCQAI>Kf}zkaPiDN=7Kz%wsvDz_Hct5LWT!`t9!d6~Ra}z4*7qYoM%2V+&WY z(VvBx#ZhsITTyrj2Faba{upmtrDxRz+^2VrFUu&sRN)1!|Ejy#e;3WN4|&LnO9+7h zA&+Dkj@K9r+^{OxR1#SFNU2$L{>L?9PqZk0GSIw#EaIm~l+8Nky$$Eez92y3?q*%O z73~hGcsTp=)VDJ{_HzI5u?kXvzIP)+)^HBB{DS8DuyXsZp2 z`0h@r<4Fve6hK7!5xj1?PM5Y%i|t>%AjHe>C1#$%#A*da19QwG;f+3>S=v|r3gGNA zYw^hApKN(og4d6tqEuXHIl)VA&S%xhufS$-cZ__XV2@hB5qMZ+(~SBu%=TdE>#SoVXLs$4F&KewS*JP2k?+jIpuUH<7!6g(2;mw_)Mobd*%bYJPzW0E0S zmAvC-J9`E@*?<&Pm+ssaX_TMQ0_j3wjO?7sNI~z?Esb3^8pvI5I?bFa6iWZT*?yIC zwK+wvv$=zs>PWeS&;e4%mF@L7gdJ;Qvjc)SlFlGUuR#p4qH%|b85F>g1O;%^qdeoQ z@$@!Q^8!4wJkl>y^V)Ro@+Ia44;exHoq^bPrU1qe0wn$hWSNEgqA%bfp76&)DWG(q z;gGZ~eQe$I2UCbG1p4^`Bj7PJ0@Pb029fmyah_k%W&WF_4;5WaL5O@+^>!67R3J~W z6XowE;|jBRc7RT>u+f|Nn9~kj>b}rrJS3f6p#~ac1)qFHN$e*CxDBZi0Tw|@2G5~J zOD2eeb#JJ5B0XHDL7wgNT}48C^d70P(k{?<{7+G%2Ro+_1aYwwRYJY;2WDt@ymufY z_k@3TpIu2Hdh{TkQuF&GK1Ku>0U%dRSLW*|G;*R0Nq}lue=rb1^jCx!WLW>@YEHQy>PB@KpBAvjfL>lP$1S$%tB z8w{O|0pzq%P$>fIHH)~PHlM4`%|0Ldr*Rmr1PdI}(i}oK~ z*W6@QQ6FYYpoP}zQOXDZGPOfB=urQ+lt6aBC5e!7D9pOZ_rr+vtPkP4T?uZpy2|4E zYq4eGdT^83-qB@TLO{(<D3?bF^Oo8oMxMaoRlP>q;x%wy8V{sFbl_}&W6(GmBWY70W@b<2SE z7T6?bNb6&O&{p*H46{}O+xm!d%IPd$#y5CTU!iT|d7~^KwI(6vL8Z2k12|EvBHgeM zEt10N1@c_ySNv|h0TL7i)4oZ|_q!&+&sr%{Z(C;L!WiO+)|S}*C%K|2jBwoCRg52? zwa-|I=a*J|AjDNyb+~AAWpC!6(z>1J#(?@!+Gw;rI17)WJ{w?FkJ#*6R(jTt5A`@3 zJb|^!oHBG@we<+?<*+THX3X_dn}~g09Vtz zfP~||;a{j{BSgaotYwd}0D2#;TFww?~JXbGmSXvJa}@oYS%yF7?x0$`|z&iKR!?2kF&> zP_BY15uMk6ysY6gGaU!|r->0jXI6CUHyDKDZ5ucLp}%_+{8F)wd-O`6Af2`iT!e_! z$Z8;M5AET2mq+v?<15KJ(!GS+FlIkHt7vOC$}ekhm2V-U0HOsLDYC2b50~f{8Dd_} z_k=Y!eoW5CAAtzrtjyS3is>ro?r|BQN|VSA$iShg!LaCCwDdIIONk>qMKGwKTeFq9xU`kpgL_6eGgFN+QKbYgZYr_tBv&Q* z@yBmBw{yx;C5DL@$Wf({^z?X7Q*)ut2@VLT_ixp=k`}t=gamCN?l`5$JjwDM}oH~E67a54WteYs?Lw3I?!##pq zIZz^s((rSPu|%gxu(p*b-d0pCYGf23U`+^meuxco@UgvDHwcJ=BF*h* zb*GPFnXU6_v`*sOhuMa?Kt`V3&xQf-kMDN30v(Dq+MQ^TVL`&qJUn3g#PA z>h3dWf))zeOJ=*=>-Akf@^@q(=MBql=2;QGe2&OR-0qur4eRd3L}9J2G~yI|uO~d= ztE45!!^j_EXA7p#I)-hgpmCaY1-HgsYeMJ>jqSBa#?UArU z&3$cs^TjL2kUa;nvkzbiR2=+|Ap9sVLbUyZgD~n<%{ON==jLL4oEf%pD+xR^K7`7p z%y3-A^`T$)1#vN>rYDRE|BER3A<9zOBI4oQ%M%A;+B3rR-^{m?q*sES5d}OHYG>w* zZOzZwyCPcJ(zB{$Gf=b;Tw|RSE2TDi`yicB zgW!ey;w7)qSO4B~<{tt65tLdUlB|0d+y8=SG*iM3s{_DO$* z#6!UsoM>U6>T6b)s=tK%FCiJG8<#%G`438BvKxhjh+%gpm;LP%Y#!ShDh>bar}~!- zD%j)=`q1xI_<$4S2n{@uY0JV1T%tx1<*Wya{S`YL#`A~>;u^!V%h*j;)KEx7*=1yM z$MkWjAmS6b=fg>$r^p)o*;&=Ow@Y4of&O+EttY7MgU(7Fl<|C#IJ0iT*(N!hiY4P= zH{Hj72V;ZV2P;2R{Qgy=ZDi=caa)@xW2GNEpC!_P)SB~ zJjxLBwB#?i)X|0N3XBRgut+$h%}0nFTv32&qIP*UdmWx($+TDPm~!U2Pu=#d zBwO;(+CR4^=&iiuR~D`KsQ9U~*z#7xzsvvFDt(3gxB7Bgdb*J2#k>%Q!{K*9Ul9)O zC~WeDo%TtJ9}8mP_&mc4?*HRTR7n6Ca}@h~KSSlO{6?}_(=Iz~E1TcOIx^LiI&CfQ z1m5OQ@8s;^%xPukmY(YFepvv@>?ngaGCS_=MVo)-ak2S!`B-k~@C&~v4|OeE zMx(SHtB~XnEc~#+NsL%lD}#`!B3Q^Of`9A~p?p0&eh;9~@N3UTK0d5f^8nCD<)W6> zZA6-3`o<@L0AKGEV*RM}%Tlhpc%B2&gQKIL;fFfTr#tmqzZie+Ae3#41sKp;J8^zG ziLxQKEC`gsrw?lkHl4yxg2998ALVdxdL%5kaPD*FXI?Wf7>8c3=lkhv&ov%iPu&g2 z6`!xeADp3vwq^)l=naX?Y_nyCMxUy<(yN2c_5Z^39A%l~9GJksq z;}sf(ZUhDUG{VDMLiuHYrrdhNIC^iL1)w9MJd1WIkXsE5$d5ST8a3JZsOOZHQOISV znLyX>z!mm*2)i*zGA=&8r46A#(ACXN-|MNOx1mVqpYVuq1$R{gl%&*DI?h3aDd>WQ zdcKCozv^0P7kzNgv`ye}q>?gwaFNBIre&4L#-d#s!>;?B>{YsH{0*~aGB{`FL4ZH! z_o|=BMY6?@lW=&6K5}OACOa|lZI@SIv7|%J1vTTgQ8N z>wA_~nO@oDp5Vq*IfsTD`c5{M@pAp|nvBMJKicCc!}|-GfL#FY2999UC+I;lrm(@g zW~*-2OIA+ATZ~8F-y030@HPRqWaaJ7NJ;T=@o6Qwf=>gGOFbRR_H&0vM~u7>c{R}5 zGo%tonj%Chuml_|qFw83i=SWzj#q_kanB_v6RzM1t20D(v*M^v%%wSG5v{#Y!*Ndf{kQ zDlU~$Z&|WEf+N$ulD(m&kD8oR6^W~^?OB?;trq;os9rzSXbX%{Kx9XUMbtljV~eit zAOXQ}lF0gpr>O=WrIK3<{-(_Q9QQZ1`TT>llygwRq`i1-5NoV2HKq7H*4fVkcl>6H zkaG0iFB(Gk#QxFZf_PqH#aC;{R*3<5x{(YJcNaoeXp!N2uLn9G3v!_8@+vs)RrJHD zyBF3HO8#HDJg9a5-G>kAiWs&WV%;cC<3ov!|M~D>Vw{JEq!H45Z-LHXP@*47aS_Lkyi8NO$Oa<7|ryhk`oyeW~vutYX~(fp2+0?Ecdu9o4)8y9Q1JYcu{bCt-x1y5VK6!Yfc&YJf24q!~e(_pVZtF8TF zG4SX&x$~DI$=c7^=a=On`cZ#b1mu-6#cu~(kr0vkC+k!kyTiXXBbw(|E(H;U|BRz7 z#VS3V?VF4Eo`$G$ctc)vGN+dQJLLo2)rS5l$D+_^5)H@x04$G%wo%G4!Hkl@ETaM` z2I=<6mnv)*fja0BE5!6V^_ZY3wWE7bPr~!Zw?mgJU#Tg_QS6;g4^G1L>y<}$*{=&U z?|Qc;*G$F}KC7m5O~#Y|NzO2dXfyS2%sW3nH~O`@zuAO=iA^RlN`xK*-}*c#;#g|q z(s~Z>Y(fe?#FB^3i^L1=EMd3sQ)n$o1Y`F2b1jH|jd)Zs|WvJ5$vMnw>$@D+_TTBj4ljabrkXO%5 z(l)K3SY|Rx%!SjP(bsETHlGdH%j~iP$LorPwp;z$CtycF;Hq?94Ows2P3!UW`5D_C z?H_=%PZUHOFQD*Rnch|j$17@{?(3_b3m_oyaZv5uZMsdIHS&5npu&w9-d#)ft+evZ zQQdmwd0Y}R)E>;^sg4W4L~te7D$HIF#7|~EE9g_|pR1xA>qrsvS&nYA9I$<_f$y;~ z`*S&U&YcW3*QtRH&B2g%4KJV#D(<=OQ)v;eOGgR)U565hG>Y=abK>`F)odSej!HDF(~t#}iyuOL)>Ah0Mc9 zBFUx1niw00wV=|KA?vZD6pI;+X%Bucy4sl`r6onXG=swp7AQveoSU~n7Oa((QXidx z?L~~(lbqN5{71xuW8bJjETuz2A;IWYb-(I%)Wt*UK{$NYo8sj%+X0B)>`&qsoKxhZ zI{!*i*mm($@+t+%r#@P$n$>Di=~lbUfq_p_Tx{^7*Q{+HpNp()M~zmpQL^yQuRk-^ zM%%qTKURsRFDbhe&$bVW7q_;ksoe{IYpB({>7)M{clgS!=KSMd>_FmoiE0#DY!}l4 zIIgH#?B{9Z!IImuW39i}^}?PdGpi@>^{R5(H9tgpqe+39lb`Bofh;A!EOefuG z?ENlPxSqS};eJ45{^7ho+t2QU{xH%r%NcYd)`0cdym5(wrp5)t}*8v;~eLRS)q(#mJ(izj4hL0DMW}eIHcZOi>s|G z_>ZgcF&+SmR)_o=&tUNY5jB%j)vCjldAY>94l`qYA;91@;u;n^k7PR8+B;;g5(M2v2@NN)q!uPCc7LJH2jX9PH17NwBO2CnBB`7O1a15Bjvz zSDKaJ=5rY(81er07%;npk`#P`rT-$f^({%@%HV3=ibxwDs^VDQK z!cy+mZgF=10a+-Z$pjbJ&?4ynJ7vKTAy)mZoq#HT9X-Sq*bdg9Hlg!Aa+&3fitgc& zW~Zb-0e&RO_t)4S>z;PK;cgwY=A?^4T3bh=h_^ubZR%ZQ6SEkP2su|jlZbA z(&*b_@nL4&A@V-iMn5BU>U%B1;~lge3d8fc&)lXT$shIRFRr&;?4nnVm=Vdp*4|$c zaOU^G$QaMg{41m&iO{}9mIbE0tdB+YN{AYE42U&vpB``cC@6y7YUA@c6)#*Y=$`hG z3dUkQZoTk+9e_Sajy20lMe$30ih88+op*YfhjXOY$thmV&Fs);R4{$h`8`hpvxqjQr@1>f8d4qJzl~@&{D#}n6U?EZmFGuX#i**ccM zZS^T_&;#RVfh0~PiAb`@^hNr0`ab?A*g1Y}a9;%k<w+mIlD8)Ej~S+>P{Km}g4~=mwi&Ud(?T z%Kze3X8;?OQR0yND>^7$^3T6UfC+$bO%nabEBWsf_y80;9g7tG-$(rW!EYYQ`w&{9 z{}ZtPA7GUjijR7Or2hMee?N5EK*1^+C&DxO{_n^D;SvRW;bQ!g^1t){_rnSo6s(4% zz4*Vtsub`AAKNO{|9Hyti`WxF!K&Dh|NkC5hZ6XLIQ1Cue@o;?B%st9b0Nb2d+^a% zKi?j)e`ULZ2Hv&WEm*S-=4K4GdJo<$-R<-L$D5LaBDwL%%^G0HG6k|X0*Y+xYyc7% z-xC3TfuqECXg9ktImeF-2_5z1(!|jlIuKy-1=W}?n3A+P@#LYt2`z)fl$vs#YRHd{;{|Jl|qCJu!Y`U1=s#B*;gR~V!@p?XZ}xv|NpF23a}*jPWn6y z|4YK*m;ht+{|106`u``Q|EQ1u-|Qm5J|?Iq^ganZIPv=Mb^CA$aOikV2_>X86Wr_# zME5p4WY3SB!TqaaUS|TTCI1L>>t78|gNaW<4g2_b`jdY|TJKYZpq{2K-O#SSkxKsk z-l7?GBbn4K4Qf^6>T^H?>Jak*r&Ds7eE8pxqGD%Hj})$=KE(yv#p&SJqJnR&AG56h z=&K=g+n6!POCETOK8@i?`LAaMrNV*JdW^RAFWEnblk(Ol2@D4>W1>)Se>PNf(Wlf{ z5hu2TaK0zOo>XVo`5(T(z#dSrsO=&d&$MR@Kn{q&im;iD>zpV*S*O+1O29RLP|Y^S z=Qb3*ga0pC0uN9jgDZr;cK*@V&Qz>+{U8zRXb1%GVNViqFGgl&G;l{+KcZL>YyVdj zzfK0kw0?+Y_7Cl0PXfYPjb-CC`F?$KD+7dXDqdk?(oXaY2`HhE{1XHugIx!#lIcR* z>fk^6s+p|H;17j`rR_MU*MpbI;f4o2n~^Xcllwn+IRW>U{D1Y|4;cT)lNZK+J6Fr= zwZxgz*q%U4?*Fm75i*_2DHCgD&_@IH$85Ey-)d_5=P;YAZ*yggHr!o9Q1`fHYuK>% z^Ep?fxYKIB;$a@2oy8|$_4Q|u_MoVDQ$y8ts-KdYTw8NT@4tl@FF{gKnF+~yH=VQ- zZ|?=t#J>J2VixsEvuHNI9&+pglin`bjI+=!!M&@MgDPEn4A@r+JjrQnS5**NsB#}= z0-2b+s<|(p!cPCI5f>V{t4;{qK3j@aTwWe)gy7m2v13FRqiJ*2q=$X{0a(9QXQL)Y z;8AB-=C-!68p)L}^*dnjIgG!3!BD6fYz)t+E=3}ngOr@BhDC=bnIw-mUD)qGH>z0> z_45~82uKZt8haotQ>IfZ;dqK{+}m3%%X-#g|6-a976rRkFqRqZqTX@KX5d%$G7gFl z=!Y@(u_AP2C;jJGXFL{9S`uA4*HNs0A0;&p$DncTma|OfJFqM=$~RUaDcjaV!?4IO z=aC$uXMn#T445fGGABf+0JgsZ^wsel-O{_^K551GALITpd zq;py4Gi)$01csGw^~ivRB5#m7qxb8%1(hLRAKC1SPbr`OR2as`m}UZ9#;fdA{+iqduf`_2A<>j(?T*z`@6<kFdlqsZbV?kGH<-E_?qO3BXG<6TN z)(n^%25_z*t7t37z~4+$!wA)ka2*wrrn(4$bUe5bfx)dw9(tSozp4RyNM0Uzr!ekl z6@dF@Wr0m6#rNt3p1kC}F{mr&=VM&4Ij{s+Yv9%x(Nu@a0CwOls}TJge2fEu--^}U zV)K**e(N3epWEjN0w-*+rKbr1m??37my`ocF~%E|z|82bV&Yxj!5&DdBM8+L3`#*h zFAxC;rL!9_FUrXXb>oe}WR*FJrN7Daa!K(9}f;%zZENsp3H7jIO*;&61z7Hn3(YP?gbdv;Ga3*`R1n>l*OZ zLE;cf_JKl&EL)nKhhDY>(%C^IHV1)+-i3BpB53oXb*S{bPR7zpLrt~X@uvs6yq zc3H;5cr?(S|G^j6nQ~9>^PDTdLqo z8BY(M07GA?J6;09QiR4SkuYbUP1IlLPUXCfowYws1@lplPDk&5XTWvV3Pk`W4RvH` z6ErOvo792vl5sw^xt1DJlFmv)D4lq%(G9dfMf10@H+QM7#Qzpbg!^^W@)-&}uc-^@ zm~#GjmKk5bur4SlSQ}nSMJ3<3kW@KZl{Yh$d=*O>NKj@?bsfV#!9c7+MPRP;1j{Cn z;-AJYx(;metr=cvRpUa|Co=a^VI>f-Trz3i8N+lW73)PhG2_}6#{h)wUn5^G! z?j1LM<>T|{xKrO)pT)oRwDyzz^lJG@K8o~R@yK~tMAFw9?1$T-zdf5-M^7K#$aidK zsC!wdfy!@st$mE?)PHxnKpi$=1Yp zJw=Oo^X;BTiN3D%37cESeMdfCm6PfkeY(qcP4bzw(iC}y-?pvWiT+kgEPdXQ@z`|} zO3Z#Z!N=#a^lEv$`STeJZ0ojnTVDYgH$#IBSDMw&6$H+*oe-%cHXnd{O&t(|hXlu) zxe%Lwy6~)YX+CUB991i+vwpfL+M+VKf9aX|_nJzxZeQ}@akJ=&jFib&gyi;Od4KEX zXBv3l$9TC_@qS0jbO-|$%GCh31i$Pb4iBM$TC|m);BmJ0b4o(N{trIJnmy5dha)pq z2OqWl%b2R+G;~;sRNK;aku7X3Yf#EAT7|^K z#Z%pEctuR2-IMk>`Vo#0^?6hpt3pNlMLIU->u-Dc9rdJTG2ZjsG9H@M=gNXMMn!|; zb8Vwlg3XGB$ZhSrnHt8kLs$KsgP!AXnuN641{Uz6YI+>pfWo}jaHrTd7fAOPPHyiE3rRGCqva5Nts)l-RuS2{^ zgC)QFUs}($nQRZ$IC#{!n`FIGNQX`o@dDG#^1~wfSuHo~3CeA3s{0v_3$VQ#0j~yv zJ4giNt#uM2sj8~>5i-O?%iqLOU$Q@7k8GF@IQiZrjvIQt%E*_vNi@lpPZkomS7>YR z>Xsa^!MOi89=$(vw8hwT+PZgnaiJz?(qZn_{;f**@ihEM+V)ZgU&tm3TfI>FcD1d> zXrib_KUF~R`1mM++NAVkqf1wtnwCerq3lK&rBzHaB9;=twO@+iu>d4tJka>G7u|eB z<9mn0+j@nP%I2AiO3bNvs+1jfrSGXuJkKojbcWI_6w&ffJLMRgcGM&~duT&g=TV?Q z+x?^6r@dx%y>W8sa>{${?^*O?bcXVq^irTdR@^N{QmhfpK-Z%7W^m7AHnKT3ZEXT( z=!Xig-N*ao<+$1#UM&>-QbEG`sapg3+wNrnayrW)T1*NlRYldN=BWO{si!eevEZu7 z)9ih(>9DXMn^Wr5=UH`oUO)rDL(M>EdUpwc1Oa?^->auv{b6#dW#_(k?n6D2Fb{kr zj#r0m+f$ECX)md-YX_UMm%63Z_svM?^| zhlYku=Y&X<=ZYGQ*CLt@I81j+?lzkF+o?5{U9-Kv2;HBj%YM<$2e<3XxTpma&DAO- z>A2J`oE&7&?%i+Io7EYFn8$1(f4Y>_Z58I|n$&mS@Czi zx7v3^xv;l=ivx9jCOG{|w=?S>RAX2=d`?}cThYc3tiCgBDm`5=c|eppVmeySV|A?% zML6i+DEge8JBC7Le`BLMd@m=Rosw(eis$?l#ES&g*~gf$5V|u=cXS|bX*HnI^Z>>WYpD2l<#iZ>oc>u8!p!(D= zj%oVEMh<9B&e(R7)evg@6`3{wbbvP}EazgptXNK7tJb;}(z1au(#~)q^I#MD`J9M2 zoeWR^fzy)^%o?Vm#hxOmnC%w!;O=bQDqe!%wfCOP%SQ7omr?6;pe|XrL)m&We+Mospaz1`+TE_oZ?h3ZkY`3^XL4Cnf#yxX z&3tenmeCv6Vs!T3BHbUH+iM$Wa>J~w6vLeSMY@`0jf?`pp+Wr@Zr<8UJ|}cHM|YLU zuzlSozfxrSi>wRBn}_yB8*G*V<~T~ODqVSQ_Y;~o0@}`KQOA>pudGw8;`Xj*gH6JE z|5cfde>Ohy{mC1}>{9(68npLm)3LYpjSm`HUh2b;tR!taNx>R(J8hDg=R|Ks>f>}3zppONh9QcvyZez<;MP>3du46Mx0Ryh?TY++*mxmnTALq@cfFwd3%&2{5_IQtC zV|o3zv^3$yC{d7zXuOFG1s3H^0;^fjCF<|W$j*EW=Tp);Llsk&xOe^Bj4sRQvkjjL ztgkn03aAOmFrHRPZ5NB?(g@ylj~)xoF`g{!h&tgV|D_*Y(}}=-Y^?^SL7bS%{-^I7 z(mCbXpLdWnV15}uu1cztM-4Zy+gRQhoKaexoLJ#Wj10x=L(FfhmM4?5qh zTx{)o4A>J9L^}=eyjsLeqZCKesXPCJu0=g*XI`u^hH$l;^IG}s$@qKy%9cJA5kWo% z#7EfN-)~}E)VCIl6ci%I|BmK(&y!`W!gd|vyeh})60Az9d^+~!&m_GK01Ec^Zx`B$ zJ(k@JYHYb&h7C18Pb?lZ`&A)zaj;gYO@zj?ww*WRJ@k&NP`}|5t*w@awrKGblFONMy7?I7ai5;Lx{U|s zFkFH}wciOGJ*=@4&)Vl9M=r-Zx{tWtOkX&eTuq~0nzGupoy%@G^yg2jkt?4T({Yi1o_RWn*e5p6SvKumngFW< ziG<8SS4Y+_jjd?S*bE9-vz{OlsbVIeRju@2&KB6zVom z-zb#BY1E_2T#2#du1Aq)wE11TeQYyjD~sr@Yn<)XSJ@AdIry(T6ycJDtI*8HtlPDZ zImKe4BI!F`J0^?#9ejV&GhNb2!R3DdET4CD4t4ay$Q7J21xwqdYgj~BVm^Yag(70U zkk`4!ATe=w@j;H7L^J3%7T+b5hXvSo`B30HnYr2uyfLa60BLa3bMAZN2D78f7^=QR zeKGEH)o0%{G(BSvywP+O|LpPbTtb{iVFobbr| zz15jLroZ^1O74EP@42tZ@3ZKg&&RxC=8t&U*X3=Q?0ON|d{REAEgb_4N>9pPgU$8tj46QghL;AXH4Zum4w*D4l!wfItt)Y2PkW+Kb!yBqcDT;8K3&W z44!6rtP=OJ&?YM56ndy#-yN>(4>FE1S7lC!IIQB^1Qf?x(44oBF>JgO`{FMCG00cT z4&~=~w+=R}n1ST>-quX6Tp0rIFAxZn0MAg-?O{eLOj19dCz@$M6p-79dmURqyhQn_ z>+$sAXhF7C!fw&LA{@{~)ZodX5glX`W|09!Ko7SDckb3}o2)|?jEqroVb_g9zzI zSm(~*`Uk*eT=bY{Lpnb4&cWqYNbi2uBjixeext$u190H7C=9MB2}On2ln!8fI0XSW zd=_nn5%Aeh8pGM41Hb})#+$!Xia0c(*7^DZy?Mt?%N(RzZM)^Tj)4sX)y^baTY(1ykBbIHrGz85>msMyT=8>l29kC?? zq}U1)AVC3vS)&|6lutV0RB!5ttk zCn2XOAcO|grUOo6ucSRe2-J^^-zoPop3#v3PJ$gS4M1lC?jJz6w75WBKVdthvI0^~ zJFJ>EIK+(%K4urrUa5wJn_foXxlqZN0Xyal)h!B0!dm$RIELCq_r36MqH{=7L$3n5 z36q6WPbvB1f?@?0Cv$fum9J6OAjNs`(A8n<`Ul# zeghQjDi~l@OnydN@A9Pn^fJc%>%a_MzIf=X#|icTWT8F3e-du&Omby{X=ZdpnB5}a z*N?u0x=6}cjzem|JN_aN0jrh* zWrAyu$hLq?IDp1GrVN~=00^Ss@I*%0oJ`W$Z)3Ox|0MMO&qY`<1sy!}epnFT3#Ej0 z!t^86d~+HLJd}v=k0$U?_gBD{)vM3c0qVAd&7fNrI2>QIvwh9f+#|q@N`Qswa=383 zK&$<|ygYI$fCvB>AeboQr$KZ0ZKxXhSrO3BT=wQ)gF-GG?ydpGV^BD!?%R80$6Av) z7nzz`gK#3+B{~_bhj04gzB{EPU5{O+AK|y2W2}YHgt$8Go-n|5*7+i=EmY$Up=`lq z&T6Ffzirm)>*t|Yp@~i;EYe{#sV|7r$YN;PEHd$&R<}?$jjYaf&=TIm;}z~`0GdKR z=l|~-gM+f0c9$#G~ zKgPbecyT0WB}diBC}lO)U}~F$d%?oYsAn+-FWu4mh>e3CZu##;4)3Uh6*MvYtD#in z36zSw!COv*6_b(`9fw(0N&(aj2C$g3CzCH6&mxoiRS|wJb8~P@w{tlK#UxR3UQA(Q zV~&59PyuOu$o9eB#@qH!bdUySi_MA$Yu-+|_`r@7~b_ z2fX5;3Fu26Q}TZ{NS1%iEo$5TXjmjnTS7PeQ?r82g|UpU^lnm%V>1+9rDr?lbx}mO z;pU=jAg%%(5b|q@rR~z4f(#r2H+a1ghxH%T%VHiFy`l;cPaR#gEM4;4)c*gFgp= z)36J1T3mb+6Yrv9S(->X`!4e?)}gZ(+?gK#}G0bF^EC|6>DRu;zvmI7%t ztRJIXEgTo0XiCCP;JKV}Ny!fqzSPm9Lyq(@063i1Urs6aGdtoiU}jX);9gX0{; zZ(A*MS-j(Tr^n8=X%Xfda!xBd15An zPo@(DCh^#(rlY6lKa%zKk}wPf%XsJoPHJ#xHho^ z`-dz6zu=`Bo+TOqhs9{|&36_nL-)9M2GZj1zbO~?d;AN90hNrT?g~|%NTAuGvx2&^ zyoua_^l<9Rw<-i$yT*P`h+(JUdvzoj4K(j3bm&siq<{(!{SXAjm()IWl`9Z8iwn<- zCF*OmeaDTm5RO;HZJO{15xX^6?24w`g>n*3VW?=YX}Sb+hquRfyj3}q2P`;8LXtj* zsb)XybQj8bxwlUAI7o)pN7k2XaU4V^QC>t|V3SNrA(=nal^Dg1=iS^~^BU1!KIdPC zKr1O&wY$QA>Hi3TIqWC{P5OUfqSS!nJq7A`4xMFwP7L7(Q zF}Jjcdv&Lm=plyN-&x$Hdny>ys8J(VYdpJc)m*j}5uY;iC!+p?p7xvMLPpO0EtH#I zHL`5f7lY(W(`Fneo#z z0j(M1#}74gG&WJdP684PI3qc0Sj-wT7zHbWztTMv$;s2k&$+t2K!+S92YTEY8{gBW z@)?An_m3_%RD4IMZBUaEieu=}W58oG8(`!m*Uz}su}yUj)rLfQ>F$lIqfnsh#W@BC zU5EZa4pg}vD(y|2yblDllfL0kLz4*XVR#8@{sKD<*MOsX1z|Ctx{yNn9UP+E?gL5# zM0t`=xBtY8OraSZ*8G2NfGIo6IDPnOs?2kq=@XJ z#qxwaDw4knUWdSd_lY&N`tL_;knA-g+Z1oX55+%obf#Yhq&`#?06dX$=St3nvk}*l;!Wm)3Gfw zK!^Sh0@?2^;)cdX-|#-mx8`=)KU9%3ZcjU&;}32AxP}ua7ax>0n4J>Ca>fSdfW&v~ z{gYSu70oMo1nGpEL*A8r0iVJPe2r%w&G7ci5MF)XW!O}qP6=kfnf*=4%o%JSxcO-$ zI)j&y<|{i;^(HO(CSn@I`*SY6V3I>sZaCRA>P8ml@83z~V?D5z$bSX|5$p}10>XuY9_dzqgL3Zuunn}B1$>V6V z)3sh?@X-gYE-V>-Pt_}S0o!k;udxjVVcu+MGjc$Hg#0>OCZ$x~q^X3j9oZetV}1tY z4=y09+VUMQt4#-}frC=<{Fkw%WUvF+y+Fx}cT)?s@Zwl@p;z`o$*U9GZC;bvQxC|S zFtkV<@VQ^%uP>WIp-a<(w#UA^ikjIcCFsp>oklYbRoyI!!6}-_?OO2S=yAM#XBl&i z{Ct7YMD5zFa1|=1sbYp97OR70NLjgE;yIYt`6BaklM7VJ+afP z=4S*ww8=0*Y^G+;EKX5S=m`d~HU=EwP}GVLvgIxtPo#Y1{dA+aw=hg{Um@{!y{g+@3I(@`j|1HHxW zT$qVOxR)I8bpB_Qa0}DVN$7Biw0Du=hn4S5F$1(Rj5kxe7*8kV=My^K+!uMIuIXMl zh#9T2J@12%nKaaN`?uK+Lb2`MctJsHN4=~kGnS^Q=C(M5FA41itljUNhxbT zo9i0H@4y#SX(XZrGn~^%(|dIB@*67YZ|xhR2=L49S0i2y)f8nDr5LWk>X%c}&&-

aDmiYeUb21(W{c?EGrN}MaFaQ_^9Tf6ZMSMZwT^^Ql}b!m%Jz1MirsO9DP9`wK4G(Im$H$?~2>Mu&l_F z1fQscMoU?nSwwbzwIsj*9}_hi5KYR#FINmh7u&z8 zQcz3&(+i+lD<5)XwKRY%ZWVkvQV9Dik}POG6xXI@JV&HE!Yh;~y6TQq<)3suGc4Al z%A#7tDpEjr-Ppn@RyyaahZjG-A+c2)iBBxWUU3*xp)NTZ=A+xQ+HF&)ES`kgy#}$L zepxjsOCQjR0K}?Ag;I;5-@et3y1o0~0IEnJiXSeiZ*?WCEl{kZs z#LkP6q!PD0)BC2$lkXGinBeEyN-I|G(!nN&W;vBt@+E3C&qJ2d>T(488M(_s5o&ud z>oa6B2V=g%1mVXCQw9(p$?~_0jdC0cwuz;uk`s{MTMnu&z55}WTB?W%&f1pI^f(vG zw6m#^rLYYO7&)Ue!42IQ&%%9~IrL&q*lQT`11V^5^zW*cCoWK7JS&EuCxm#{@Fj|M zC+HHtLwPhuBpyDJ;CTg|lVQZx+>&2HYfypIHTfWe@9y%t3uxhl zE!j}q>ubh3-D#OKZnLA-fxDE5KZ1RRN#9CdeSWZHpkTB zGA&Cyonvd_Few=>n3iyv$O?e@x}97?Pb)3sQjPuomTuKLV_GN8Re3RkCje`<&Rlxl zA2dW;XU?i?^6Rf{fH5a_6|{a3cg`B9M2}E}fqWgcCL#3kyNl}3U`ep))6bdD0{C8qg8bEbwrTF^WLHVb$d_;dZ@^|?ot=^@iv$;E zH&EX@9o^E?n@KKt^vg+M#ZFa0dubsANJywQLt%wrjtwdH zakGiZDqat4*0)gFVMze!C)gU+>EGJsNc-fcXeeN_B0q5>fN#N)qbVvJU6o^R(_nUq z@<=>8U#GsahLwEvnw_JPl<2btGhzLpy}5gUa2X5urrMNdxzi6#5muW4gf8WkaCmYw zt-84oEdY?Km_|LgRKrRbQ+CN!)QL18A1x-Ar9b_7 zG2tCLx&D(LU zfibq|ur1JTM>_ZikIOeQ)OWX<9QW8jL0bVD^Pic5%IU6;F^^rAaM`O7KRI}{V(4A^ z%bcVKS(!2}I6nR0b_n!fs52YQ6G>MuI$h|Z!uJ=&WTld^7>n75Ui#h1j|fnAr{+h0 zx5C*^@RgmsZWVkEw}*iCRH(KBJo8cQd1$Bg3QHA*u7 zZM+e~8~C$sMSbj?BTvXW(L3RCyMVb3dNk=3bE~=j!DS<$m%#i$iLlFozLYdk&JPpo zMfj7BAU9a|pd3rd$z`n?#xY9v;Vy6UUYT1wbTiYg(u zXxH;lkHzb6VS4pkZY=O3Ag+?$0M-l9px>i@tZqtOo9pB=Q_7p3&WqRS#3U$b@Ig5p zpVnsTp5nTRn{vrP2G;5X#y&#gipeH@pI%jC{gn1{4~UQ**b6WO}@MRYd(M8_1B4ejJ&4B1ztzPCh$&n;EJvn9p$|e zh)e`=8l((76h7@fbFE`ZcueweaZrAPRS-6fg}^tOM+}>l6Bc~x_w<^%0vEmGt}z-xA(g#O;+P@uy=f8~mqY7hX(zgLAxRFH9qd;1S>%z~ ze3s@i9WWsa8Nwe6eM2a6C^(CFp zC2P%m(4{-ez}qUO)0yk7GC(I%`$k`}TLS;5ZtTRsu+rr2p@q)&cR3M*6|6oKC zSJa!W`g=$4TVRZ!c960<1id+k`06+Rcd~dW@J{l40*g3TA`lV@70&}O7l;o8=sC?O z0u%5W%x2!tOzaUnTGVx7Bh^lEJpsN&=rhSrDiQCV{;?gDggO&Cj2KQrOhBnwkQ!wd z5UMpyBp{W}MnO-RXCCC*EjmDIC=O2K0aV7z6{Z zaj__^-x>V9i!St3#ovB{N!4Its|=BpaU6?*3m9Ehr-K?;yxEzEw%i#xL2WfMtEln> zt3Qhd%|L)yTjE?ADAS@rge7ESFvTKrPY`I$Yt@$Hd*G^GTtpneRgwkD&p|^SA zYAB!!%;--GN}6Qs*}IeQ&@7OOC2H4b0`Qp=dI66dZ@OZ5^ z#XV)!6#?9Y&P`THq47bDl?V~<*hu1ihKO>XF87)yKkU6vU?vj_fmgsY?nPnmdF0z= z5DIkF0jlg2p|k|u>se^yam1AX2i?zm!Ibj2CFD-~PW~tq*VwBSPv`!%btSPz?^0W) zXru(+_XQ#pccQCg)Wl$@EXNza>ciyi9Q=_T&xo_Md0T-79{uDhho3Ik-Gc{#t%Wd6Ad_LO|WdC zWUvgFe);dfH2^$l61V8x7K8k)=stIfN8a`i&uJ3F)tWv>E;MJjY@=sK7;?~3Y+<4i znjlz!6T{z1BM z8>ahdN@>J^U@~)QHQfR*o$pNhu^6B`;eCaGd@}zmfYF|2*DC|Eg+`HJb%eXO3V>D@ zNX8o5{NtIb>$xbHFhb%V#n)!w=zr!V=Bp)5CH41|91Xn6uZ-J{NkNI(Te2{_Eq1XJ zpP2-u*ISf2b4ZG>srvTnvd7?EeoZI&GX14Ek`~eZE&V6d`rEd@R$e4d1BonY<9X!w zf{!G^PVi_1KM&c_WLZ{fN(ulrvUbwdshVnq0MuLH+{_s6(fipSuc>E6EI2&1Nepkd z<)&?W_*?;NW#cS9eIrT6jfZKB>2TjSsanxC!@YDz;pyvllzP+8!W(3%@d6{~cL#@X zol+RF1kt$PWXs17DKi9v*v-Umov*YaI7ohoh65Y$KF#)Z=Oe89$vvL_dPRR$u8N7zWoMc!hdB{>^7g0=>7)bq}Fxzy9QX7Y)Nu zdXGieV|B(~`7r|)7nVt@9d}eqfq2!I2pcmGjw5JM%7=rY;GVTJ$sQS=66lK5#T7I` z4qJ;U46ex3)GrS23VYd0^q1osy0CCGz$+1-61U?q>SMv+iOV{>`uh?%e)N#Tf6e*P z6g7j1zotQ^%C`z#Z59*qfaF?&SUIdXwu%@MA+4B6#GXK~uV;%0X7*!?hRM-B>MRc@C2`NSv*FvPg?_RPuP&vG!agIw8v$pZ7@6$tfO8&N+` zkVKszZo`JUQ@w!vmhw!KM)GTw2B@S1C7xAXRq477hHN}LdPH?ta#Agg>pfIgf7U&P zR_3}Sp#4o(KH#gTb5gwf{kYJk6^}Ok*xqX0jM;sd5xPS`vYr4T$ktECJt$_|S^|u$ zC`hcn5E}$INwJ=8#Whj0NXIk3_xXEo;{6Db$cRqgP{#x#T*Lyn(Q&Fvw7Ia}Up zWip~c!e(hJ?p7m{VQmQyo7B3xx*t^AqR!?K=!Y;9%4WBvyml8+7hs{{+dq5yn$U--E*Mz(U+B8<~?3VY)Clfbq?G~1KErCJ0+FnA-?k8 zl8zSerS=;*HDmGrBSRY7vM=+}$oYy`kvR5eeND7QIfDrSe5$>q~ zqiYYG$&&spJAlH&#(h~4%#LUw<9&N-r(9`XTOAAN2=;MX=#ex4^$e55mc_|KF?Lrl zvfLG7r@Vv4rK=2!K;dDt(%9Hy^UQc!y}?o#|7t~VN3hSs!+pVz@4b6LY+ld3L|6>5 z5>E^he`OX{58t7Sq`x4C6*N8emjx^`Vkri@Euz`dn+^VGoukj7EAqG9`!X9SrNg`C zQHU+k%c=_tq8w`JxM<4Snozj%+Iy^wzsKWweObnfdIV#U`V$ZeKQJy*zF$^l(N=X9 z(2*v~PGC|6bdPo|E{L%cJfe2fI9*sd9@b=rns$)rw;CU6e8~DDc!YwJw_?_G!MaO< z9?=YuBoOl`Kd}>j2rb+bkqqBGOHn5?-%G+N5y`HNt2~sH02D%XDU*jg^iI1=IYZd5 zFFz9|RyW445l5JIT`9^drw%VRz{?nziJu=Wu>XBfLYN?#=ZBx-uP{0&5sO#oS4nwS zG56QMDr>knNkU3PMi!durUKGIn1iDCVuOu{I76@7^H|IIRbnfL4gtzH9bNoQznpin|hX7BZYlT4+0eLK!b{QT}TAhVsm5t7u*mc+uj~_eh+4r z-sVt+nSj*&-F(u4sr3rCS>1#0x3V5jqx_B*{U0hZLj3D}zJqIzJMA?E);{>)dLC=; zE2x?GVWDn_JA`!7SQ`H{&lHOaaIh=pTf)`i*vJL|0iuNoXTe+}hjMBfg%lN-^Hzo* zIwffOzY|Xl9`>;;=*PtMN$gd9AZK25wN6TBRz{}dx%_v<@9)}=zM*g%M7n~b@Cx=2 z6S*RO z5L~U#(aFd-Z^7rnk6h*yK)b-tK_Yzv19y!kf~l#3rZMuVz%?w8 zOUR8%7NNM!uNmsXJ-NZihK~qyZ4+!^;Rq#h@$>5F7BU`y%}T2kku8-l)H8NHt~1+BV8$N{GUIPKoh)3isR0jquz`$C(ofZ zXU!xcbu7a0@5t1G0wTb{O!jW~q-tDsuDmlvSfpGzT4!k#Ut-P`%-mWO|BNI)IjE+^ zK~-5%%#ZZuI(s^g+@SY{BHi{~jYQVX_0eKFqLpS`XHy3?W`BynTpTGJa~gP9 z$v}VD_Czf=h~^^w^P}umhQBZ_CJ~w7x$z3{`ej#@eGPZ(GMhx>sgzUDBth_<@LNT}VXJ6r zcC>zJh&+-$E#ivdL>_qxP7oIrTH~ffxLOAT7n7ESNeLJpUUorROYz*Nv9%K6K52#B z(Kh$JD>D}0L80Ju!htA4h6rokhmId2AtM?;!rsg5L}<26y1AzG%wYokA!ja-v1iKmq^Gjg zF@9Kx;&4*oN}iJik$p#(L{ugg}aXM>dZZMs{zV`r7%$8Z>QUV`O)(oJZ75!J0B-zs!=x>zbFkvzYUzjE;^eHmHjO@A9T7XobW|4SRh?)W6IE&?&({^tDju1&xg@ zNQ)N5p=~EG5<93S?bcGHj`O*g#U2;dSKs%<4k>1rzzV*@5m$=gQY?4yYs)Zad?zm$ zedBPdsrOIu@gO5K3cguy(w{weAtGpPHKBMzPjC8ZN^_q4-yRh)em&4shS#6b<$?Qg zVYJ<;xH3oW9HRq0UmL+?1kWN(a!eNs*tuZuNGjlaiROtJ0{IjV-Q73O75mB1Isj_I zK9d-~d@v@66tAe;P>N>`Za!AH60y@6>o@W0@w_4jvp|-prZ$<^a~}+E(!02zwPHY*04_5@IY_2U#PVpFTbXE5$PAKRverMiJV4O=*1V zFElM`G>-aGCa&H4*2Db`(NaIyK$H-RXvY=Yi#0Vqo)<6O;%$%qPj3TZkPYyPRxbis zWPlo-7?n+H6My+#p()t`h!&bO3?%%JOBWnc)oGt8PC#JvEMREzITZlvvc8Q}%X9{U zF&(OXCxu0@b#b9TH}7A--^f4E!ug3aH5A5fg8^`_fJ*@l214_e8Y#E@s8&2cyqiV! zEJH!<@d85l!=2;c7@D%`lCg3?BA`$t%I>vB>Yp>vEE^!%j%?*9Oayu{hMyS{QU5#( zKqhV%hi7H>-ALKlS|+tU&CcfrR904kFOcGX#fD+?(>8Q_kg($Ue<+|9J4q;H1eDk= z;a)+Vdg}Y;w@O-et|dU9r;K+F>Oj8_ndLEW&xu5fdDEkTq~`B@_&#Iy6BYhAG_X>L z?h{uXg{_ioZlhbI{OQC1LMyyCQ|r>}{Qo_J!~x(B-2@2xrr*zrHF< z;GO0h9o}4xY&5gV+FC-J>w_p_B8HRhb!r}gAKfvT^&->^%Nb{dN50T5S7*okj#HJD zamA$zED4!1Lis%%CnMbb#lQ3UydD?40ff;K<|Kf9%<3<8b4N{=S83tZ7cxvj@Al3& zIaABYs5T*kIacZEw8RVOCl$v*FO&4>Rr1ivuCMx1b=x5)G5h|kH?uzUUlK!bmplOL)%R|#EhoaZZ`6(iA3$go$ zbwkybw@3$)X_5rVgK3bdso2Et1MlHFv}s?wL5~Sle@7K7qc6d=css8CaM!e-9JQFR zr*Ma|Ximsx^<@FtA>?6NuNZB5XE)vHbvJGBvTy!=kQGEEcaT#DxJ31J9E>B|E7gox zYDyONjeEXy)%Wf(=Y$i2#zOLFA-Wv$KCjQjc(40Jl95a@heCTYnC`vO z@X`OH$E9xd#}Q+zAF{EO3PQ z3J`A(LyAlkcfqo}1p28J9U&uxz&B_kAC`%RCpFM1=$D~pbL*HczLkSlAepRK3Ok@N z?p7fLto8yfuekbbt<0 z*w|Qxq#Qu+w`rfDCbuZ6DCs)V9+VV835T+(8noHw8|7di?y0_O^UA{IDVs>hZ+;@~-O0JB{z*o% zt%qVJr%K{{IrKrIpgL3Z<)$ycDsQ{p97gy(sl1(x3HAs~ z<{9R}XBFkeQ_D9eay?cy&1gT1qJ6&C2gFxqGV&)MO{V<`Ue^8VA79=cn_cT_CUa6^ z*tE8#(febdXSV3;L(2{ecVAaDhU{&jGteg{cKb1FLUNwwax>h?uI`gB4P!c4?NJ7k zujVkmqsTj!r(k_R0)p&~QEU&%hTCO7JqBw!hHy zVWaCnfl!mGtds18!iNQp=6#xSZlBe=uw~r)kj@1H(|Q%V6#Di@-5&h?LpvoUAC_Zn z59dW?|2{)=)#As9`iFz+a-W$NdViI^nM+@xIJ{!_VJUsAljT1^PvH=DE&ogHoAqSo9cxzVZazis zVSGsGXx;tRyDJ2RAVs|0`A80=|LWzrn&!t6eT`%_IZeG(j%lvltNRgNFd3MWo7Z{|T>Y{dKBnA)x zQBk@ZRJs{J329I|rMp35C=oLObtNj;kZ1o;!Mr^{o zx^8V%I_Wnv&es9&G*nS(GtTetkb}N*L;hqhgm>yMOEKDOS?!!b^3o>|z%i`DDa zeI0o~Z4XarTKg)bPjG3MwUp)Dg|FTN%qqk74x#e?vuJT}(W{HEHP6}Z=&H89H{?^e}^>x%Dq zr%r!Do9)A&&CH_nUol4>LCv3sLyuEr+oF8u4)LcCPcbd6Z2lD3J=3k0{1QC8(%21m zEgHPC8*(kG=w)d*xiQdJE^SL8MYXDow{YDL9Xwz5Mrj=Mdx`lfX={&*e>`+2Y~!hK zxSDY{fVB~x#hqQx4LuWDU%%0h4_L*x#>5V6o`@`tcSqgmA@$&Kf>&itZ6YF9E$w&! z)woVR7z6YhU=z!*c%qUH`AIO##eLq{P-SINP%g3_qBz#LTlEdeNf%5H} zlgNP3XSYV}lqVSnw++bso0^aCKBeosx7L^&KE|bUH!CgIpyRq2-=c%-!6y53C|Rq+ zW3`mRXYmHf zIU-u`j^Q=01Ha?8RVk;pZdgv=?CZZ2HLq_?yxGmj=sj%cz<7KHjK)i#=&qwk;G^)C z6h*-|uRjUjKx!v-2kUKvbgx(C&Jr)}f$3HzFs_s}VDg*bP9gGb3NeDx;MMxoy3-=! z!cyUd;_}K}oP7%o@EMW_IRW}2G&YR=c3>k-m~Ek>q#Nyv%pBf3gkldd}R9Vxy!|cgwEsU+pl4XViI=VYf>U&Vc2`~ ztQT!#Zl9sZkrb{Mxj3`EX>?1lLRte!D^9;WQm{9YJyTgY9~OM(e!D)Xo1< z+NuObR6D_?UzZ3v!DGse03XrPko4)8kL_h_Er*&)$@MiH+;#PYZ;tvxW^9PRtp5Fz zDEGH#zs_CG&dLsJQBIoi+ub4Zk9#k}kDq1r)Q+4t`_nVK7o8DA+DnyRqyD~|9GErR z5m**-zn?mJke4ksXWMp(MLdhad5ZNDluTIsFjq;Ja*q}3ccyKH-pEAkM74;GFWxe4 z8(|_TD1HiOKj_u%q=eSph!axBA6NG2sdKm3!E0a)uq##IC7;1fUi0Z$!rBCu{sf3) zN?fyh6GZ&;4y`1?gBe$q&Q$WdFmw+#lfTPI>Jz}>;_r}J`$i%D)-Yu5c#6HY<9OL# zG!`FqA=xVsORHBC{NQ{ta&gSZuuk*}HQR)LSzpMtjEdymX>p&eWG8MIdzAOm*!a;4 zm;@s>T~3>&P3>A6YHhH!2?eb}4Vo<~XjS?XQ2CeVJorf$_{p=PU|)~JDa!Or-+bYQ zjO!Df+@wtM%XzO&cVq=mqw_(-ruVQ+y7`d6aEs{@UcDj|7Nv~Wm;g1JhBV}RX8M1! zvMVH;<+%*nH@`$XdTuU^`fFut4o}Ep>-7=w}Db2SHOAH=i<@+ zHRI;;iyZk!;BFS=$T;ex$aG79lw#UYREC6ui4=B_PUUN0@w#ZXudIyhQZIWEVbQ2q zHF<$|XIp$mF=OW|u!?-I%&RXjB-t(wx*;vh8J=b2ayA$EHOW|5XwSCqxnCNy(_%dT zxM6d{Ywbp^?Ufk+@xxJ6^;viZgXmP9&tm4PYjU=oppq_1Z3Ok`R7?OlES~SQ-EjT) z?H7^S9&8K2+g=L#lREX7a2vs6e_MFm_6!GJtV>C3gy0n)>a+u&67||dTnl;US+yZo z74djJk1hR^+r&W;pZN4PJe{LSf7(=33mz3$U@jkLnXeQhz`F6M7<4w zf=t2uaIFy|YVjlYC5E-zL{CN9m&MGR$P3>IuiEX}ll3&SogCjbpNS~f`BODVU*$a5 zT99>$z=#oQt4DRc@O{;}biTFKQmGm@>KKK3SJ_v&HiLqeXcf5@nwdT~?@Pm9WIo{_ zZ?1)*%si{{3tgV7Nvo9JBp@(b(BNkwf_O(~2F(;M%6Vtw@b|von7N=avc>V9soxqi^G*U5xM1=|T`l~86qt_xpxS1NCN@`y%!>#_X1_Q?tb(?^qre?&-lo-FdRgLlFb9_QJbN&m1+(J$Mv%8cCrVRj}O|7NYyqD53O? zt>~CdB?2@4*>12i37FplhL7cnV<4px)Zm_3Ys%?!ch-29;R+?I-yW{KD;+~sA!kQ2 zn32fLGVC-4VV9GGlQIK5Zy&ZcuK*m#KJrwt$_si`@d{u5VAS|bOsuvC{28i0nmNEhYuZ>UU-f!taDw&N zOd8Xu=i|Bk)T_7z_4n`1zS}_UiY$U4^V}+5TkT%m{`u-OO8kn%DL(OZv+Q>F+4T7> zua@`3ig@-je=Nk7F;*Vxqz9U~bp@mxTB(2idz6xk^e64tdmdhRO*URkhd?l0-3 z7_cSIBJO^+a=0-M;?q(dER)+mLQ!Nq2PDYi){S&e9Pqqb<&J7E;7w~SPVrIBw$=s~ zTm5B!3R?8Of>&cuY<6}FdS=l+7PK92+un+2P#tX(WL?S_zBczjCX{7xGQrM_#Tsv= z&0eZW@&Bp|mdG9jK2Nh~a8Q^HY0C-@H?ERp0ost4(Xg zYiLdzmP>=*f|~snZ~UUj@4MC!`I_tB{;i6xooo)U**EBR#w8acZdma=>D@zjGDgDe z8)WEN@`g(D95M48ViW1OGd!j(W;$bCJCgassf9(S6e5j1Iz;a*c5Kt4KQ#Ut--T_0 z>daB=T32S1d6yhRTpU>laoRD2iu!aqMP5!H^R)bYqbX*oPQoY<^dsjp5iO)RGeUoVy%uB&(fJ9l0Jipwo z^+k=8C7)~8)%lnry)kUArL2&W2nE0Fr01?TXT-kj7toznzwFMi#;xH5yW}hT87

;xGXpWd!QgiFR#@*zO}`8)T}J5 zMDb&~S~~3P)y2G9#@4sr4k_nR3;~wKc4Yh8{!k_FO>8cfb9mWpjkooD-9RHJ-ktP& zVo`?`Y5eaS>GfUTOYLEAS^a&*kfBN`XI}o|M+b+!Ygb_he(&wN^csKAu@t{MN ziwLH%phcV4$c-|u9N#j26@Rt)o)7T2t$>quh5KmtOOQlxfU)tqpxiE5 zYr9+HR@28}8Gk^e%LV5fhasHLhvarp*gbE{g#;ORkektL@+6SXoIWKAj5o(!;P2y< zmr=Og;_Py&$fxI(XK%J?z`K}g>~=gHh|bLfQ##{{R!p}N{29RL;AG?(7l=S^B+wyJ;V`k4uoC>z1(D&F98_W3;L2D8hvc`BdlOQ#`u`6!OL_ zN6x;r6jAN+p^viss(+MkuhACjR^$!R#%l?vp)9vbgfEWj zI(?kk{Exg|(Y<8Yz!~?{*g)u=q3nTujy*R|k(S0u2$h2L+f%9f(+ARjD1ee{;Q`_Nw@Iuftf7lWfAl-WRP1e7R7BBga!9mmcM%e;4ujE4!k?w7s zuB&IqlMn5C)li4A76IWuXIt@ZD+ZAqT*u@5KeICuVtt;t(+K`t= zKTggh{rUF2I!-HQZus;LF6RXndDK+et`h98-SKZeie--KJhc$GZkj=fca&pc?UG$D z>$a;!c>fZ!_MbJmy+s_vd|cEvtu$Zoy+LlEG}A!Ev~Hf(xJX~e>X+<-RwcBhO31SO zx6-In%nL!9BC(U}#*rKH{}-iC-9K#qQH}ZiQH19Zh97!*@zd-n-rQfEgLY1aPsf-Eu1+u1pohtL%^?>ZVb@Ul=1S4Ersdc%wK_`d5# z&SLcQWoT1cd}%y~dGzj9r47E3rSD*2VU62^EArV!LJBoUsi;mvbPUice2v<3FBf?SwkE{CM}X&vthE}z96*;rDWH=uB@7l$mpx# zQN9Y2*lmmH!Y1|J36x8X@Kp=r`d8FRfWX^_f(rJ?0WpzPu`cgHwm$CPv5JJkhYM&i zmHLaZ^Vcw)!5KM)`k#Ek6AEy{Qe<$=mwmkYQ#Cn-_PlH_f7qilh6;__HjTs#G33i; zWb5o;%Ws#p_xU_Kd3;H&+hTSkGL1ifr<yl*Xv^5T6c(R6eqT#%c*os z$fJ4g#OA&I*#RC3lrM$YQ8mx^Qs!lz&>T`ey)Q+-59Gw4z3BbdEfEe|h?nS>=u^ts+ccq=tB zoqW5aAx6>dFrxSRD#LN;(bZ{7M!XmlrF6W8@*V^4U@-{>lu>43E)@IFE$E<5)IIKk zhx=#M1+U@z9)o0gZT6)%i}18lG>GBL-*@{j`wv2Aeaw({mvuoi zjdX4+z6DRLTBGJj;~A)kr0B^DdzI8+z$X!Xwj4fD$E1tY$HFIv@Qb{Z*q_JGffehmx53DSm-A;rb8LQDYv^Mb@#|MOtEwM?SOylL$)t(zv$PiR;apaIE}OkuOVa4-gb!5Iz3Pl=G1*D_)KPT7jgYs?^aYbvg-J&zc$xk>h|*cF z(^Ba{3oWIakWN+-7sM`xZ#1FEhP9;utOJV|ubGFRt&lCcfq0nbqW)x2NfDo9pA6Oe zka9@tqI-;Q>&_B{S*y}mQjFy;-#tBEto0-@_7%I>N1i{|x=cQI zH37-D6817>Vv#cj#`7-mAq!i&aD(lGAj2ExsYMavE9g(P^0SM`N#p88ywwu-*P_== zYdiGt-e+>9Wc+j2e#c<(=?7T&1P%!hbNgW7)^&HM<Pr`Czlhl8IS{cwz8%;uBmNRb?D3UqN!9;h_znH<7^>W)F9#jVV;f^^SlC-4Co^6%#dMTf zwMDLuQX!{aH@UTYiCZoGQv?nHewZp4T(|>^N0S;v!dr{&>~H8*p5!xN?hCFPE6A47 z8rXoTUc31p`99)5w|2w?f!``ah`eJ>71KuYk%Yw@3EyeZ6uy>Cm+wYD7W*MezmPm+ zMNe5^SySm!N$vLl1g0XOMhmC}i#xwWr(w>_COF%3<>YA+j!szD2BXtwPUDG8YImP@ zM#7IVTt5XqX6a*!&t^wAC0yCy1XBSE#6J*!`RXCME>or&F;8f!V%I0hb~{k=6T$6G5D1?*fa>=W! zv{5%!W0L+NlFU+cZLv!OodEIx4gC)bMESY8K8`fiF(}!!;%B!0`zo13LJw%^i!ewG zk7|6J@NM{yF9Kq7-4+n_AL>0%|2jyji2~2(*Y5WKPz?+~HG&Llp`gi>@+=1z(&GFw>7AKKto@BQ_y73-WJE~Cgp14imFBeo$P<;H zJejTf7q}pB`$4H-pe4`zH=-kQUV?wM zzhR|1xo%dCR>1^=YM+j{Tqf}<9ut03w^A?~-);w9{rwvZ)Bz+q_a`|aa`c{4xtaklXN_z-QKo9yp&R^ z-D))X!Z^Y`aUbk5XYv{E?;~nm1#ck&EyEdi!WE@#%3r(x9delf5JxbJDa~takf7@* z5hCh#JV3TyFyXDC3tAe^uatS%GFENQK=L2WSvPs^I5E=y#JKzf_`d|Ubs7RA|5e+X z=nQqLiZ{2K(uwK)7|fa5heWGS$ZIxK4m6$<5p`H-{eTV_{o8MqXlZ`-a*iJtZ5Er% zx}+kYQet)@6o-0=cqFQ_7?`nibqE&_<<zN1r-$XpW}WLJ4lY^YpLb7fv8&nZ0vA zPbpKpm9A3S*{HfJPa^3S}#bNX(8rpwJ+gpbE0D~;zPdbk6N_!S{yGL(&kv3Spxz0k>+lowSPzi!9yxgPaZ zTjzAZ_aO~EAGA=b8BZwZY1zUjk-aX?svBMH8r{{q+sD6`-E$Y=l;gpa0^1G8B?y#a zqo?4*P{;ArG%xp_)-PV`m7#F{!iD9+V$1^Iwmh)LQ6-IQ9JiN}2@~GzJEqSND#aCg zE?(=Un^6j~0wH07Z{&pq@tuKhlek(`?V5L=7o>_Bs79+|e%GuuY*^E7dw2;k_6x*9 z16?p3D<*2aX(^1OGtDq*JPdugFYDWlTqs}vFR zX3??_3psg&;uWNS^Dcfsw3`C&=ko}{ z7k3kDi92d4=mZv9zxdP`k5x~8L)vQrk1EE>e5sVXH%XB8M>AHR?S}MRwp=`Ee&%)j z^=`=mZ8xdax|;RDi+)S8P`_-9Mz5y9#&TP)+P>OC_AErl_v7#DVSQYKSWnfy=Lt4K zKP)VO&`r0y6KrkWmSpBq(tH$5+t&rku#?ZFxURHxESAc;?kZ(>^)P6 zJJ{IUQ?TzFkG52eHrUxP8{TM`0f!O?9GvoB^?Wf}e z_K0%aU;zgzWL9L0JE=N#sEcCI65!mgulJho={p7Omx5o9(st_Bm0{vZ>0HiZJB0hV zb?&wO8}p>=jdFJ2i(Dp3e_6s`1wt%3=XShmL`Bw*-%EH045agFlp46kd>Bf7RUlgs zqxy=2WFFXWbRJc`fB_0m!&8kH=)z$CiEa1HV$bePhNwBc0OIecMR;oE-V^>SY1rWN?Y#1t%+23fIc@YiZuZ z_d=?EnFCStQn`fu)S0rP05eEmWI;#=o>GTM_i{IRnpHZ(W3L>|H`IHIKE9F#&sGes+qKPXS|DRBaQyZaX&q4JE%1aj~4gz^t@*XIXgK1R*!OWbej5W!#k&$RM_Bk zc|q4`8lk(1Kzb4z5TbjdHygo$5D0vGi_veIS%8`;oQ41#>6h1`BqxK<66gKdil9+d zj&Xq@}p#y^Y8AHu>$VFR0N!8?zc)$I3MFkvGKZFc?yB6^;GHv`YYtn-^%8{0q}~$tC`_%Y-*os$uN2%T zbGV$Rk?%Z{1xw+%JTAi*T2N*!U@kC&P65X^QS_ikbfm)4#$5scJt0D0b3td2%Ph$s z)J@K`sr2+iVA775gvXj{_gc)=e_R`k-*mszA`#r7XQYQ$DA4^5Hf|bgo)@>i>yyh%8FiKBjd)KBj9?3>#9ne_0mU!_ zY8kTa954VVU9(LcU z*_iV&>%$DEPU9!u{8=evYO*aJRld-~g>eBMfsZWa0|}<)9DIn5Co4o`u;(n_Avpj# zrC}dBBLGK1q?MXtO=5$7aeo)5It6e__)5rGz2Hj)AZp1E&$)UB(|!SzKJdI#+f3SBc6Y23l!YI}=DyH7|+SxLC|?!6gu&)hTR6Xu7uPBPTFc#_ zj;@>Wi@%>O26wNtx>Kz!g1GRzrCaZxQ<~|IP4g<^Neq|YB102ko9QoEaQYkzAW-M| z_Um$JaM1wxb62cZ<$0F1{A6d?2lx5lk=%KiDjNXPnVme&@&rF(9GRXqa%1eye{{ip zLs0$|@SYiIV({m#*zWWjPe1hCJ8T`Lq8xgD;}S+}(LcyS2e+#U9xSZM;E8eaxV0RU zD$(5-jRg+p{Nb3+%eUryKW1nRxzGYQQ(VCl;#4pEBO(JTp9WMK*7KV^LHBHF$x^f7 zikbI_tON`)NsW$P^rM@0wyAl1n#if;EL14~f7Z1(M?(f4yeCjUi~h+Bs;LudV!n}h zW&|(34IE%f*=|f}R}vI_2BCrgIRC+yU@-r?%%VCqmn6j}Fygc^Qxy%R7>`k_niK#+ zyA_tEz(y5UkwT4vCO54mSr_=T4fahg7cO#IqfZ4cs&NOZ zJEMi8W#1hXeTkRW26h-*(Z%;C0K1&I%4RW+o?$h`DEJ_esB?|y3(wHd4| zlBbGKdQP4_TZ+&mS^EsgtlOO$@0%1FUf+|I#9_*C=s_&6bu_+$AE83Zg0QFA0vp#? zN4wv`vpyDC*rMEq48NsMn+5U7izqZH-A*qFt3>7DU*{V))BvHWN~oS1j;NyB00w6U zi}|n?I1r~z}R$0Lb$1~(Hl#)L+AJfudnBs*(bMa^jh>b{*EdZ zI5Q!%wD=9~Dpy|$OsdmoZ@&hX~t*^R{%i2WW zOmmx1qW-lz1T0_L(eJIN$>Tj}C;SYbtHMk`Dn$3KhUhe{*tv zgDydvDXx{uD9Zg+1qVd^++wG4fm^CqAzM0&wU;SwH3_1Cq^eK`5@zq}Qt9-!S|6U0uJDg7b@W#gQl zU&siHPybRR1JZXv)#oK@#1FeRpK<))yJ%wwElH8hr_aF>eRkAMclnL~D)H6S_zV+( zt9S|!t|s!{G+cQDvHe2^#!_9CTuy&=85pVj{G0*#r@}PGpZcSr>8a&iyajlY2vB2Q zv2b_#_7d$j4^NXWFU1w4mVB1RTuFd0Ize<>o9>c7r1QB@1euskCB0o0pq(w9z$ih} zq0{#bSCz9S`(Yl@(TSfzJyo%RT)qv+i)tw4} zsv0|gG08Dajwe)C0++K>@2V4*WiS8*z^zmGwi089MWd2tIznH-E{ z(Z1#vd)9ro`&>?}CMTo#B{ilxYhh#bck_hELTu5M)s^+r26L&?M%~>&h`aqo$eEE> z@tM&iv{SH2Fqf%KbEsIjzZgG7^(pk?WVOcs0f-ToAU4hB4+Xjn`^W*3i77Os##N+1 z*9C{$mG`}qcO{+YlWZbGnIhwo44B6|+;spfX=R(?(lZb6LCfq*i86*6J+!VK@=*Wo ze(r(}f;ZbGuNXl2TrBY^NyHV64%7hE5d=^jT-PDmG_R?a zkQO;%+o2EXgv+BI$cifNhkb9JuB9Obs_o-=Fq}@ zH(g^MavyQ+pQ}6MLpWZaggNUBTd@aNN_fnc$=-#Xbjl7MWvi;Wf4I-*2JsCN0JO@G z=E*<%BHR?Q1#gx7$e^<_*`kvJjPK!8oXO2or&jSj;*LL|Z!Ki({9bR)uOQa~Ci7Stej>b0g=Eck#(RYh9O@#nV89Ft9#?YSx|D(;qI5M+MuC>I?R42Ev zq?ZYuaaa_TYqQ-_*x0nmp8J1hWrC@k)(-J{><@~Cnw@)X6=2DaVYe(frk3h4}t}V+=FWH4^KW?mPybsvy{55$f9cf>tXb=_>9{0lfZA)Dpe2%E! zU@RJAvnM=Z{tr_VCEZfd5M;`|zAr4uwnD0NCuN6Dghy{3$c5>DuH$vvIcZ zq!+F^1Dhfh`yU>1;<3c1HRw)qwEgwh{#oW^EsX#KtB2+G-q*LSSJC5WIst%0SJN0j z+nDHqZ|XqeU?7w9@1zD#$+n|$(fGbZoLCnt_!+42J?JX zL2+;h(DqaX2+6}^F?7a&(vX(ROCq(H@;*VEI;voJLg=r}ReG=#2z?ZX4NG zJ8r|BZT6Fl{lwg0TwB*|nWUk!kKl!E3TEyN{sE~br8@kxF83i|FswrjZ_Sia`q|O4 z_TGtm5jp@gb<#t@5EdE&rq{2bz!;QE|KXz-{=H8uYNNfktYHdC#O~w^%!{Z4Uz~m{ zDvC(yym*B;b2xNJ{wMCoHjhrA6qUh?7%nsLYQM$%INcE!lax#^c6U7AA@Xb#+5&9> zo^Db^XW+CGNKJeDhw3JS*sJsCIbx5;w(0moEFvA=01&4LfH<06{1pDO&krMGF%(sj z*H|1q(BHyiav9AYg{?LE=M=PzqMOj;zW+0pYPTK#r|-{5!aQpLt5B{OYTxsF8OxfX<9S=*DFCO!urt-*+oVOwn`=?1>+ts-5R+Bxj((Xs z+?OquOn|`5ctPPBsg0&oES=c;qGsN{DablH@}7p4>8sY8|E0W!QE2_RPoxc%G-}Z^=bW*>m+m)_tn9?KR)cZ04WE`;RcM2LXWj^dqkCoi~wr&N5EcVCG zN1{0MI0Wk-N`5%w6Z6)i@IAUPW3m1ukyWha7Poc%T4AhWbIESnOhLiM(mpD+SU1Uj zzUQhjj<4sm@?~jbxWqRmW=u5ntb6^y5lUkaGN4+(>GQo-aB|AgpD@tENB$CRig{2Cp-thE)Xo(K)i(CswQK%Q;c}g8FVsp zi?oV$WTPJYRtkLr#vRsN&tIP`M_gc)mZBvwK#$ID=zeQc=DmE1-Gll0^L=F~j0vL5 zOM|CGWoQ7TKZqSfwLM0G);M@eA$wKWF*^PS1&?jsPeyd_7SSV3Ul1Q&k4yD{HFCPy zI{Qkj5S{p~H8`C*3TIze4xtf_&jrT~=~?B|mg# z{KPnC;o5mouPyp-Jf>y&9WSqr?^(rJP3cknmG;LLjs1dFoz{~s-yg-NyS)W$8t;H2 z)c;P;e;8tz&O-r*)g_8f!;(q)*o1NO?P;>^fIa}jlQZ-T?;dM#4Yp5>P7HDXU3QoP zYW`qvI*9ULIW#mZoGlrEK1g_Itr$1Y03;xxT<_;6i+|MRySbN)Qx=OA%w2YV?&au@ zPc0C5LPY93cJgyU1!+{{0`zYKT{b^CJUfrMEsXW1zX=Nim=pX0FeH62m|vNSnWNXl z;PkqAZnfJVxf##me&H@>Zl7;L&qZ%rEjoWXz~FHMw~zd+sbU(Xi;UEey9>J|jKuCD zDqFm09N#O7DHB1~C_j~c`L8%deP7ffr0b$0koA8Hzzl`Or;Oe7(#K=ExIl71P7e!r zxqL{MEn0t_Q5v7s@QyK}s#3jnqOLc>RcNhcv4t@bdh_P$)?V~gg=0l_dnrdzo)6R>29VWZHHiNO}buPUjFT zQO@V5n0Nf}p7=HSczTO;2~dUfKLjlO>u`KDAvgu1dK)tOCQUGY5TZ4e%6vHvO7?gY zlOmNt!rWud`l^6iTno0GCoVqy`vSi&FP~3#aEL@!M)t=_-zS01m`xce@xPhTUQM2+ z$PPrVr*Ta+ne`k0#bR9Z-ao9*9}7m<|EqfLGPjdqd!6IfNG!#I04$R2dgN_x-_0Dk zDvSPy*)X7Tt=K`I#;R{}Nv^KCx!Ru>K~il1UDi;oY3)NdwrsFGUZ7lv@v?7)hZSGp z+Z5PF-H3W*G<#q4Z%8e`h7!#t!;wc#zOmh?JI7H{!K#0??C{C{n{2EEaw`ggc zd9;MRd}|3jc(X)bem^Q^ZDbzQ7gr0yt=L2Fj} zxn=|mWEfRI#82@`sBO@Oz~kPiZtQT>x~`tqdM0{S@S#E;1kzOPM>o}QHZ7_`vk zrpa8dE^dlRwK32!%8aJc2s9!smh32K?_JRp;Clyx(j?*tz*1B^)E^^iCeWRkx~T}9 z?qj_TeX8tfYN9&vF>ZZzJ3Zg4sAbTvpMw*m!q%T6mwCxQ!btgbvc3bQK`R7u(*1ghrpdXEfzH{}PT zfabext3APO8lWGTPecl}v$1I2ihU`4VNK!#s(kR2;vSLkJ_QhoQNA)gdH`Tam8AO=eLdWu)kvzJMMFKg;5*5WZIX)6HqWDWl4_t) zk3Q1-jIaT9eq%aKoFMB4Oy>WNg!>#N=WOKlVYur%6v`cu-51pT@PJ@tGlKmA&Ho6T z!;o5^p&|q{UtjW@gFt&Cwqx)`MlIQCBBrlu(nR%V`E`AvVm>OoqBWyAweVHg3Cwg) ztT?XAnixM-bT#$m@wS&3yYKZ~##%4xDrfp2;;0eFY8J)KCp@ zqbY8WCL|2G(GbJat~=RJa?}o@JRfo!+1@lZJFN&z<|7zX-${|MNjM zbdW53Qdi86;)cZ_!LJoHI?{bp1m98tl6Y9rZ0FNjXTy_!Qk(^_;oL$;_3#8}J7jxI zN_k^PwAOeP;H#xn^T>;&(*7++;E)5@H*Hh+ds+txNdBvIO|>8}D2nhfM2VZJ|5FYd z=aMc4&{F}SVgcStR#u8-jt+(@5iAGjC272AnxA<)W6I?}x&Qpgvt^Ed#tbRp`fuG- zcAzY^dzXE#!tvv16xplYq@AT3up#{(0*{dmlQC0O&*A9(@23O~Ex-{FO^KL|S`Z@W z=T9PLf=MZ}va`Fi_XOQcG2>t3jQ1);|Ml3A0X+G};p3cZu*DyYG}XUB5IS^ja?xy) zs2>|7{{%b=gxco{Eebk%%m54LtWm`}E0w)_pWjtQCpB8MrT>)r_dCVsKrC`PP;p0R zL(|jvo6Mpc+JN`Nmrm zFf*d0%6~m7SwOj!jio6~s!5g#mi#`?K31nO>CaZ4PmQq-H$Vi1EdL#K+a=ox1iTE@ zk_Dh_((aUT(o1ixoS@k@s&NkkATKaZdGgeVjky^`|EI4694HU0Y7+Z25NJc%A~i|- z^GoY!iU6DsylZ%r7Vm!oApn;EfX70F84svG_q-o^4(%8Sg79P9(wZ(eW19YRY*Yl{ z_ciA!MD?^QNH8}niWB%8vxH=Pb1$YgU43kMwov2OqkmeXb5He*#Z=Q4VpNtyRRYj( z`|Mn*=3zK~1|)O=ci8*^lk~M+3 z-FUw-PSS2qLV+XyYvX}XRf8OVvPX;&Knyw2qgU1F-f*a#hC7REFPnVuw{Fx0QXuJB zg=~>G6tE(p$-Ng9?jkFMWoO;_9p6;yyZ#h7r2L zh=UQ-X?d69{YpJW2Q6$x!)+4k?0O<|M!WZc4v>pHa2(^66+`r0)*mU+jG`IC#jLPy zK;`c3W;_0wDX_%#8BmOu92RB**sLG3liu2j8D1MBl9v?sD?IRb1W-|UXm|`kd4``^ zm)tkGA3+hG)J#LAHQXXwQc>xRdm+T#Fm zHnl(s@bA?dgeLW`On`>g<`?mU+^Y24w_gKYg65wEW^wf|)E1T;10C0Wu`2~mHy_>K zJAHDJk!ME#@B9$pzd<~o!~8mcdWHNV9O|;gX=aXV z4L9Qw2Q4X0KX&}yM>!-tpf%+{C?^xmgFFhZvM$O?4t~Y_sm|K^n|Ve}xV4Lj-EXyH z?UgI$KACniwQ|i2Y-ZGhC?|rO; zZu0Zym!@1mQ;ry|EcQO-GZ$xHpkoHe{zLAw|6QJ)uy$tj${1&$zN8*e2Y(Wq_9rAi z;6D7n6nOTVe78MY-^V;n7`a2aKG(I`@5*ZbBQfy>R&4;wKhMLj2SBQA5g8=m3mO7a zB393~%EsE&QXejnoM~@TO#g@lx9zhG9zUVhEy-FQO zX=wMk90%Vk6;?&-sw5bNK>4;BHCJ$y^B)*T3i-ldu>xLI{;cGrDge3T ze{iI~Q0nOm`ylSB(7oQ;y&tvEs(-apvKYJ494(`ljQEFDsQ~a4Pzeb9322CbN&Xeh zbTC(ApIgm19i<`}{j7Wady3C1#L+QvtOl)D*3nVXMnXC)|3a}}yE0nktH<|!fx2|c zu_`n`L)O>?d_7BjG*$-8|Azd|Yj!aTb-kx#`ZpHms!u^Z=pCq1rujqte|82m-SkQL z$q<64K0;!EVZsf@B_>AI)|Ar~`l3g&K#rC@wk&qBk+B|SP7#ee+$>|)wo~=xK(RN>$BYywEmw~J^=j$zzl8;(>gU!=TG}nUuRc0X!Xu@mn{>~`(J;~0-vVc zflb0qM!iPF1iWXn3{^RhEI2z$n@RyI|MazrgzNY7dQw-X>343usk^4WOk>|b@vb9OhrJOw@!B0)7JL5(>n$VO=bS&8`lDawHo*{Z1@x|n3ns74@$BB)82Q-Q~Cb?b5P`62vK&($g1plj5u}~ z>EIaI(IMIUl!l$1T{6qav62}wvMDor?`)aB>u^q=&-?rL@B81kKRkMHyU%@J*K0ms z&*$Y}wR_+vmvQxSK5^5va{13ElyYc4FwR;<)vv_;s&Cl})#AnJ(hE!LjHurWmte?Y z@aBO?qs*jz*CtJ-KV;4AO)>eyF0oS<)0!ShSy;3BOKmo}g{m#OdW(1YlhE_<^Qdsb zrPmjZb3^Q#yu%I+XY_cY_(y)runBQFr!AnFP7DfPkN8tYEA=ppm7EpWY6qPoeS8jH?6mcE z_78wXF(qW_IRUMi-sE%y@jt9@wQ0kO8L9I-Bw7oX0$uO{xkQfGRs7&(vd^8P&F?+W zK6`q2P91Uhu-N+KTq@PiR}~j z-v!OG%M$g1KlBaZW}6Epus@Wx&bawv{wCGp9enlm@XIp^iq*e+pD1#0-0Jg^((;6@ z6!(U)b6iXCXDo|zzwq<@;OV`BjRDB9HuU%!V0m)3Bwl?P|A7JS%FSQ|;ybC;%(wT> zO6Gv3kwH-gFGJ@({CJQlBqv<#7Vov(Tkh}2Q*o#|e{!R3z^E&hlZ}S`{D7k^`?bew>&qS9N;iAZ2j#ngC*}#&!9el1MS5U_ zXGo(g)T4(RT~$0a4esAsK3~N>IsgX^1xEwBild=p)5ih8Dy0*y2&J#HQ@Zh5fta~NN`#}o1W@^0=hZ#b8yDk_0Ft9F zZfH5hD=n1BcJ;$tmW8P3!h1^DQ1gVt->TISvm>6uXc~sPpWju!*Pn|^+k?&BdNKnG zt=ISU$d!>|I-HMv_ulgA$Bqg^WaX~5S(ebt(-L#c3xDx=lr=a?teVLNA2NCI%>#8) zFS8WtH*gZhh>3^>-7_3S{GI*1oXv5~5mM8W3D?g0y6lE+2KE`Iv1PJt()7?XGKTxV zAyh5~rx^*|6H$>jsB_RPAUk!rK0s+Gs|vDVhSZS)?p_?1nFs59;IDx?ruvcB=jz!T zloe$aVJ%Lorn$wrNSkxZr=6ZjD@1LA>DFZqOfmtQG_Gcz$b|-i6>IX`(+fbPqz5FZ zna)sP_Wp7v4|@f(ZKSw|4~O4zhTc3~JIs|0U>GiZK~O`sh8F`u!B;N`F*vD4TAw{N zPS9+Kf?m4O`7*U4vlC>kU`-3;@So~xrvD^yv!6UO3QDPkv2g%W`X!_OW z=FaE5M}jBSAtq`&JOvO0rAL7HRdPkt&Fex{Bb|DM0$A}8rkGbS0tM*$j3EWS0m&Ry z1CR`f^?x3{x0QM-FaoXuEy7r%s%DP{p$5yHWT{~_(;sxorx@-@m7;JD+yXsfcpl&2 z+#HpnX3k50rjjKF?&sK8i=+V+l_yz`G#E1eu6IiV(u*bgtaB6A7fuq($-)t*gNB<^ zh>H~eEU8*c9AD#*ZTkDt;jg)G7fz!GfX_BS*iY=U`az5OL>zU$S^A~K)eTve2 zhr0$3Y$tit`PEt@!c+}r#Rukr&K+oi5K!>XdD&O%l~jL5T&VJOs9z`%DM>I3U6qoa zxWOv*cy^~wXD%Iw5;woT*{|-D4@eQtX_?e_pv#qZiLeJQ&JaqKXHZ(r2N|ydp zH+m#%wExYPwM6&X4Jpf(LIXuf%QOz7!+gEzGG9+*TIN6)_VrmnK}pJT?KaDp6Y6P^ z90)Wl<1c$8BKA%Z3_oz1C17CwMCe>X{U&W~isRaEs-^X@&O2$fFkigb$X^lyyC4TD zFWS;xyBLg~Dawd`mnsaw{And~k*$~VdP{YM*B#sKNBta3(rfQ>b_dTj^R&~ud0yH?wZ9+=xX0<~5;bvJxOoFh(KV(>YG(5*@Oq(FS)k9j*Db$(K$a2<)T_NFK@+0b zy%K$nw_2iss?#P`_*er36Yi^>XR?vv%sqg%W4BE`XT9(mk-u`K|{LWDx+M!=0wD{uwJEQ&8%2%$OoHpl`%FSPe1Ar>r%f=up;|Iul^A=26PJ zhl~JEttQV%zM+dO-!Cz>Eju1QT<@e|y0Tqn;$j(X4gyDm@=FaV?++y37k2t6qmt6< zE}TAT17A%%x*!1v1@E#5u)1c@+kOlW9of#Z<`0pCGxI&(?O2|;sIjz)T$TRAwtdjG5--F9^9iGogKT zaFJwoAqm@7sb-hTiWb6qLA^W?n7PI1aog%yAuao>-d~ewnn;gliQG4iMB1L%xZ61X zlrduWxbPQ&RAkN=?wcf4zZ(UpLP)B9?P}GLk0Z=|d=+;uN|GS$9w-cM_GJ37Nn9ya zo`2f?%FwF6(H5xOMK9Ri&q%&^dm1yI7k=pE{^AjkAiq-<$yu#wG*y976L zUtK%f3=Rr0fL_{ea6C;_|23jjD}j31Ral#pl=|$L>pfq@Mi4wm9;RuvDDCDXl)H}c zFhLhVn0#Y^&fGiRBmhfd7jzEvQb8+6nOi_x`Wp0Oan115MIdzjo4Vw#1B6*$Q$@a} z7!uED+!wv&;fWYhymG3@1ffU`(DDY%v!18}wV425&hL6ji*nDrjhL+%T|W*>qkc}tVcs@ zFjAtG6=VIEP#BS;_BtB}du$i_^#?6lY<^TI2l2V68e%O2F{A4AXF5x_qw4CrL)$S0 zh{Cw{9S4E+g|;aZW)^=Z`Ih_xvj-N&ha(hW7@NXfGzK2`Y3T$E)8R}!m{cIRb zU0EF#i3pA<%dxV#fO~5kMI1ktxE>bV`aOYFcgqu;iZvT?uj?P&I|1<|(0z%_1`>Z( zGyy#BwY>^-K5BC}Z(ipLF!qzrqo8>ev`bya3<1|LcSTxvPcBj^G6$V@O-7`st$vg91mGo8l}0jj{;^rH)3v3~Tjx=md5LX*yf$ z$haO7iF}VMY>$8abv#~veWpCWWnnt@6zV}wHusQP^%v;>AUTarpMWVmPkKYjJcdLq zP0>d(`dTduKSgUhIr~_m=0?|$jdZrt&DD~8vpkY|d}3x>Zb9Rc*cbiu4nT-@{aiXE z_?aZFylLa-mcVk`vhLw86jt4LdiZ|f~EC9wq@9(yE zQo*>;nI5H*8w-(W%K=@j8y1(YVr^hd3lx2^E<9~=%-s+m!q%B;D|w1$m>IDMuc{i# z>-*KqMkcl8$CFrmKPIKP_EFn=DeP~OGXpdA`wF;+Z{GN2G_EczMjVFdQeNl$-m$W? zolwu1&1K`*V^OMDcBbt#Gt?YnTnw|tr-Fi}=LHr(XSmkdCxO+P{{7ykr zgHZZI+)Jr|tvUe&iD^+7V)1?35J9IOtQXUuo_ghyyHorYiUVpGDxqxhH&fgio&ldF zvU&9pw8jzw5>iBD#C3zPM8<>Ftlw*MZbg!w%vM80#1#M3Df`SH}y}rhuB5 zqq5=CfYrk{WLp~QMc&aABb6}qZL6cjIrbN`#dB?PY_W^t1hG1(RwJ{SSV6_aXdE)2 zSFI<$FV5<$tb~|#Gs`{LNnwgf4{DDrq~USa8P4Xxg%Au>>=Y9P_ClU8W41jCG;m83 zRTlE*3K+TR`P7Xk=cWdu`%*R~C9u(r*#m@>)XaPvE(iq;D80=?ok!7#MpsyVsmMfM z=>`$gsl8hw$(IAR`pZ6GF*Bg}2i$~9mL`Ep%!5Rju!qT4&DH}os^inHea#qgXr%4x z>(-(i)S)CUlFngX45LOMR#0OG@@_$aA$Ym^UWGSBc^-nl=-)OUgZQXiiF-_pmW`r* zMhg^hej27rJ6kOv)px0w3=aR>;;(+-wU)##hZlIl?E<1OfP$+*0+JQ7n-U&%PkMTf z2q3G4&0rORE2ndc|LQ2ni3^}WQ_BQUKqj_i+fNq{0RS?OQ};_v?25M;o9k_I~z=xi90oa{Y6{kQ*89C`2)79eN|rH&u^0O5*5GWmi54ip_it_ zF7lPh0!4>to^KY=(*KXv5X_=Q!eSWgbE1!rHtNt$YwnKQPt!L=uop55d;pb2mI6I! zc*e;cCI`_gBgiMu<_;1eWiHcO<0kLbuu}jk#9q>vA~1?!~!}eh%S^_pXisM{`PwLqB&4Ytv46D!YuXg6}fON<6QY~ zos8KoaeZOdQaNF!;kh}rV9yV(Ew}Qexol$)?9|$p&(FNn!Wr&EjDKZ(wBdwoQ{BH) z1QYidvalaUdfs!0(ibWfYL?fOVkWiXjhVK}rJc4;#(ow)A!bA`G2joPH@Ph8hreK4 zoP?@ypB+_x2i@1}ng?2kBR-M+tU~PS$S+I10W>fYTttip%YS!ZS?~(JC&?T*8pgPE zDR0OoKal80!|Vo;m+Qp%B>$07sK}4_dd7gY01l}RZ0`ZA5HRY{CQ+gQDy-#xO`meL zaDBE3aKXD*7s(cj8?<*R7mt{$|McXaP5NTmGPJ&(Au>JFVjp?1^i;NE?fl=3`pb_5 zvT_i=U$o@%hFx8Zu5f`i!3AuITkdUG&=rT{S+8*medFG6|Hz zbYWL!0kY6U1fU#q4WD#7nUZgWRxH&uqNG0*7~68&y1NEURje27B8?BmXK%i@%P@5L z8vi)vyA5_B{AwD(N-!Dx21v&>_*=9CN;xDS6^9U3cg;7@)6C2ss?(I=ym8}(<4|;| zYVg*}(_sK`m^WF#&=iQo0GKOx?e&;4c~Zk?pgVBdc7BobD}YKzx3VvBRIHFj#7BvTKMMU zzuha*$U{=cE;x-UV)F3-+B1?~+UNd4Zez-qg`ER<=OH+w3gh!{dAm>JXeY|Jp zw}ytmzW7>E&q4aK2Lt+Cyqt7SJfr{FX^i$cpMI9b4|<;{!!jEDjh{;cZd2ZH>5Yg_ zMz(3S&QkE|o%*(%%uAY)*ZIHb1Ho~alj#26!ul~uebK&$*Q9u4B=9&?7$M_BAEN3Z~!yDFCHg{t2cQd zza+(Za;Q$RU0g#z$N?5Pky7a|mUps;e-ExX*-zY;uYmbLG+Y0-Q#iQ?^geik+FF(W z!oI*Qpbo$-zKT7j!r8>hJ;*UW`{|Pw=fA(32zn?74SF0-cWOocw?Uclzw)>fwOZWI zG4SqtVz1xzXD60-LLH-EmX9;2^e(jkc5T{0j$?CDjKqQ%CJUTC&408X4&gQY$iwHG zF597lskd!un&VeCwR-nf({kfcUjETT^sh}O5#`}3 zarkkkN5EEn{3=1qIQA$cz}BDulVA;nuqPB4R?za+@_IY#>YJJ-->9RY9jtEmz3l>8Hw)1qA<_L%V z!TNdl_HzpYb&s*ov*uf)IWa$dOZ6uv{uGPh)nNiSrjREN0Q?#j(=>9B9sWV(pn2eH z#nc3`?VL>lTpK;>JyWf89rXPvx)JPqrMY=+OP*V^`SEP5eTOa107I01Cu{g8!qNSZ zCZ8Sb-$>Q zL)l~I^gT82{Qmr4%0lTg;iQHBuKpX=qnG=VZp?XB{l&@UxIolUQU4BFOeJEcQfE~= z!<@1kTb<_?+deoiF7NF!J_OuMd9q1Q?5jZm$CJ>4<$W46m<0qvyMA4be7yFA%y%U% zyksR5`NX(rH*FU+Q_sHs(Gl+%&iDZ@xHsx~Vibe|TC)Nn&A-nnwJe1J@XqKKzt5IG z^4S}0>9F7Z^Y##f>0;xCwqKZUUAF1~Af2k>BUptu4&v5pte6DiXt75bHZSf}A|p^a zd%fHSy$LGV$mb^k`s7~&`9-GFoMZe?X1Pw_&td#<=x_-P91Wc<2K5{y`>qOKoaB_0tWuApQhobh z(7d9SCZIEVQF&^1%qKSR6_YU36y9d6W`R7-q2>RvGbB|M~<|*V#)QyyDPsa>_5#sh8{Errcbm{o7;tG)&6IB15y5)(E Z__HQLe43|8k$B*r@@)-;JUNr6{|`iGuV(-N literal 107927 zcmY(rWmH^E6D{ERa~Ku zGT}Up)SX^q+)q1P%o`V0JQ_PzbQl@taM>lGVBUR-7m!#-A%O;c<^T(^lAz+>>dsi& zUf!%kG{mcQ+^#IXzEs_qPqVVP+wOFvIy`Aw?sPn!wB34$r(OQ#?JG4baS7E(fx^%S z$@n5LK~r>tQ+x&CIq;1#cx2u_{NG=Q5o$ogMBxADOJOv9*cpZ4z5wI@yBrF`&KL|l z%uIOA!~id|Eml@o_{gLR+2CY)1>V)eZGzI-hLn=hBOUKw2(V)#@Yr%T#giBWcub)1bwkIUT~S96A|9O ze?M`_=uHe!XP3(EO3BF7QR#$n=?5Q02)Lcg+B-QEQW7qcR8&{AFU%fXSf%)~f#=}V z@T-=}VK7`z1AeJ!&q{>tUuwEJt^I?RU;YXjmAzI`R+K{V{LUg89FpdY*8CkL)4dKH z^ze)b`dto%AMNbL-qt=XW_G+_maeXa^QhMs=r*#k#YtK6d(H;WeIBh=*Q({w7|P#( zIGw>7e$&zPw@3Qm1N^l8M)q4}4M#fIATf@)BezzWZXS?ufUpY0CDT9PX@5lrf3z({ zn+2L;?F?AwbQc2!qYxmV=flr?+xtkWKOC`vJ~6cy|ELdgmd&Z)$aZhz-mi~2{>-la zp}*#^Z)fleG%`FOQfQTviF5e&=M|DTx17Xy=AF0%7PwrF6-r-Is}Fuf4_amAG{tbI zT>(PEv2MOWZ@SkfgjUlZ-R!GO?FNH(8Mnt;E@!DpSd}&Oc_}k?VE~;jQH(85ixvj4 zn~jzQHQ0GS9AiAaDT6})-P=;%KH1c6R}gC>^3@JE95pG0AIP86-<^lZ`yW^g1NY;n zf{iCjOu~;cw^pl6513shZ&s5c3TU$4?uQ|TUAlm+9ue>DhRwyzO<_vl)*|H{7GMJj z=v1^r`F<;esi)I@L0#SyLBHWrp)i=jpk86WAYib62Vo$z-`@yYm>=f^UeU>Q@}Z#+ z8GsDozs{&B>o}3~K8>-(xe@D+aC-lGvuLJdDB7AG^vxR?0aOmJ8ycN<&j9qvz}0f@ zuVF30(2Bs;q*w0)Jp{8Mo zdx8-bi@rnxT!R7~_tU4GjU`J=3<9ym({Jpx(7;K6TWSaZw?xW0Cy$oSX)8|7!tt)= z>wLi^NZ$HM?7Ab~0+B+y;MByTfefZdlC?!4F^K}K9(KPNlGofU5(N~|2#>F~ zMFr;LvyZm7O@i;5rL+IxNEm*y4tp0paW7Jly2nu5~@q7y|LXa?75+Z(|5Tot{6HpO4?%rwfM*ATo{*$;i8w2|_vkjnmD2$p= zU?H}_X2emZfQLBc^L^nmCN8=Z+W3zfKAkKgj1*SCjr94glLI;>6YF#*y(Q>3k?2zZ zBKa#4`j7jQUoJ^rvhH*2yLX}JO3VZ{FFXbm`V4*t;pi6D7N~uQ&hatL$t}qa9{*i% z{>{?x@$~dh!8bmh;o;#l#>FXVnME=4%-fa;Th{f2;+AgZ@Og?Pu^4fC1HdVI7@p)dlg-3>EEBX#dE24e3{XgPQBO5NQg_H>bcB5^>x9()#>RJ!4?iT$ z8(gRVQ1>4n9SLC-!YXI&V#suV0$J0`s_3YR+@37Rd>LQg`8H(zpO=&YUZRE-xfB+M zu(gFXo=dh%{e?B5-P0496N^&GsxkpZ?+01R6P@DAwY1~iujY~lrsUrI@6-CKb9Scls!ohyV3mwzrHF2^9J zre?*5_Pz%9;SX`+>rKY^vv&`0Lx(r*FFp{;lNh)VPNbx9O2T_kON+-Ct%@Eftktdp$E=KVYx#hhfpI0xh8F}vK_f9=1qb(5S z_)MBMwlkK+zHj?-jRW~&8)oHly@YUac9m!NxRQv1ESV=``!})v>V@`Ix>415qm!RP z<35K+n2UR{w53)zvO~O+{o02JQS@Q@^||HBGOO8<)hWW8?_~{{ivn|@+UJ&^n%*+* zsI0Uz2}p_6s5_iK^B;Q*!W0e|C-A=944X!eCwE>8uV;RFijDQ!f3KRH{Qi+3!q8!d zG1p|h+(W<)9AW!%Hj7CH(QCt8w}9E6o}G;#bYGKe%OI71OR$E3vCZHf{?4EB?Ot0W z=6&6O6C!aazAJD37-NTX8X;yn!=YCb%7w{`W$;jOyLsfK!Q^I@eO>Si8P5*0>)PbJQ z>8^iBOnZA>L#X%NXjcLM%f{o?vtAcz z)tZ{Gx9^#t$vg;LY_a}D+TI*5>5f`!X>$jBeY zg?;X?LNCG{=Ga11uR`ZqLn;;;GS|dCZua=@315X<%Ql`qX|*{O^Sl$ip6k2~WPTb% z+s2~Ikyw5Gom^U)n^dLCLA^-R@8WsDD7LcM7tfl$?V%xJN()>sy#jbeF%1>uOGjWVigv)!X3ne2zB*}?wVZ2dw)dby1yGWnO| zw{zLmYP4MwwtAmK8c1VoY*DJUMRGx4q?Ow+SxhW63l!xI&NqCf53=%OJ=-`Rqo=f^ zf21JbcazjY;q3^fcSyQ5d8AD|wvGfdz~-;Y<1UwAYGd`mDk(lzpFUp)njkw(PmQn8 z#5}^ii=czE+5>1~wh6PP9tghk(^FVmQ-Ug|`@L*<(V%E^a&qCRzH66#5!Lg`{`D)>lWw)LzAq!M z-6?j!zu=I60DA0rb7Eb!_+0~Zzn2&2_1o6;Z$U!Xv`X^x5yTwjrL}Tex(04b&j&<< z)7BcP{><8{x-t@tth%TJu;qJu#N>st@-E3s{vj8Lnho{=OLF{VOYnJ{@g-}2R38qS zUxm%(6n}(WTu-oSoAnE46L3c3D7@rQwYH&XYxHMK?a2lX5JY?-#pt~&vw)xR&yf{z zZ1+UYo*x8TE6xci>W;sg$9Z^gA3Vl3+~xwi2u7;F3*$Yi@DB$icD9u#bl$bdN8Dei z|53@f;{fD8L*C6-GqXMFQvb!z#hIw6^?_oYizy}}^JsH3X5fYU#4AvV1t;9-6=`7h zpI)oj=ZFTmyJQGo@=WMgwc*)ilKmK!L-bSZ&|^;xoOKv3t(>dpWxbO;z2~!2@eggS zEoe)yYhu15AO%F6WvR>{v7S7CO{?!^5}Mb9Q7+KFLeJA7Y=aPGGJ>@8?*oxdQp(0w z9xJYLoS%I@e0U$~sOf{PrXn9Fs81afD6S|Y6h0txMo1FTYu4g=z%(5y&>TdXIb-H$ zUwsEh+iy_H=e_22fn(l4;h?RsfVWRclS@K%*7Mk`?0BAjhDWHK_z zpR)NyB-@Uq)~i#KJ9J(}wR`=Ut6s|vr}q*?WsQ-6K3mO@OsZTKI)i)u_3uTL%EI`o z={mCoI;TR6=T)?ffzvG|5z%CyR$LU+bJydqTA-oEMTGp!JtU)zaCB*A{-|6r4dxQ*XwPZ-ULjY|1jQ25lhqJfKR7o|TQ)k9O?6b2Nd zv9%u+bkRI-*7Q7OBW>rLHsTn)1G4V_&hEkaTxzDGzsFYDiV^**PF2uG`0|vZ-AfZy z@m}Gqn{dAqbr06~mPPN8a^L>T4hW{xFJ71CNmsopMLq)C2-of%j}ZW&j#2?2eAC{_ z(~Jgcbc(wAvk2Fh^dsZ!-+O#cn`*|F&Fn06?G=V!J>Qx?85!5Ytjj?~e9YU%y5B+Z zCco(BBMV325kGCetKj*BPAIAO^5~QACap$0y7g}>(rrQ&$_CN0>Afit?y~;N1S>Pj z!DZLOm**SGS1S3QAcMZ(HhA;ak;K>>jCZgCG~XZ7d%sXvr(eN{`KpCxJIKmlBb)C z*Td!2#ikx}I>|7#TZv?l&L57KR%y3aZ2 z^RD;jt3GjB>G*rA-nYS5$CagYSXiB&V3tQWCK~#mf|Yr8zh}E{sDi*K+CBXaYq$T^ z0@~;)ceC?J`Rg7m04LCX|5wn$#wK}jGrFi9w0jo`-dv>}$ej`sDNlYFT@hI=Uef{ z2Yutm`ttGOJq5|w0ChHz$Ym_N`*M;GtjoAU4Xe>2bO3Ah!m$ zP39pP`MP^oSE&CLV?b$ZiAZi^hzr6>Y`l7b6BntPFuF)5(TO*p=z}o^eMH0cUA0_s z3Xh1&v398#-(y&!e-n~`*V~@|ARY~9epM%g?k7RFgR}e+?`-DHa6Ppt^l?E0w+A8t zS9irOG7`-uDUxRb!2RqbfK^;ub&I&G5lY*d;Dlj-| zW&eZ3eM{M;zC0>%{_+@5zCsKTcMfW|vaAHw3f1}ZR$c_9FbltH0zai>3CN04H-2Ru z+l)E-tVeXZkAJE8R_*ylzp;y(ee1ch@@w6>EGh~b6i>SK1AU3*r1J6+|G2pPZf3B{ zsOSOdR}~fnnu+)KT$%z#y7pFw-hsFd&XB@~%vb%CG1F6|0Kz6Z+;4XjEJHfBIx9_n zKTAzUAIDayXEDLL1W*~!6gj4V1`gbuT&)s-!uhlSNB6dU{%`+Cb`yyD% zW+yfz1fTb5O{W>bj6p_+s}!OK0P{7nYz@SpDHkb!l_Ce+71q%HU4j%Q<<;nFgcj6`~m*-Zap5k~w4 zd~6x9woNCOMliM7EXL{FH>Ce_pa(LboO_uG6#t6gmo($-exAsSiB2e(8}~_&7lD-T zH`oRPwTexY)NHtaF@f)TklU7MSN%U<;C?FvGwBL=L5&_XKk2>D%!Io8&6-Ne1 z56){sP`?8g(w72Njv}k5s>x)x+tP}P2g}Y!%LD4@21m*OMoLzlMG(``UI?+bbSNpR zOtUy>rRLMT>u|rHJCAg;kkP*ZyL)()aPWs7kB`4h`HL0n1SMVmkD>%~E%-h7kU=jToH$ zA^e|q(cZNCTeZIaOY?1$qTHxcX*(A;r?R~S4LMx#q=V0r@bYmLugCyOQ7fSv)PXk=F;|7 zuC|HAQ!kmu>nO+MFE-cC9W*nWiekcI3#}&F_;);Q|3AUp4Sp{HNC$=? zZ9L@LxE2Fl)oe*qWw5`OkeSYeSU8dlYGY%@))a%@XnsO*wSxU0o{Oss<_AZ&Cm8`{ z5bv^%3XN1B!?!FEL1-Jty&Z4Fn-94yahFtlOp22p4!i$pUHh#;ptePD)(Jn-s^yfL z#)VJKMI78zWSbAaTH9H9jVychp#4Zq$CczS^ot-CT|N~dMNZ$$Ztj$oV<7va?NH9% z!Kv_1)w@<*uf%990t%lPX$W2%xeFTvPGr(!H|f7^p3&_Na1RJ3PS0G8ZUmqhK(l4l z3d-vF4Gn)Iz6l8G_Fxpv%&y9J-CYk)%`eOEYf1+mcGrkH3kmbRMTyS}S}Ef6$gEe_ z)qGO9Bz}e^4L1#ucS#;?j|ysAo>B;s`1AX?lPRnJ+0=V6KxzT>DRIHt6Dpsv)VJ*W z)OOlSU@`abzHVdcdU-Hr@+S+|hLvue>~k7(^Ys9pswl5csS3fIFw9<&Ln=aW6cLL# zWK}ngEl5shHx?a9#?}POyaimqVx@qm=T~J!_v_4S2`1ArN<>%+DWLakd|cz4;VO9V z6B=C7j>oBItKcR}zU?_xma&n&gO2*{aDX#w>*P?V(3E?mLu>km(EBgur&B@JmgTQ= z{ExC(u|_Tj=0pWI3oq3Dco}8EZ29t`8iL_N7kLLqHXrxWpK--+tt)2_ zXBmn-nrxZ&oin`%CK_{cNV0i!I>hMeDhAR5WAK5@k;C#9>#PJLc`_cNxms1+bl;iw zf&Yy6?)52aeQSGY+cWXXdYYY{mHhHj3&Ca!$$K`8jO;cs3`sl|IajnevdST^9j2!L z`pd+yC+eJ&%7!lQafi2Lh~sA0hw6zF#!U3-fcmjb%PaLl_42 zasDf%=Sk%3oG=VkTM=V#I(eqp3+28wSE2T1=A< zRSy>mR1+^i-1`f#RemCL6JbjU8HxwUWH! z#*gS;&tKHPxgA@z=cYNS>T1l$ma#GCH~Yj3BA}h+zJoT7EUI|$082c2OSm`9MQQ)w z5vhkdgxrF90~vb;G2c=BRG_#xbSQ1BmNp^Qz;3gbI;_}%Y0k$dR-HJ{Bb!=W6~zv!`=UL7Zfv5o*)g9e`?#>HT{%SGhd7EEI1P+Dn zHHGeHN6zRBI#T6r{u&OeHC;DIS<+(SsZ9H)3$&iuUOQiopN~YcG-NCsGE00I>LV~; zEFj9)6?e(T&mI7yHjX zbK?!-B0{I4$y_=HwMliz7jcTBx;aiKjr z4n!b5sIA*o6!cQ5Pz9N&fu|6%3ngr7M2_?}+JpDxR`rZ5TP)jZVy6pn$M_j8fbKti zUHDrV-9DAgg>}%kYkwybElILiDtZ##K=3zP3pMis5jNk#-Y-R1)%K=vI5P5dc+m2h z$9p18Wj8Jw?ksj7bIrWv>1z%A3pqQS0j#)X^)iMb#_CaX7bS?^RZ5FW17wgmw)I8} z@f;yhEn}|RHQItu*xcDRML>fJ8#Bx-(4_88)zy$Tl&1`6#6N1G){EUT2iW zIY{8M2C8j1)#H5}lBreoSKmJ8PK60O>z}s23;Ng~P0?o|JCpCBFGW zEa_0jm4cZbz5iHj{Lu;LZA~79*``BC2o%uIJW$e=kDN)YjF!x`%=nR6)ksN!ZmSI&waPlhbiT?_(Or${QIJ^w$?H!UrH)c@LQ3M!#VxQtSsM(pj6I4C2-e zM3<)AOvM@Od~sZjq~;s(Pfz=hcRX$tXm#a-#~0m{Y?1R_lJCGu+~!i1TO++>pp*8i47#n!%2HKkz$ zB#hR%rU`#Yg@5-B&IoE?DD{VWBRY`qG69~jVw#3?m851KVmVY$_s)RiURJ!YOW-I+;h^o7L+@z14V zYBWe=wfsQA`9y5g$HZQU1JU)IQ6wh)%mU#*h2oL|@dk(x?`DEuZ!bw{2R-I(C2`1% zC={0>#aMTf{|f-qZ$`wWwCNO@k$QH2h~vgq>y?HJp~F(Y(=nO-me3+rxIqKr$eF4$ z6NWm3N`}6}Aw%TpWmihaHy@*2fh>dC;NR$F5R%FSMevLVq+SQJL*%tk~N+6An-2{KU zif->ZlU3!uJv0&y!iTOiOraUjb1%HeQ>m$~*RKM51Px4S0eE9z?JGU~$WFAZ?Qw2J zmGpwd^hPf9y!6UHZ|%Bt48B=Z8z%S7!Ob_=?GBP75bhA3WM8kp5zF~m8v8dXD}J%u zGJ#EksAqUXx>MC^l+?`9d#V>f6!9p^k8mfT;1$S)QZr(Ix9o^t$?P3|6eAC&v-XEC zL*H5OH=HRZoH`Yp87c-h&3JOOKk^LU%nWiVXv%%*5D+p#3bEl`@;rfN?m1w$WBPB|Ul| zrvKJ5pym6X^3#^_2T_}0pvP$hP%PYdSlrxdBjm)RHRW|Ix0{RaP`X3-INsTyAo~3$JN<;0?ssTx2)Az` z=+y}GMnDZzkZkxR6Y44Ebg)&=WJVws2lpoQ+vNWev-De~L(LiRE#y56(drWAGhLfI z_qH=59kG(%h6=lyrl^&RA+o}0s{g^(AnEhlw~k3;zTEMpsk$Ko!4H) z=?{NT)n%smuBH2E=v)4209;Jpm*D~w2LN-!QA7nBZdm?ASz@{6!;6IDKVv>usO9la z%vI`7CrNzCGv9n>fb74| zuf3$faY#~)Dazmaz{N@1Lnqlw^TCp?%#DD7gUXpU8pD((g`qU;<)Q5P9(43EV;6uZXe__=9X#&3IAiTZxh6;W~=$cI%vsJ)|+rBRCP|SBo z>s&DGqUU>)B`6vE6MZX`MlUK8x&HWXZgu}J0?&xJ#C)lrOP;ZpRSiR2Uoo@Z`aCRe zeI7B92V1C!RhwE^myU7}Q&@|u0nnPhzLoNc_pdy?Qx~*Y)sP5{P z)0$+95bI9hD?He(ddL!0tpwVgY&{QcYUbQp-O{Z6`nJ!6Kb#>#I1Q#Hh~U&gJUl>- z%BMpk9%SYQWmRSMqPHaiczXX0$)bKPgIzfMwRU87J}uK-i%ek!hyCO%ogS`Q zdm>nyMk&RpSKcaf+_7UUR)fV&wuHm!a+>9UvQ9$4S;4uSgr*kNKLDJ&4M-gWj;%QMKm-+>pM*4BU$= z;Kw;{a}*>7zS&rL?>A45nvGlWL{9T~j#>+%AfGZUe8(66p&B3ygy?q?c4D`n>SyYAh}z|UoQEIFPPjwq6I3|=K(4GgMFvItS~r*2z~`1Q zVjd;soTRlfjgNKvHk3#BHX|3Jnqu(220IgT@GQ=RVuD|*D5KP-JlnR)q234d%#8UOi=&XDflyh z<>Zo)lSnjlVlX}F=&QI{P7^uN<9r-C6m;Y4gZsF^_7>F`O$;5Pc;f`5qcQn)jHo@@ zao&opRN?ECm>Lq|F~;6CCk!W~G1e6gROQ#yLfHddk>9#A@YU$c3?CK0Ap$75_^I_9 zS4vlM>tkC#+lC7OSvbHz)NuMaA{Fds5od{^rvCZ3rkl6?`k+6EfA|-Bh!fW`9+PC4Bol;t+==Q1o}20Kxr9ep(TM8 z0KW*JZLtZ=b?GzDdpymiiJ8&}M}MhF11c1>mPXEr*$+eT=)2sln5>u*+p6;3U3^M@ zjY}Y;ARaD(pJ<^K=Rq>mTIM)VbeXSFJrss-DeI`%?ZuRp(5+#m&ogvQvrV4z=D z%Y73EGDGbJNe#OTyJSS3?h+hC9>k317*4-64{zVwYurDfUsHP0LF`^D(;EYVqg%l6 zod{o@$M2m!uv-@t`h)rghmX<$i)Q<#2eu$!=s&K?)zgxAV1GZTb_R1r}hF~aa= z{aroB+WH2Gk8Q5PdQYgR2k0)dgY4MGVy`5D0d^o^v=?yXXCOPwh*{E3G9v_)ih<*- zF3AG%CZ2UF0OZid9yv%O>`=wg9rc}`YVCu*kU%jz5C zD}NiXJQ$EHnx9W@3sI0YYqP@>P^A}DpgFHm(9IpOLA@6b zY?6#fy+F+!C5W1(9jg59p@Y(g^k?$Y{T5ZeiML4IO&fc&>qSLdpt<|k69Y5Ds8 zM_AHUe)F(RmkEv;(gLWT3ICEljFSw#Y+B`>3bYp8XI>BmFhC4U{n48iEDct*P)rh7 z%dGUGAS&wzdBE~j50aEBH`E*(6pZ^jUTdmFN#pTC2#)+AVH0&)0~;yEfjor+&fVX6 zL36T##5+J%aV)#z&?)cy-jRNp3Ao%?22}C*#HTNV^c9<;A%$3$0F@L-Uv=)!$Wbty zM%1F90w0d&leSte0EGyqTUzS^>IuZF_e#0ykH;YITm=vyff#=RL{rmUO;ghDrSG84 zJI(umdEhdKd%YcK<~&i>x)m>o*{r*jRLGe7QO~n z4Wpu&TNIPIsYam*-CHw(y85hxn}+#AjtnIYUu}sr6t&Y^I&pIbDs*777L9SDZJ#%W zn#SJg#wm>>zcjt?-EZmbMkN*eNg?d;1}&ZM8FR)4wS3S{)-%uMuV1peBezhj11n&lh^=dzIX`>T$DA&DusD0)C+@FNEu@wW4Af+ziN* zrqTW%kqNE?J&WA28QB>RvI>!)cXsy=Bt!MZ$I`H;Ph!82@LGK^R@X}u{V>&Zao`qp z#mrXl+0p4o!al+}AxuzG<+{5j0U;a)Q#O>g{-`P2CRXTz*cGk65UK=I<^>`W{XuQD z{KD_f6qb0_YC*YUr+t@5eIm?If!5F1lkhnyFnLenrUOjJQin|!n1Znq+{M9bm$`|s z*qg(cohbv)8R{;=QS0_Z*r)>LHczruHkAcrd27%#r~lnr2B1dD+HI%=YS?dK)-9)L zFVWlfBq=Pnl&AJNf<(DSfy(AxmQ7Zf zhW1RMjv~S+4IUDn)5!ogUBARZx|3kj9Dk;5aB22#rB*S;E`n3Y+n z;)AGLr2CTDq7bFwXB$sU@v2_9k^b^Y2SqNqz< z1$eVH+1BTJoqhq z8>dCTEcfgOEh_vCdXh%&Z|{sQ`FAdYVXB)BMnSok^eMi-fPn3cO$6;}p2fgP!h&2O zqajR9V$O0CY#Eyg92lV0-uO&hT-=-*2os{DzMf{#pb9@sT@eFlTwy{0=6d129r?qD z_>d4d@=Dn>3s+CikD$!aOcnRssjan-8v1?J5Jxgxy=j}YMHx35p(kA-zfv^6#B;we z$2aRG$(N)w|{e3cZW#cjRpoh%Kmb#4jJ;m$+ZIRSW{U)!AYaoxC$pZ#J zZxm_veanU3njtB%QASbs2d###R-3k&+Y-&?=}0LO;DIlQm3UlHT_jzz@xCF4b}h!N zO6CR`a}y}%2v~{-liKH|+{IUCGI%_{-3&~nl`;>}B770Uj7Q1q*8I3S>>1y-k_4a< z(jMv1v%re@)Y7zwOTntO{WhHu#ZQJIJd<#N=zUN-9WJ_FFZdI%i5-dWRVYuF1p;#j zF|o3=p9d)Z@E1-}{#xT_Yxh;O&bS5!bF$@%e;BHh3@P3u%|<{&o#L&bj^SY`A2IS3 z3W<&xS^sfRWw5k`qM0%dB_41yzj+K_(oj<3<&9h=`mn2*qUbE=xWPQtN(RbfI6Q3pnb-v9C&?xNs#KR`Sx%a9RKQ@?UOiFkDT_jkG?_L#8lu? zhn;YU<|j9+caAngU=w0u7=Zfj&Ld4Ok0?J2sLWdbThEQ0y$l)qT1xjr=y|?Y<%3v@-benKOzQoNb!xi$l(XVSXvDQEb=|G zy#9B)I?)zToEucn$I{&W4AU|2%2FUgiZB$mMq@Z0)h)o)F{7=ys7nh5acDf%m5ZL} zN1YaK^BmQlZl9MI^la1&pe;_%n#U+_aejdoelzjlZJ=wWSu`Zn`3C@>1t^E_lVv!f z2Gm3(>#vkGpnu4HP!V5@g{@a$ z*Gh3AQok1?0B{B)GQLkWtj7x`#bc{SZXz-Q*>8_-WH`$g_5-O52^B#~A2QDJ^r2{|1JeSd?p?SvzPIN1LuOvCEw>>hek-CAg9O+*<@vG}%68VJF z<^T}5{~MlX`UKQWw)C3fw%o_Vi^5WV{>0V#s@NEDBcWy{0R1X}531=Fnz{)H`qx?= zLNslWfufd+encRoxM2%hc3r!d&|MbT2u@DS2LQjr?SVs-t4g#mV9K>KXS|EF5OsA^ zb#ph?Vu2n0y9`DAArnps>4{Q%CAjS+Z z9sX8Q1(j7%G*xN}s&ZIP&CbdSR;Yy)lDR;rO^>#0VrOhq5#uG`g|xCcHR?F2isrj!>-}{He_OX?%W&|_4y-;?RsWT1VsXs&xc+qf_|NO>7`s4KR@D|Tjop#i z(eT=Jqde0@w{ZQhZD3k>F{WqRg;Ckknzr(_*Im`8F#35NcN%AUa`KeF(xR(rucE#* zVdbfxz0f}N@iFjD!B(8{+Ua)idhxC>s^K6^wF!42#e*%0Z32JQJ;2$^76)V@2VScZH_!HSqid%)M2AZ~>DbF{q5hlG z=V(GtH4MY`L>aIY0BbX46ERh#sbPHV2;RBeW%*2pS#h?x;u%k&%FM zeuiA1dossqmvhHv4m}XVTT0?%rJP1_*hM&gQ$#lt5z6Y9+olrdy=a!i|C$+D*rapznTF$u1E~eNcDs)_A5zi5Y-~=g5F%>*7mRFhC0X1vHJP z9mBXixCi=|*`}L6yxuV7HasMCEs!S8oMSj$I_77t&Qm8Y?Sz|@uF+UaF+wI1Z>trY zs?L60IVbO`@pZ;O{d!KMP}=9cexhD@x-uzi*OHuntS`;|>F2-UZ+4AQPHm%@an;6q zIe}L;gx%R}y?iyp-y!-E_cAe5qTzS?u%i?BsQ0`^!`Z?y-ehZ&aLZ);o-E+YS;U_@ zvDT*-<8VE6d55NvR7VQJPQnSVwS_E86>SCSIK1ZOy7ljN4?&YE&MlGpgUHe!@*A$r zzxwn)eevpndl(ruEn(|YvuTE-1W!)DB@cz*6_Pyk3d_I5c^;S@5uXa!O6exb&=Tqh zW_kbeF2X%sU;R`>j>3=|NRBWOw7GX0GjSo$n0y%ePf*z{;ZuLcsDs&wjzi)J>+}B3 zhL~5LSt#~*K;$FP+`>RnCe>%=mT{~j-mqqY;zjE4J1?csv2>$-fDJ>mN0;P=i9QPk zg9!{BqNML&Q9apr;m`musK%m^&;IHjjrzzRaX-~)-p^e~Z)3(0k}SU~O2enCy<^wP zftZkcUTI(hu_Sd1S0{jtiWx58i}(OP7p4cyuJPpMRMe z>*>(z*e}PcQw~20j`!_C`T4{1{n|xP+l;>?xBs-QKD=LLYu42f?tbkB&qAeIElE)hv-n`?%A&3swJRc#shvbo&tH+=-qm)%w{Z_MG!S zZ)fj4pLWUXxo=BPU!oD#s5ll$&`8&;^`72@nP%sxzuu^pF0Ea= zy4`x50{&W2#G0)`M&FkdY2cYH<{pznC1Iwtmtycs9WVUY^@lxnS=1dI=UqgAy)~ix z4oRuw1}-z6=iMoh6p<-adChNKx6^P(-n%7-Y_hu=AvP~#jOT+Dz41PFMk&~)!qy-} zd2A`7#kSWMoB4OI0gE8WTx+@$W)?P|z2{TJ(2RGfcfOWeEB zpNM=iwN7;kVb$=TKx z{^WHTd;{0QcinkUM37mmALW&K?Hi(BOo;!_e*q*EXav+Wnp^Li;Z~ssRGK<$Jss{A zPBa{AFBTZ9wqOrod9tjND!3LYaMJly-CczP?;*jVCsU%IAO#IzhUtA`tjFkP$N4^p zE7KCAyV#A6e2zePXc1q}AM#?o-6ebHa_7>^DgsZJYey}y3gH8`=YT9$>a9v;y<|-g zK=ngHBBW1sv+ll)Ey@^rd0ekbkk8@uy_STLvh_}I_+*}UQ(&IDeMEYx!v>E<&ZJzfCy?YL!n=?o1IYX+N- z)8zm^k3b~0=%C7s@fD{Fqsu-eQWEM<${cUlK%4jn(oV2W;MK`e@ZvSU)-~C}Wdd&L>VBKFk#{O6m^AnqSW{Tg~F73KCP|0G#xK4 z)swbFPxP$nY_y{k&!u;4G5lehLmK_r%9p$qiD#U%QaLF(K3mRBMcuFnC@Xi|Cf3{Y zX_pPj9z6>Fbz5`ihHubS8&p5Z*FdmX;U034rkzffs4jA)rL7+pICmfo7qiOPqxnezYR>aD|~`o1?%7>4c|5RmQ?QNW;)?jCxikuGVNAwM*TG$^15 zl0!EPrL@v5-Q5lM@cljadG2%npEG;yv)5kvzU#~pCp?EWOHPLsWabLi-4)b6gvs`Ykmzg<~`=a+<3@>{4yiM)z)402ey@WhqXyBw3o{dctY^mJ{q zO+~%+_IpVW)`MQX9||x2W8nyv?0KS@xO7m}g|lrZl}JwssOP0Xwu-e?7$tqwGcD&Q zCbwcB95(w>AiHyN52cl=<5Y}24dT6KhYPMoTcq@wNKoX#9{2SY^zkQZ#?YtRjSDi> zp8Jz8XNHewxbay@l9VQPV4x)TiX1-tU54#vXj~;VG88Tvk!E0eh0w+nH5{c=!t5Qc2B_bs#WF?D$ z%bSMF3EAyE8vAVd4=*{pd!R64vTa&^@<}5+Ns=sX2*_;T(pAge1-Z{f6mzyeJxxKwLKP-_$L!| z8!ge7_4WCQg(X~f{EYj}{qL2r`G2bMw-+oduf<9E;^3Jp4?IIacCkL8(r|~rR>NHo zB^=~frQ==DxS{jA1=MV5syX-Ke}RA68~ji`i?k08zOP!3hm&=Z^kp7lVDFOab%B(Juyfmm{wSe`2|bmX2`DB&%Bx}ji1r73t+vj@*l*` z&FY6Hwbr{~tv?Sp)8LKZT_r@=g679TAp?_sh!e)Q)L$~6YeP>@i_N|UoCf^x;|uY6 zxBZPc=$0wl|9&;CTBYX%7gNO7=`@XBqXz#zQNvin2@=eCZn7{di|@rpH##vUqD&8! z)+iymyCz}M-}S)by&f|s_z^b_*(0e4dLTYXEu-VOKLGFDZT(App94o5Ni6kuU=zDo zb!Yi~#Oz}w7_ffvHJ3Q>6usqU-fQjkY1lHx+lK|UbOSVOeX$B9@&9s41l(%5gJVct zxZr{@pY0Zjcnr}92&-V+>Y%n*R z$#=ci%vSI;fJ!27GOk5M(Vp+3Dg2B!AO8NS(0C9W?I(9U8~)H3$n{9Wi` z!%uMxT(Ap%$ zrdYss+3?*L9tl2-l6{p0@oQvJ`4Z9T%o>%Q&nkBEP2VXx)dMT1jX?T9x2EB_oq91KX# zyDo3I{`KdYTf2JMw*r%XBbNmg)DJMsb#N+6Dv;FT*QOn<#4^U^?8z<#Jf-SS3qB&ZY=ryfXe!8!Re>g66Dx8TDs`KWxu?zgrD!7ls zH<|GOw!OQDOs|nEH}*DGk#q!U5R~MimgzY^{AKLxn^8(g!HwcgO|2QnOAA`?(Zt6U zzXMRb89~p@073!kJD8vVwH&>@I2hnY>u7S!eih+AlD@UbsP$74#`nMS&|8p#{sup{ zm4S@3zqF$@qWg9(Yu-{r!^wo`M%U1yUAG>4!a?fP9DtrD?}NELdQAk-BRF83n2-kg zC1H5RDy?>1pTw%KsnqH{(4Vx%)cX_NWl@C>61$tP94i)LC*LM#=S6nrCzram_3kH< zrf~K5Gv`EiQ-c%!k;=tJQ*OjOhtac<|9y=v!-cd_R@aoV5pz_TmNqRjeIx%?w0Ws; zjYcLW85=ln1vGSH@4~a{kp(amc7t{8@Imp!^&%Pf*m=yF*0q=J<3O}Us-~54*ud5f zlEiyiMUo8^t_iC!`B=rIzAm9tyRK`VV<*yc7g7CoqLAYI*1LLl_8e`HNu3*?Un8M@ z+o*5a<;@DS*2nSLzr+yt0fIlDQeZ~$GWP$@$0S;b0RYi79#Ea`bFX-UF2jke*%R`} zz2)VLF=ck!?w%${VG|)DxATIPzL$V1ZypJ$z^p+g0Vb8CBgzIcB$8oD(AtkIa2^?? zI!?h@<*&SA?|9kg%4-)=cJ38XEpGZMs3V+BQ$(2O8g!Vc!6b$9`GN}%#o`mrI~aga zBM|vWFF-B;Lcfib46PHIkIE6x8 zg^esZ_LzEDj<{>%GGLj>k~5S2LMy}cn!!^IJ>n0je{9QJ9zUg6b$3Ai2V6)|Km z^sG=KRat1khe)T%BL0&`gSf{Pf&<-2QHMa<^ruu`O) zsXTp?IBpIUhXSBY^0q1@7GR5jD1_l`^@SCv44g~TD%J$b21&hj^PMYRspB;wWyw@L zhq|6E@5a?vY?uWaZZ1VjRKL4>!^|<3f7Vz0J#=3J>X|L1n+bR&o_t7H4wVFSH8Wh@ z%9O8(baSuj^}ILIY3sc^XTds+(fb07?u|4Q1=!;!O4`TM^<%!3D^CG+?GX_Nm_yS< zo)56K)Xs;Z=Il8ZpuHBy)tx1qGtzMHh?%)C$#f6jOS%vHAC_#2V&CXdn^X@ zpdq}}#M=5`j{+3rN3vfqd)#xpT!2HN7wXVLV*M|uni3Ml`&bV9LGB{d^7EYUNER;W2M{VK z*2&B62ur=O!A2;&Xc)foDB{fHRKv(5lTzL>s179Kb@bJm8RVxy88z7}*a2m5{|;DE z;fpE+CIqkKvL;1uiuwWS9$uG0Jk#VP z_3?HaDo0n>m-+D_X~O;%HPT4T+p!Nb#kiCP1BsFgjEx3~>ZL-Z>&DjOwfw8h2O^T! zap7vws*^FwaA&l$i!<$R-MjU>GgkEqX^b(B`_XUSjnSkTP~GCvg5<31?B3bIYjNcM zGC(|tkP!MfmPAi6RgVH|s$G<*48D~3kVv1J^7|z&`tHEe5=JZXFw&E9-~Caz0ToG+ zHm)_ke4Pm$=||8cSZw+m*cXd*yO8DLYXjb{7yT90JV#65eTzh|m7n>f*{1ImjBrv( z?|XS3<$`hkPL1!ny2OvGt?11%Aus!|_j-33RpJ`3+;=ldu&{CEWHB2K8zo%Tq#(0( zSw4!dZhPVzwabp>7un>AY&=EnVMKlZJDwNhf5V7*wNP1<6O*e6I#I{bUwp=}oDv}^ zOgtOVWs!(ytYQxZywP8N|5srkb2@;A{wbtmP^#SGrA>fgqtPQjmY_+rXIX1TM|7LB zZR{;NfPDHm-GSFX-f-G#C@S7U@;3~G-!_?>EATuv4buRyp?IvR3|i3N8c>CSJknXIl5pMsapqIME4zO+BBme>89eMI)FE6?*CSxo9@HQ znHg6UJ854wG?!-d*qegA!2GCj*=$OVV=@L9=gcI?)a>#UAx^b*;e1 z=e;!(GZu5$_x)a=0`B{bk6r7&3suD$bld5b*_Vgc$V{ff_KsYd;TNb{fhr z2~;WbIG=R|ht$DXc||j^GY%^R8S;H#2T2rQKWb~r@Vg1yRcaL^oiK)y{<)csfj%7A zB^d=I2``4INMZo3qrBUqVNBqs#*TalQz^lO;-6K+6EaWSgX&4g>LxYay++ekZsi`p zf}K@34a2j34aa%+`qA>XJRo6Bn7h3@PJ>w$Gu70Ge~tC!z7!Q$gP+-Voc>K-1ttI4DrGG?pBM0@Pj`08icHLoQOU zQmVw7TvwvZjSB}bCH%VRkkk=qQ89~e$&wpjyYCx zQDLBe|@BGjnfQxbd=| zNm#&#k4a)xU{lx=Q{ssl6lWx;e19V3x7D@vd;o<81~dK3LFP(6;eU9EgAT^dt27J@ zD3g*K=(GeFW-uYlT1D`yv6@LOIp~iwYnz<6?UQx}b4o=HKNkxeF-kK@Bk=HdQSge8 z`tO)}9O%Y<&+uW}aCJYloT=kg~W-6`P9t9M-Nuwc6(~}q-7-3cz+E$T<^h9^B ztiC-c`ta{x5+<2odQ3g&E#G<;($_e9>o*t^vh^1k_$Vy_IWh`lGzOKN!dmLU{1LD# z$Wmz;IiSt<0!Kh;s6V6}{vO{@IZ+z#J^M5uL zC+OysbBZuTDmQc<=d@n*MI4tbCvu)yS=GmVE5@+_ogC=}%HT#)4qB~l?1!6$d|0L^eCJ0`iM!GxQ9^RdH_0I4x^1M73|BT^aer*bQf2P;4@ zX5RI|7y$B7ERz?_vK_k|C9qkYm7tjE(NmyZ97yVTwBQDf^k@djn7WUm=AO8=KTyn* zu-G1Pl0AlR`~N*fiqw;pC>lOX`fm2Kc@qxy?)yL{$;RIUZ+B}qH)^e_lc@tPE6vtC zF(gPGu+LVmPpk%RM_&Q9dO06oHTN*g>OCkUUqzHn>-pHkYWQq zvu{tr(|663E)lNDhm7sG=3b7+aT4-{a3X#AtaDtVsa$b6d1j|4bLp19Vllz7NlMII zEpq?QuFR19RY8QEjsF=nlN+{w;qiM%NQl*hy-Hlu|1&UyfsPpM50nyghzO(Rqv91L z$$`R6G3q~eCqkz7*-e^m7}>~Qr3hEhA}XEat|J}5-U=UfX@>gVf3G-!ro#=v)!H!G z1WBOGVYspy=tK%7Z8L0rW76@n=2GVU6ZCG-TU(?G7)iZ7e;q}Mdx-bFyP1Sfz(r;#a0lf`dfP_*ANsGAlXpsL#^7kF$DK!lv*eLWymO3ot{jW&I z#F|Lj!o2(#08&qDJtwMhN8$l)7e6P0=FQG) zY6XK=A$RZW7PS&>=Kq_1M-ZM7QLGKh;6!@v)d;E*8^LM}&H$Te2w;N&x+=#d4S*5_+NEHCGvhaF8K}zwg}t8-Ee`p22V0VZJAkdSw8yf{80M>0$?$lKMs#cRwyjk} zNgmnCGn~jgZ}d;GehofG?)*fav;*vyOJLPQUuwkCeg`4|0BOHm0)D9pBjs}dD@P9N zQ|mSwo6K!Omze&^30|8Uy50^A)fwXutYY9l=zoqEQc~MO`-5h3gW_hWdzGk$-}H)( zqKM@A#Gmhr6b8xoPJWM98OQb;il$3{a5Fk~ukEZi6Q>nH#WVCI;EiY?52;H_GByMu zCrA1OmA?YQT6vA-jirRCcW@uc`2kItH4Ws1xYwYXabdx)@(A2{O$U1G=B$FC- zY8wze89Up1^@qCqzCjDx#A#D7m{wF@S*}QvLye9nG0WEUlaUT|19-Vu_%@z{w#_#c zVfx*t%eK6Z!e$OEgOO><0HeGAnQtaxsIvnONbUp?c8q5wF!1{o9-tg@J@4Se4l0s# zy3vx1&|_$fJen+F(L1~8)^_m~!Y}dzU~4JK0L?-@vm9{LUJ%2XVgnwgp#mA80-n(S1+NXI$s zeOJ@fiVz>N%_}$4PLl6*0<(Wx4*w{(SX-Mr(q;H=m5+NE2~CA36;?C|+p#n>`ThlJ zf|i5@_vgTz61$79TXL>(S-iZjqA0O{yAU}-Pi0|U*b|IW#-7Z@iiHS(S1=w%qZ_f;@Z-tWJABxryf6$YhNvJyP`a76OL16!$T|hE}xPkB|t^oNaQWkRL24f z>R#cAtz03zR@8~oq_pw7&!?S1p8W*N#>Th6Bo`H6&HvogT9+QSqR{4WViqx?PYrsE z?7MO_>Pe`X$kD~qkS=~W7vG{%+)M)0d<1@wqV{RIi(PevM!j8mZWwclK(iC0>^~X3 z5#6<5Etyd}r!LX-^qj_0rK;-Wekz|9v!Cw{8g5nuU019{VKU?>I({>B7qkQzlAYqYo{tW|44K&ZAgB1kN%THimM})DXwBO6^;I7j7ATd zZeB+K%PGEFMq=& zlGZ=5jstiry?c`~%q@^86M~or^+ZCz#ZH>*|iY7C7SD$;qW3n2YKg+>Q_(!qSkjTpef&ZMnUq$_*sCF zWKWil7u0vwFAM;&Z#(VTbd%&pmz``B(Z=gfTgRasB?uS!=USL$NBsbK*cpck6hr&g z=1{1NY8FQ)Kq$JC7LkT7}Mt&wBGH@g>hqLNc>;VkalPq{6o`V^OPtz4OU8HOH!_2V$rp^#<687%in(DmRe8!dd z_20u8iKdM@_ISme@M6KqzuIrxew;ZHOtyJ2QPcfBtB>7%D1yt%L3A3n$%1HG=9-f` z8}FVhQOQmgmGu~paJrz(=@I|#O|OS}7DNu1la1tM-wo%k%Z!bVeslXX^YP-Q?LLXK ztMll^j8<*=KV5BD;tsN@Ji2-Q>_L9UG=;=20qAOTTK_{HWV8$@m-^@!7Yn>A;;dP> zSCbI@Px&_l3MOMZbO{`jcK?yzR4Yi1pw=z^k60iWM7vJTKM!A&OQ88w_V$Y}Lqeq| zL!CgJzqEtR050Ro_sU({&78`9tKIXQ3*tJW-vxko{;a<$YKdP$q@Nzq1^z`U>7*;R z0mLM_qU89gtQOXc8ZLf~zDmVpcr?^fi!T_KsJK84)yge4U?NdDps#SME(xSY#cj$% z5WDKfO3Ggm^m>xb1~Mv8y+$uydxRX71n~;k4dMA+;Xku)@AqxLm2<>@1uB&RBpRxS zy)B0Gx+9O1k>3N}6hJoLFh%ocPQ`AFeRWJ$@ViAD}3Hn3ZMDt5MYytRM z01E6OBPoknYxJY}xmy>~NiPh$1N@R5jVibuAr*UIZg6oJ;Qg2@_R&EXU~R!2EFTO&`;c&9Cq;e8|HfSq)C+)bc=+9=5KxKqWy#3{&GPm2RDz1@FxQU+5WP4((pVVRCFw-nxO$R)he4Qr zmnde&K@BHxYvSY77Y0`Lo7=UImo_PLNYBH^ABL zeW0|D(lIRLv}VN=o_i*Zb=966@(59VoRR?;Q%>2QNBs3YE+c;XV$Quk%n!@g@_tg` ztt$hHqh0YfA`i_`toG-SWp4o~CUx^WNq&0v#{&VRX3hBm;sHG~o(2{F7FvJ)9Dxf4 zlGF%(aE!pa0^unGyzK>|I#1kgMQz!0gwgeD-h>K8@F718kClC$i+5Lz62Blk%dZb}Eh6~7 ztM%lu=DaUn}I)TFo zMM^zHS_eJA=eM@R$yv{GbEX)t`)17&zZPuG7~RbMe=xQ73mFF+fR1Xzuw|2%$t`_VxkuGyP{zKY7M538e@rtqJ*>zSK1>{#;I;%C`OCxFN#r z=dmvv53$R{QnGG!K-p$Wr#KGXf0 zn|Jr9;2%I0r3j9<{oWSvIX-*$#fmdZvW(=b25V(L))F=?Es4Z*^<^3*jf5(J*R|eSC)~=3fpQg>@$Ei)WBc}d zp%0@|q|ga(6SP&^fP7C7ve&%oBI;)|4EHU#+kMOpou zJ?tCotv)|}PAgSB%-@*W*Wn&p_)P$stfxOHPgnH7g}z<4WLu6B^EdYBC+w6Sle!%S zlY&PSEfYp0v&%svfFLOvDorWFjqE44+u}(5RJG$mdpY~X_nyKr(D#L|Y3cXfyXi5F zIY}v3sndsPKVsCYG8U5y`JpF_f@!!|35;p(Ki);+eJ-(OGpe;Ts0`j~YVw+FIw)P3 zZ|c>Y<}s>u%VEqp>157k)h&)cEbb6WiZ{%q)(B!ATr<0ZbwX))bC zI6c^-_wg6sSc}1fgo8NrWHFzOv<0r>db5*0pJ;8|#+?-Mm4|V5u~^5VVh)Dj&I!&o zE%wCQ+&jp%e7?TMs?}+>qU@RglUb34g*3?s%=2GmtRnl0Uun9>sg`1fu^UGc6~M38 z3{2G1m!ZOL>8eMFHl0Ge&4s8a+RKcz!ey`KsoKLapR2V(UWvV4{CWV!!KL z!a;uZKLyr&w%npZyyBo*nj`X+L_bC|9<&qR;U(pz{{<+bx*u?7u^l!A>iXSbJ+uD85JnpqOvd?^TembmLvnedm z%7iEbdo=ZTft0C&%WMY&co{Mn+76$CIx!Movfec~`Iy;r~kfa0EliCwZu3tAFnE zQ;rrbm{1)9%V)j8Ak&YH>dIMmU{?ornS_+s;);AyRl^ag7y0uLk>K59{230>SPJ2$ z*Vmg+-tYg=|4~=D#1sjcpW|GHU2^r?>0_R|ty|?c#~FzedEL#LWb5f@N_KP4)O_AZ zwpei%G2a%`W=b*cU)WzUWB9%CaQr&X-}Rk=6QASI!JBvv312cP&8NdJD=(~8?ikwQ zEzIaUE2&bJ5=H1Wqbe^^eb4c6r$XR?c)WbbbLg|C*ouukvBSet)XIVQzpLDFoypLJ zg20F}i!PYdI^2br)?dts##3f3GZZV7nfT$#%saOf|V|j>;$@^Zz{C(=lD^ z3eDtTyxa3Ie&Z)ZH`A_8T+~8xeA=JatYlKLUlc%^F~VDhog%n`c#<82y_!`|a|ymK z{Nt|twDl9b1$VOmH3G6fsHxiWJ{~mf$c47L_QTA5C?9@?7hwt2UYhxM`!U)C5p^BY zJg@Eyay|c6_56KzsFP#9l&n;kmgySCNkyAz_}?jw@%UTkoB&VZRfH}$#u7n!fgowq zxb*S7v@jB4B7cc@NTkyjJS}Y(b{+PUBy!C&otQ>`SG=Hq;d0b7`8HIQtZvvV?nGsHXmL z(JVB|j6Gtj&qVsK2!cJdf$t9_cSqvHm_f<=6*5x`l<-y@~^KJ8AorST%t*|Yl-chbi{ZZd1ZZdpi%ollCkJDEqHtUF6ZH4!eb{=!=qf?5Mz`yO zH)&9#Br1+#lk(R$ZZ7`wu4lH&!HyEA*D^StPXd(~ugO{fa;@y)E_d@%3 zXDud8(m^-d6aP7vFf&4?YHNjyXSSyc%NiiMXUS<90y2M`2xn7)9+7Z{GJ-N~8Qbrw zZ(ROwIHrSEyCuf>K!mmV!HwTrn*=XTb+MX^hgE;x+u@F(la!F`pb=q*mR1wTO{EqV zU0Kcq1++6=!vWJl`C9u1kFuK?S)yni$h7W*m#zpE$Zphlp+wliz!=ha-f2N5K!zwj zdKtq?Y30g^*j64_(xg=0e(_b}n>lgRTg0y99*vA%M^jHkHfM33zxZWe$mLe}SM&BH z{4wnE-T!JR0067#<#7mgodX^FytpD)b7ccc3u`1bpU|w7gAoGdTAtnPajPgI`C_Pr zX`{k0OFk)iz?d9cTwpIJ&H;f&j?Yf)Up^C)TemEAFh85SXm53jDscGsNzl$p{uE(< z0}0aQijdeT?zC9LkQF2k>f0#iqLC92V>8;`iA%pLI1^|2Vw<)?pNh$SGh}uOk~JOSVs~XNYOD6D89k z*I^Czd}vS$o1VGe*1J912$G!(-RFO^%*^gNdGE$o`|#&u^0YEDi2^J2QF->&A&MpV zzj1wOuKoLdr>uV-jU0(SabOQNNh*+u>l=BSV{b+2>pl^mtAta+bJ?SG^WFOsr!{+B z6CFa_X=bSs3}z7H>&CT;*@0+mGmqrHpR>N%$1KLEA~OqOqKR{uuccEZSCsQd)GCr^Nj_ zJ~y*_x>VJ8Oi2ZrKIG1=X01S^N4;ZAPylbXGR>=K3&fU?WoQIC>%6nurP)`y)nogf zdmHs!O4v)$;O=BR!6Dj2#@gq`$6`&QysPxVgNGZBXK`yUGKWcOep_L8I!OHTk60Ab zzPU;CZW2lA3kVsK3Y{By&g}Tt6(UhZeeCC${_)`D_kUlC1RHw3<-!lqbgfQKz%cRV1-;CfI`Vb`5#`wdDQH{J1~u$p-QPyfK6AV^0?_uQ<|?=oHZc z)rG?rNEE!e^tqO$8D-~i27aKTXr&bQtn>}IM+aZH>$6YXniKBdKJ4o#67%*IhYoz2 znol~&np?@5o2%g@H;e)fMt~L&(c0+w+diJa;>1oNuV8y6byqC5`<&Zt&C^|FVxO!E z;z()6{QPm(Ez9WY%knZ3@#393mM8ViVlBWNf~SoRll|f1L?^`Z)?2Sf8ziV{isCv5 zv)|}EA!gl~SZ!i2{^K-V#hBU7T3Y8&X2pbV!z_t8k*=5I&V&dgLqdC=Qie%(*z|_y zREN?^udEBQVuVs7NGW_zoU-CzbjjkaH(6SNZHfnaJmWr>vCNU(~G1t@+rge_6q%;6pCfdV~!|wE|Zs04w-&e&v6#K&161G-~^xiYzQv6pph zg9ZZ_D{$-R#o$3Yf#g;!Y$BV*UF9cu>rJ$$-q|14thT2~{l_ z_b~$h-Q!%i@A-ef&!9!5i@50u>AX2>BUXz8E1)jHg(#Y!SWYI~O*Kiky#Df*02qGp zF)71t29Oc(=XC`qdsJHUl3}FeRu9y>eNG2wMWazdHw@K>!ok<7lEQA&+K(HS@iBR_ zEMqeHFkblzVaQqfhP;vFzuU^h;-W7QXusicGO*TjV68XdZ{^w0BgjcaKz>fkTGMrN zVl`l_3Zkz~-!KLIc%fIz+M|}O9`$jiUUBLm)*J?4L)R6S>Z~wskU65phB+j-NVE&< zcd@F<50Dkcdkhsie7*d{@jTAIc=jQ*Ats;Da8EPQD5nCynah&7@B#$j=q82nX+~0OypAaDPQ6fQL>@tlo5mg6LA^ zwY1DzRXLk9ty{-o^0?!U3%d=2{Pv1~s3rj}_Gmph9OGm`Kx=e=-${gFD!9-89VQVX zJ|Z50^Gb~(5|ih-?jSY3ULod9I6SwS&g;R&l8g}@l|(Y2nv!ntAcul1fo&grM#N8U92`B98Q}v}@UF%unxG?WyY`-=^!-S$8Mj z_;K%i%>(}FuhAqqAp7MoB0>V$08cTzc&ZF<$V6nB?DMd%=i23W{{@d3;XZvjGs~7L z@oc(&#?Y}GW!k6}^CN|wRlr}b&QA@nVjmEMXN21O9QriHq^sm}j;3X|a`&p3msmM3 z3&zT*MRQ~+h64u-L(|#b<``^9HteYbZ|p98M|in2nxRuBnqjX zPy5qV5duNu8@`5+C()cA?oH8S-u>-go<4bTrL&GZHfAEVomNdRfw9LGU1M<;oLHck zqYjvjo374iId(?9$*-c9AoGZ>rSW*vuE+rkImo0-<#Z?ZWB7cg>X~r^sGz+w8g4$@ zbnmqrqMFF>N$*(d9Q@$FL%52)G$DPhLfgrozkXo&|be zXr~Fk$NG1F`)8^)D7qEnm8KRlwSP1WHG`W{ z7x2AA0ws2$%S5qiJ=ysiiBDTWN{Dg(emQ%^~bzL#568%S|d(kZld==ah;}~Zq)DnTl6mJ{*f2xk8k}$Cnq?W zkg|S6BepA7#t(w({&VefEoFW*Av|u&nx`r+BW)*CC(I*)BjG4R6CHrtvtAm&At=CN z1!*gpr4o~WfrFk=NKi>aA;SPNkMv$gY`Fg1B{RKW^x`b%9xbro*zKMWN16wmot(>C zN`59E8MRTWB<(AqZO~x_5n^cfZM#1ss7}?UQp;(qrFp?VL}RNx!GT`D2zW!Tl*FV> zCB{Gnkm+A$iXN8ZRPjr`<0!W#)Age|u)`jPQ4SXpmnm{}9veMKwfz8zI@j#YNo6h0 zh%bn*tfbX>R{-g+^x+X@?fALT+nXrq47`+cx$q5DRmN8WV0DR0x=-d2OEvLk9f?&D zDq;*qIL{zc!n9V)MRtwBQH^e_VT4s-n`)y4m#s>lv8o_Wt8N2*z3Zh_(>4P3WFpS5 ztubE}k>UXHcL%oyRwl{e(Q*5Q{vAn`_QrC{vxQ$jT4)^v;|$&=4~UcsYr#&q-I6_> z6mqWo-|jVh`80qknXV5_a#%Iz+b_AF>Q$(PhVMh%-;oL6pgS!K_DE|G z0Aq7dGhY0yMzWmHsx@-T+{J&H^l&JHNlOnFvytO+U-5uug;|({@&TBYOv=RF@qm&b zkgZ3AI&2&hD|Muy%ih`fx2Y;3Tilf$ka}{&O&Eh!67wS*1)a|Vud95)KBDyV+B4J8 z)GcGCxV&jRAF?|_)P!N``M>G6?-$?_9?TMnehNTK9JH0m4QGv(maJQ!=>h>3Kgpou z;Gx?^CClr{r`sMAijF6Glp@Xz`+TVMAZ0i|xS7&895uDHa+q|`niDp~txbxj#50&4 zVOow!&Qr}t_GHzr7axe3NNMQ`QVZ2)ZiMJ~S;->yGX8`%d?f**4?fnlY zvic94E(}bd1XwmHs&Mn^a7hPCx-Wj0Hiu_N-Tem-MAKmgwNHA$Egv10Ka84;G)K{~ zh7;-fOnV;8^}hKmV5l`fqoiC}63L^aE0dp88uPq>C9`hG@~d9p-T%cocfeWJFxbag zl7QOp+r6*vHw2hnDejTilyssk3yhzp^$Cm|mdP#x^A(+y6J>T|)ZYfw?~ z?g`dmEh7r}C1U%MxY=y!b@e{|^wqD!haWSvYdlSHKtqmCJ1o3JX9%Kai0s6)3Yj{CmK^Api2?0ILj#%$0RC zc1Dz{F*}qs%Jwa&j;~%@mh;kM<-=#4&x`kz3t>=pliVqD^3-}_)=<{r02rD`g2Sqy zJ8LkBBa8lfN)}QASZ`^OujmP9*_g@uUH#(Cio_RWIhYeq^po9bND%4dHY*VqqL|b3 zJp7n7XX}LaFB2knS|XJ{3%D}Cj)M^;pd4wt^jGPK(2+d+cGSdN#!RJ;{oH>?ID~jr zKVHu!z(+}QKsz4FeYgfPFcmGqF9Dkc+;x}H2M<`O{m%?WV}H*4mv8V##rRV*d>9)m z5dYHWDt!h-yGx$eQIRghwvW;uxaU*i=4W|vAZl%=XaEJ`$v42^tYu4r7Rdf1if?x( zzEiQP_qhsL_$fg>L?D0g4y%E3+JgbP91fI9f1D%6d68O04T=3x;|FF*G2gau@QL&x zQ};D6J~Ed;z*}So#lG%DC&4+z?}|SNg7kL&T8dRS>ue=|f??}eRs@(@uH+K7YqWVl zKH^LJ3fNBO&fl`Tw2Djb%fjmR7)<-j@g(TrjrxA~fMV5_krQp#?)Z1W!en?Xan|nP z^n5#uaP}imw&v~JK4Vzk$+mYCNyG5ddtYs6Wfk)?YbnX(6CtRm1|HHUkg&GE6H+A} z<)cMDVwn!-*UE{i#=iNVbbIkT(wb2jelq^Bd<8)a+@Dszm;;g3s2S@u%M-o6LdAs3 zab6(!)yMq{@=L~yS6F-l8vXSLm2@Fqde!-*)1IO2d&iWqe(!e<)|eF08Yg=OV`2V^ zX5F!53)?A|q)#p-Q;>XJASf|pB|{H@otd@djYRiw4Op-9-3?dDb3EE^Tu7oek`IxH z`)zE!>^j`H`Sjjd)7a2*7JYB5iB&K`aLN*^0ig7*{UshoJpcGsFgDEdC(sB@EPKQte;Zt*O(66tY=<${-fOA6 zJIK-2PQX!^-#|m#(6gwoR#{R`j{9E4XA;crJ0X* zy@H|&4>_C9my};6dyc5F5Ia%#cCmHS;Fmb?_m=mpt(Ug*N!i#}p!)a&t87+2#o7X{ zEc;}+9ffO2ry9{#E}@k)iuEszPXQJ7sB*d8>pG6|s6W4n)dM%|ROsJg1{s{l@4l(- z{1~z|b^^R=+3_%22gflADEqU&7O%TNJlz9{IINHoitrIF>XK_SBO_YDxO|KC@6*WJ zo9ZGKVPSdR9VW{Ah|ElgSr2^e14Zxz zx^ilM1F^y~Xxe8;&>PeTD)t;Djbt=GeEKFQLh>u4XSuS{w2OrMP&@imBcadkEot(e z^G9xrwvQN{BCLNg9urW>B`z{*0tDNO6ecD7s%g)cv zx~^#`#laSzK9#riya0^fB=E0(PszS8qTd0Q8ukCn7$!aWqxMo~CEE6z|8W%U6dg$f z1yEY)<`>y(!eoM_-+m)``BG@H?%-BLFs|U@P4%U%X%W}2{|*ydCYCUFa6Gz#h3YVc zr$j;_P@WXC`@N@c0?^z!UU;NXUQTt%0^xw?aM(umo;+%DmP(@ehD#+=0K+-x-o|0J zEq`|LQ=HzA31`plN7Dn5R@2s=XG8kAtwr`lW8s-aJjsUaWsqd|J3hW|Q{DYx|3$|f zcy2ljUT?1~g-k8}+c}*{qq=5Y+}SGb2800dh$CYGoAp7+%(_vW>h~`y;;g9xxwI6) z&R;AQeq=86j~CL0?=?9FIZ*O@OhTN@s`1SBEkp%#~e<$(25Tu+BadKlu3h zShQEm7LI(LoD}zcQgc?5;9&eoNpH1x;B7f?w_=5iAEneBHd$wQ$9ycq(-!H<<6tO^ zHiNdr!7;{~Am3{Kjly~<*UAs6*4@ekpcoL3_JDU5Pi~(GM2Zjrj2Mj)vEQT+U5w#&|K{px@frzuw#+ViJ>G)|0OdAgRL zloURbw+g@v9}URSmkViFB!EDJ4$u@{z5W%v_mJ(vf55BM_sso7?;){*9hVc>xDQJL zJR3KLAw8I3;eppJ+U@WdyG=&hokdhAU#oAug?7BS^q%8MAa7IFvt&x19ec#O9%J{P z-)3y)n~n|@tyq^}W^R~*v#xg_3AhV1o7do8U0p+vNA+7+H@CVugs}oEW;i%-p}ZnRks^3FarVe$ zg1VdU3QQHOQQ9AOFI8gGS{a5Hb{`}e{NH@*(5 z8#7duLGi&meSkOHs3OuP2f4<(9BUt0PD9S=i%%QidWEjXQDAuU-p)bV%PWLe;})gq zV5#%~I(5nrhYj?-#tS$n`c!e(6<*%T4#F9_o5@3zUf*^F=?7X}cS;pXb&wWe!BZ$J zDL{>?h`N??&ZCF{ej51N!C?A}zOc7HN@D%*P(~p(;*-yQ1)(NtrBH3ExiJP%bC2oZ z)fwEtYenQK607PMgs#TZgg$4EIdOll>q!&*3OYAXR2Q{~TvOvD;JktJ2|kb-7YQr# zAXM6OPe5C*bYdefJRI#=@$R?b2D?)a&g6e#-D)?d0~6-Y{;e6T=xR)nF4r`jlQWbF z;A;{h#TKB?2b2fVCnkSQJUePyxSNv}QKwMYs7M@3sQ2^pE;a2({5p!o2t2hFMF}{A z0I0FMZ)D_nyH#^7*J1Ga_&#Qp7U%819sXQFLLH8ozr0W2SU7eEavnOS?&){6bl#Cn zrc@p@L|1T#G;J593R$;Qv>fbSh>Kl|XE}Kas|YPa2L=?s#r-5E%}=_ol=gSe{YYm7 zKx>qTXp%mejEMa8I7iQK#4Z{y@MH(Nj^v@`FxT02ovEn=CA3q2G?uET=u|vV^F}OBBhzwCeDQ#V8&A*=`)K%7Cg(y=(=YL^j zP2f{z(+1q*?)HuYD#q@fvNr>`bBW=at-3F~dA`%OYUT zLL++446vV?pg>k%N+6zJ7sQ6rf_O-;ALDY`F>vFI_5=e}00x#0Hx*{$q5{PykSI}X zoO+yAVOQ!@{7i=I_B1E~;3eH{^@uX~QOB|V|A5EAyAZEx0`N#YF1`Shp&$tbOR0hy z#D=;L!q|3wAf(12oWzBn9qtCGs3QwZjav%}lEQxPC<4dn&M9YY(1L4V7_nMQlQ|+9 z;;H^o<*9Q6Ukr$(>_IwUHwxuQ2sW2ks6T2bVKUmTrfnC>wBYX_{DC~Y>Q3%t_kWn7Yv-g zIY-MX*wD+r5xhlxR@VNl9or-P7ncI?;KRS0fCNUkc3^xbGMH|FIDnmcgU}O|Tl=NU z^1T5Yb5GFi>%olvQC$u%e8*KjT5vbszMJ7j#fXOf8!DpiESM?qanmr?tlE}em_63h zZ0cxPGq8-?qga|Xj}&9H_m8lvunYHD{L*GrP*PE>>8ikyc&gW@%h`M_!%@>A1H|&R zT5TN%_og%i3pTpxmLM1Esj>v0ulMyuPU(_^@0rp8VFPQ3nCKtf3qVcLY!az5$Qlrh zowJ(zZcyJx5*Rl^VW*$2N3faYH#@WAnAhB|m#yQMHzm~A=6D*BzojUqAI(lQ&5AkE z_^hkiXKHe4b@U%RMwaC?VV=ob*md7!mAlHymN@UQ)gaV96B^04i7|2?5+9@lIk3pX zuZKI8N1ttQ1zPvmuj<%ch zU{KGgWF8j?B*l(Nn~}?$Hyo=dEAEEzwU*z(J>7Y#Nr|3q;9vD{XTv*rnbGSW z?YbwaXpVl~){4AJJnL8hC;whP)nn&XPh`}$#!|)31J0tZwIFMDg<6Ht`O(N_>hE&j zco7p0Qs;j;bUW}d*?7&CzSw{B@p6(0q#VPQ2}@jcPE>4gp5aFpjIi(7TI~cr zIgo&27Tc9drsnL3V8XP%^|mz^zgVqb{T#+OqrGW%Vs}>_;yamSfgfKf2D0 zpLv6s^Nx7<_SsLcIb-anEX=2VIVk7VR-5BbB@Jl*xj7XtGGFkk2rO!1q>+6^W1~W} z=($R+<84)t5Z;A$yYA)pJ=^^pDhwqX%g=YRUSsyY9glHZ&A@N|aIe->)6x-#spuef z&G36kA*7kRhpun=Pq4bjO^5?)P@|2Zw%9&r(ctOpwzI${s*b_9ZW?aqVL1)s%HEXD zTf7i);rXybKzGBDc3<$T>@Hk#c=`PZQqR|vS6Gj%Fbyo>FUIreYqT_{4m#>F4K1Nr&Z@9g4g4Gz-ncG#d|43u}*AA5_!%&CE(rV&X7sq~Ee`NvQpcHdDL z(T-TfsE5`;#H_@6N?V*${131DtG?-OtTNKN}G=(%rWV_9bCmjkvdj_B5!$=b%uKSZ=R09w0YQG zG$JuWT_{PrWD|cm@Ji+MP5^wU!^t)EIEy)}zs$PqtaGbG8i%!7$2+U$kLKjce6)+` zJ<&7PExz>t$8_I^4Kv1n_7P(ROXt+#4SF@P9`GHS7QM2wh6h$8H zel%kinaovlCTjF9#{Q6Mrm@L6d#z&C(v8>Vk(Xt;|`?#y{TNp9?r?ATU4Z*Z2k_#8VO zYopRSe-q$1d$A&B>-(O^f+vUR7ob#f-v#`eKHR~aazcOldxgk(0TDUp@ccBmS+x0O zTo2-x@206QuD6kdwNv>wP5a{|HF)9M2cq~sA@IM3SezzUn_LwqvKk+>Rf1NZ((xjr$h zW&vIjC4!p0ylO`bUVmv`E+lVs+ptIBsh|iR<5bRWqj`Hp0Ve4I==Nve_<<`pa=RLD z0PC{S^1|3{Z7RaIqRZ1&^dP+NZr#F(mqgIZf`46eUWATCy7k;Vcy)gRD=cDrS2|If z8Sf?u5Cfp+2p?|xIVjca_Usx{Qb7n7fro-}5Q^$$eyCos^D=J>p@>-5h zd6&(vV78j{aO^^zHwz-yjcn{bzRY6Y`%;8_HXBkrNkKXwdu;lw=xtJy0(}t!^ z{RF~t>U!RYR$o7Px%P5}Y&T++=g0Kj=EC9e$xBz&1~fkgk~;2&(0^ZsDig<*Y`FTcHXQ`Qe~uBB#fo| z83elLcoPfXH^cuzPjEV@I@oK-Dl^=|W;`M`I>Kh9GHn*~T`_OFyPS)y)?@Xn!8@VI zDWZ`*{fq9!KEUj*qKbCdIMrCv(0txfv6S&cO@suC@p;UbywYbUMQqPr^V=#y=(wIh z>>Ku!T0}ywPp4&52~vW`;(o!CWNXSpc~RPG&!sa<1cELTJ=B$S4B&c1<@j8~uy-jb zZ)x|hFIqK@Ka(lFO4LEk=A6m7R{Q9+BTK_^`!{-%+h=fav2@GrzQ)*(*ooPRWgy8u z>%1)lJ;OPo()sy=qH(Wooqaf!mU;H3JZ3c}%yrubV7;a&90K3u*lO46o? zE9}gwB$&CBzfzHHxR zL8~kp;AaKX^cxttxZ>j5YeQjdn^{?Z^I&owd${h4M zCglwy#Lc*lk^)>jg#q1xMtJ>LoPwDP9U3h!N`bo_!1@I>2v2gg`B8yj2Vew?l1ULjRW&Q{P9_;N&9{$c?}JQ)v>FuvjPGI zIqFs^l&=x+SJq~om|(xpJK=ZJ=;&$7NYFqtGva27^{BPY}m zfAJ2joYlh6DLiSD?(vac1eR^qKS&%_5%8^RP` ztN^93pn*OVBlLFxXa0v;Lq!9S>!ZWV!;hlDTc?6cfk_TP&m7^m(`^j#=opnA-{Y9% zZ@X;$hh|(L0pe9sPEu5-Vmek&ug%y`3$`c(XH+R5Nx%az7K-hQ-G06jVEXS|oKOr( znS+Z$7`yIhBWeFo(a-prum*pYHNX9tD0$wq0hSO*4}0M!gewHe4%a$ZVD<#HnJ++5 zI~y;N*k!ZThCpAO;punQ@@rw!6$#``~EXXxWb;tK~#KwNXpN>TTZG;x^)laPsIJQu_Y{$Pp!=s}yqSc|$wM zIZwT|BU=15Dv2u&q`3xqLb_U-aDDm49T;#_+8bUk zuYDuGtG+5S7R>XdyK6)d@4kyzp|{SRIx{W5B_{K7pJQk5XNNlBJxtl@t$wPnQBUIo zJ#tBJ7z2R*n?Tm;CCVu}ZmER}|GrJPe2ef3)o~(+a{i947lSMUyiG{iSYU<|>%4X{R2gzQffexm>)i}^8SuDTA)jYExzI z0=pYN^W8(I(5Mg5@6eQMm@*Miy%i`dvQvKkyeVr&c{#sd2P8AH-fS&fK0;F&YJ|^&O@S2wd zS46{d2+D3vjzV;~@B#W83rK#QPs$h0Nw>Fcs1I~A10|F_iQ-@El<_oU+b(pHqH>3k zB)ymwBnqjcf@7r3ZH2IUzj9(9BhY!&aa&=~u965%t^A24@6g#6J@3uJo5W5l6<5#U zX9#rHk-L7W=>px)AQ#k@@;@w}iMtbEFWu`q#TAEOTL`6K7AbZ+>o=Er9IrXq z|FZk~=6Wrw!B5iAoSZ`Mim69C`9~(gbHx$rX?_G%IH^Iw01h>WxlTmfFL9|Bu}nCI zuhysm(9(qP;UXZ(Hy;v``>i2urQRbW4uIhDCV;gg;yE%dkm7>mcYTWe&endnPK@(u zf`pggK|FYFD?KKjz z21ZPc^zR9}JAxm{YpzG})44;l=IzuWa>2z}Zva4UH|;g?+SJdwIiP_cFDUokuMHib znOY1dMj;?6g04^HD1>Csu;m$lJ}Opz8XU?be=7=aBj&0q-%}cvDvkMK)GD%2=uG@L z36U5?j6w`4-g-GNlvS~J>;qeBBS|&|-%&ML^BY@90H4vqk+D=eF;%cjs@tf?z@N~; z)Y=Dn9MY#ju$;^M!opF12Bl#^z)RDU|j)cU3RPe92M#%ljkUM(Nj%#wRn+z8&E}cx$)X&mX6lUlA zJ7dIQDPXxs`K}$x&d3wbO}QVr2>m+)Zww4SJB=|TB~?F|?TL2C+RckX>wqLu`d}<+NzH7uqV;N0XBd7rjtgUbtd(Vde#Pu@;_wi-wi%Odk5x%^Sh9*A zd%991xuM*^LN;n~G8q|GhqFbzg{Igi81MfNby_1p={LbNAOaF-@;nXrcR-*F<3dV! zo6_^`0@A_JFtMLx@6iWgMd->k%^Ps7+xnkH#&CC@ogMxQEN)Q?lO9h#<+X%?pIkU_ z8TNIF^|dQ-xIe1~r89|={2yt$ND9l9^cU|;qaa7W_$h5ltahp`dTRsRzF+fEIp$rG zy!4ieBM=%yYM*NwsJP*ywcFMBj2JTKZW1j6v@tJShMtLZ=@--PB*{M7+AS_r2)0>T=TCbA$(eO0Rq^LI?D5rTWMx_PigVt$9KwcU>m z6ql5d+NRhpVMA)En*eb!VXf#qzplprc6y~arTMFMzne>8U@Ya zgMVQ6{p?ZgC5(@UDI^6b2U>Zg7>hFo%ML5my*kWwq{~HyL-jgMKsNS|X~G8lz6&M} z_TTyX=VR~zC-hdTa~kvj48JfYt|+U#e?VWG+KS zdR)c@mTO@HqM2SD;!~DhZwRFz4u=6twxt5gr>ZQTya;BD?4i`^ z{KO-8n^(@bP8)>0^5xqnrO2#ut0t^}JI+`1Fh#Fu#o_+*@>wE+oCGDrP1A(FB%rE@ zb2uq;O!njXqxM_EJ3Kw?xd4<&kQ+MO1P?~ssBuB%~OeGC{!N- z5@*yb-Y5*w+v(@ZZDU(cmR!HMN5^#ja}}6 z0)n=l8{_ssK=8=43wO(kv7Hx+?If;|bdtm1^(!tpE`9z?$VWogcwzaicJi}GFjHgL zsM4n(qq8KY~Sf!hR?~YON%H_bP)LNlWZr z>qg5ZhR;YL)OMLy-TtHzYz5SCH~5G37qNiIhl{FgDD)*!w!>yaiD499!o5q_!*+{6;Q>UTz`l;hRSS}wOm(vFpU&o6W z$L%iE+p9YM(fQxD-D8>gJD~29shk5^{>5cwWyADZUUN#5c1=epc~}Q)X1DLZC<}~> zIQKa?@M63oaY^gc42FvB>rsLOVO_j#hRLxk3nT1+6~jmC6$u|IXY zCeOqbaD%#+rYLzBhxn4_36ep!!ySU9ZbDCT{`2RUp5rcUq}=Gu^&uKiH_8Ygnm8JS zAIVoZ3G&{{PkCdgU!XoRJQ3B5<>7T{;@sw9`=PX?>^sPr*65ss6`oCe+t(cKZ9o-x z9YDB?N6-oa(ZS*MS$=9O705Xa7oCWnqUeoC5Ak^tw(+a?mSOF|d_M#P33Zvu``w>T z!3A{eLEp;dMF596)eMsf*jkAP6akHZvy;>AY3j9xP=vm*KeKSjx)W3z$n{DnVV7h6IgPf zzZ9Vku4&r_IxAd(3K8X6`lH$hDV^hMm@a&6!DjTZWobcZP(9ErLqSJtMcgLp6H6pM z^wSJWu5k~!@twGbg;U{V=ZAyBN0-Rno5`;;B!S6UPWNKUyZ07$DxJuhxd=^#Y&bAmmPOods{_r z7c40!!vF@>T8<2RS_ZMCWSMrb&;ouAj6~+q?6n*>*I8s1*a0j7%jjK$`mOCw5eve2 z!{*sX<9vGE%kicAC-N@V%(&FJ?;iNNOj_LzfAz*V(Tww{lmKVI@pP5?b-1Ebv)^ZF zgz?UOfo4s++4Ky)rBjy`*!EU3nxG(R8KBT8N?Yqk1FkkNYe)Et>;Q zW+NYac-ha2SOySk2N!OXk?Ha8>Q0Sr^UD^od=$@w}_>5o>4)ey3py45oe47}P_JG5QhJl8n zhT-;X`YjL9Ng>Oc7Y~gJx`E1N`t4tr|}^r@c?FXpjZg7 zC`W%|DR@EFeKm{o9wN>jn2^|(XFy;u+!TiJ9gaHnd87Ho!_~ZtbtI2&(L`BQOwRGx zgp0`xS4LbbS$=+e98UaKt0}Alo4x=n;#kCV2n>T*ZbQH*fTFz(LuVyJJ>Mu4=5!qf z`WiZl5FZl#BjG|M>g1bMjpc!iKc^v z;T8h|cLq%=Y{;ysGn(jjh-k4%Y@L{}P^bN2UGUR35tITj&oL!G4lAF@b|bl1rGS&C z8y%GtTR+sjr{0AyeJ1{%hB^d|a|NSpyjC(Au`f z#zyP@X}qkDNc}tgj=3hB4831zI0THEe4~Dko!{v%|NB>>Tb{+5_eu3q4GWL!DO}U+ zIfLgL%b#k%;XbqK=(*e?cpaH#D@V#q0M;*vn#n2>vE5QC7L;PEj7WSEs~;O!8(tQM zw4t06(<3lv>)YSo>hBZ6gCk0Z6k)}=r`{1@Fjz4ZT$I7MdWVnvd$*-!Rgc@rgQZQR zt+I8a7#~mkV(H;hku9hFrFOxR>d8toSSvq+$vZ@?9&v4FBxLk+)2mc{98w(DHPRX+ zSco<6@86StfA2uiAz87Xvao`cgWLwb*-%8@_Rq18yK0oRt0QwvezqkG@V) zl>w?fsk?C7eAG!rXIZ8QaFw~R;b#7E()*f;e9AjL;pWAL+o2b2+RL0Bed~jL-B=FH z9iR@@m@O)>R9)v;I?$QCg9M)i>?znyR(k@M=Iy(R$~PXb0gU|)VtA*Iyg(`pASMua zd>@&^c2V0DwD86Pp~5l=r0|W&ok#1iW6{OTl(=}c4JUvd z9#(|DKr3ep`0N*k6{)Ig@l>BI%Ux4|2Nokz1&E<)7?r^2PBpfZN{6Rhw{Fp5n_Us$ zIe11B7JwG~S z1=#@vK15pL{I46eXjPVY!vqPi%Pc3s2my?@X?Gf_h?|c*BJ-jjwv!gX8dJmk{}Jc} zknIzHDR`M##P>no`qlq}x>MBE)IOr%%5(d4y`D;YVFmVUG|HbD+!{%>Ev|L^R&jCo z6dMlL41y(?-3C^cF<|vK%`-6{9M+9Rv|ve>Vcv~$xmNzh=X=?SbFsv<4p&61jf zy3TIq<8q)*2*(j8j)1wFO7i^J6pK>CdU>HOsH3q#~{Jp4AKt$LBrr04%l zZ|k+mU1b&53wx5@nABXC5g4uvwMBRh8oWA&y{^_#H`__KNfMT9B{GS4jA`Ydo+!Zi zfX-y{G+Qp(4f8M$18NFjvnYRo_V2>C2UfrP`y2#)z1f+DTJWY)k0%N~(XG?8qZmmc zgp?mIL#862@Cq!)9ip7nK{qcBL6>3s-iV&DoVFQ8Dv#l4cOe`OAMc-l*X(`&y#KiW zI@9nl)}%)z@}Lfj6Dt#%wmbfl<7XErQ?Ol*ngLD|lo?eZ3Bt-s04me3x`|Cgh|(l} zqbrBy>?fl4$%D9SIt5x&6-~Xi5GjUC-svr}e?QWCQ*OQ!A=@bszCAFJ`p|vVNYQV9 zS2cG<-y5RQ;c(<{x+2c5GoT)lwFYz5Ni z(rY*!Vu*HU$(gwmFggebV!%=`5MHiH;qub-W8|e9^@8RO<>NHiud}$wx#MaIbFdkK zq=V|vB25#-vEoqfSbv0@ zc>2aO;feZcTpxY=-hNH)CEE3L$3Dhd|FWV0AOAa^3v{?yC~wm@nt?pL@td$8nezga z2+DnmeG(zN;@bL;*B*I?9@OVP&DhdK;TqyL_wO-3nSZJ|{Z^o$=6aX!gJW#;%Y;zu z#WVGTwD1+4;EU(@eL0IRANT>eR%4RAPf-dLwwbS>@AGg@kz4`p){wU2aij{qBs{Or zo6QKzm;fI)PdD8#+|X|}sByB<-Y0w35sk4^FzoQFqiaWF9?}1CC;U>y&En0eTo>%<{cJvO* z_Me7%sc)qtUFJa~Lat1ptAf)}kF@vUAo9=GBlQ^O#Z$c-GPZ*{G2aq4X+dV>O|2>k zm|Y*fqcwUtVwH{={g|Z8zJ2}sg*|(KnDQL1bT6duHp81+14$WAfiN~^n9IT zFa@n{Xf90b`O!E3DHY-JQ)A&u(Q4fM{~U>e#+pLh1FSnYiF-LbCa`=ov|S3~eY zv_Zes?F}~74=Q#dK}ip_YRrW(ZAU0t54H`*0ZHbHf{o`SvwE|Km%=fu(z?T(A3x$6 zwV!A{yPq?SOy}uWo8YRN!?u-HR76i6)Uzi^V?qHY)ConP9dYv0$5)q3gA_v8MVwhZ z_qQGHc>1%4pjH`p*7UJ{FDN(NGIx%eN-E_wiHYI(vH3F>JzDBKEmi5mm5EJP`@^;~ znM!4+D1Ia!y#5FJ*?h7^&_De{WAM!zQP-D%(h7x2j(Ct9qmLhbLnM zx-d?TZ@(`Ws28u9cl|~czP|6uP|?M_?xVhUBT4>-tB!0pUf1%<`dGtL5g%c(E_Q|Y zO%aBRUl)>sUFJqi2NdSSoz$=+cV>`1~f{6Q!|I zf*?SRk|g*KfZ1N+ONCg!w@O(v(}ZiZw0~&>Sg>8rS&ziIT;u@xCY4XNTi~;K`7QVt z0f2K=GNj|2SJKg~w?pL80oi+rSkS-|F|=J+e>ny?QLg2<#G z-=TMMFlzdPy(m$X9ug&XU!Oz$&pvG+&?opLbtV|zfj&uOl7A272 zZF}s-*QB24xKH_?H@dx;tnOpz z-G0pCkBpeMnToQlu`w35luX-V`z^O2RR@k_A{Z?Xm?njWqe)d~M+30ZHGD^T^m0qx zPd+alPo|Tmv%is&4f{p~(WcOJb#}9um2|`Q18+-Vd23TPuARSir)1dth%pJ%Aem+i z1(5Zq-0q!<)ZSK|nHpZ-s(D||hp_a(8tIgxAbmr2j=NWGL32&wV*dVHv(xzQb^sl$7gMm|#py zC@D&FJ%kl~Yx7@-rMZ@g`?=e?GMCk~cW4e9I?R_ieYN~^Q}&}!#c)(6Pi+Q-EC(+p z)D)L#fO#iUY_y@~kKa~9NcFz>lm zs+qdBJKH)>c>}^yPVIb9{4Xi>DUseS*G0T(?^$;9I`beYzeBE)R|RKIXQ$VuHVP(I zH-PlF8YnnnJ_>-A2lP$Yspv8%>p!QV9HaJr`E*xYUIiWe;@%#R^}hI&$Qk$C{QWWFr;h ze4egf&2}rqMgY(W-l(IJ`>lX<$IT}QfIU<~BE8qq)A8VR3IK6<&-9yl;Hsm`vBFQl zv9juT^&ZqM9Z{4XgYX1A45dP`6>lbtio$GuKg=Gu6{G|MvZFvzypfgxaOf2}2v^|Z zra$M7Fw(p;} z!lqL?k^cz_kNuwfEsJgh0DPF(at1j3?hKA;;v`bi|GmWUAH6yp)(m1E1>c;p?6eM) zPhD>8Y8Ts=Z_Bc30hb;2Sd;|F%I%6aBC~ERpmD1BJdZx(BZBS0Bcb}vK>aA9-Ew~K z@WX|MZ_PJCQRL?bR_^5>Ga&)tP%_y&AHgNZ4gAN#YWoPp32?SL{{5Zv|-nw^j}Ui&fBRV7~ECnce@(S^|@!G+61NtiO{9a`W4L z`+XK8+t+W{K$ySgJVZZ8-tolLcLjboYy$Q;^bAGr_b1NAO3CBH8aWES$+IVpyo!tp5(qG=Mw86*yL-rQN>`e-%^*JFo!w z%H#-Y;+%eLWbF`U?h+QSsdMxU)g?9YP>LAY+x&8UpMu9Df7PqStFf}9o6 zgB)~Vz4p_eXu80$p?pJ4?QMsmP&G=&gOtb^j4jQ(cAWwPZiIssSk z@1j8TCIVNReB?qaNWT3xnwpsqY&VQz-*6N!-&~QjrcKR32Pa(DXev9YCBst9?cxTLm1p3&vg)r-nmCOt zpM7CMg*}AI6&FCB{HDByCt<@x=Mp99cDYPF$JBXVBQpG%dB~U`!XrOryL5YWadC0Z zGW#F^D&e8q^>#=(D`O5ODbqwS6iNW8a0?R^#X<7@K^k8fW8MtNEvFC^U2$P2Tp zR^v|pkb_GKwZ7MUnI3u2^P)rzgWjseYwU8pTk^p)kt%T|@R1TAXV*%g=F~U;cavGM zL#l!=mDV)KtHf!?1u~ZvyAP6Ird5f1Oz&*O!6+c|dk8s9KO zZ=O8bhb<-ljOJ4YA?Uq~Ok}Se{{BmExjY5m>VLN%HU5yD)NDK{>fMH&xWe+Q4erb# z!NAt!MR@+EdtF;Y0>UCd!!e-;BBS+bye=&!1p6)BP;dC(xzX9la968IGvj-T-slPZ z=Qlv@RHFxg#IJXUD`sJIPV~wV?e231WEd163i&|IHJToBBfGruH|Zy+H>D-UW^Vcv zR+EqvQ=Udi{}hE=8vL-DJ3xtaDYxSb7Ax_h=~~=0tg#F$D5kQ3PEb;a;e~yXa;m)~ z!6K%BUd7j?J`+d8Q~yuUBNdj_bh#V%mm-CU7owK^P4dI#zrpURKREV4j=Z8)Mv!bQ z)~PXzN_A-z2WkqyzZjEQDQ2f3fQGT#-<2 zNE9{zA5aVnATluciGCKRVe^Cgr zeVThld&q{z<5`0G5L99#8a!}I0#|TpLAx9bWkSlntKW4BVT6`iw^om7P$&fg-Qv!7 z^_!zhLltP>Kp%yaGY0JKPk71kmk>aIB@%v%Pik*R41`JuW?WKKSf$-QXgvQb^e9== z;S%aD_+m4c+zj+8y_a#!jZcH`V44o<3hH!>LVCOz+veMISASrU79~#S>;3sOo}&CF zj(15-%?$K)3OH-+d?{t#-rsD=@|~IO@wS61oI57x1`}Y)dQrodn5`_eRvbARY3+2a zDNHrLqD{_xYWR+S2D))b6fj5w)FFash#ZtA!3 z;2X|l?ZXCQLAbd2Tt?`fgj+*>ejJWuj^T8mX7`ig)pUykj~4qCR@bL)*MA9Fk4}OF z%vjs~3XoO?XLH6x+q!37Nf zl!e`uVQBeB$g0pA(4^?0VcSn1R#yx&j96eRwx5iZ8_K=>42T934r56XEC^E@E=yE{ z;s(;}fQIs^#$u?YIWYs_0|Qkhe8z=+PgcZ%%qzLC!|ip#kK#)LGyH4o^$ox^qUu~f zvlJ-1`nf+c0*&hmwCt5ZT}twN4CrG%|hf%E456RMN|hX+94Qt zwp8VVN5 zWUCjbNd?;y2%6u4@8TJ>)?7jvg0R#(K>{#|#!GA9(6wy~1@C1-;&awGOc)byP&&7; z5~+;eJM=v>56A(Re*0Wh4@TgCRiG)d)$K8o{>6{1zkJ{SJVQGz@I((Guws*+2Fm!R zpqw064A1PFB~a{KVsNTP?IP`(Ejnw8il|^e4C27PY8$wc4eRpxV4{B^BvqA?WA6kCJB_omJ#&mPlCNI>>?pP1B? zb|7@!c5(BXB7xkVHV2YfnrJz_oB)ir@ZE{Ur{Wb4=m+%M-Q*lN;iFL!Zx=m5Rl9V= z#Yb@(sr(khH8R+=xM;w%WDE$26H3Y=ByGG{(O6CrGhm2S^X8r;a0uR}kw*&*AxiRv z)xeCJY;NTjEXr~(V&;5C(~X{)YuCYSAPsWRV4xdjaX4Bj1mHl^3`@@>C8Jf&4CA^b=+>$lvRuywlWSBV%utyLxdI-2~`fxBb*f@cS0mH#b(t!v_ zamLu7Gv3$6&nWtWhA*-C%(Z!gMogmE4(u|PFXiG3M;YGlBOTQbwwzaV&FITh0bm2v zQ3p^iU9jEy+*a%c4L#259yYHWLD~9G{y8b)y_jjazBU=wu|);dNiQ1b>Kd6SXPRRX z*lDNV{4CRPf*HR`JW%K#*p$MbM{6deV#LPYLp(QWaaDNpjH zy|Kw1Wv-IkyX8G^FBWA;qH02UdVs}bctm7hV3Sz+EPvCBwY!D`DrPsPCt-@a^18#pL8TTT~n6z%YhT8m)^m5%pPJ5ky!di3V zz2RG#F*I$~WP06q+|#v$39rNO=bX$?1b;UfvPSu@F zs@Hq__ICSEi)>wS+QN!p!IJgck1b_o_Y*uWH*(;MB;*qb|n@Y)-zLps>kD# z_Wg}ca#zsssOQymGJd>|q>vXThHsgsarE&do=g`{b6;bNFCtD{&E|?~r+*<*5}=_D z_`+aLM2cCQ(LnDrX~`aNHBQ{W;X_8>#3h^kuuST6*|FyJ_w-8r`s(-0z)hCxnwe2y zWqxC$i~>vedWCD%ufbn+maAL8J_gy;AoU`otW7b|nVsOa>I7q5cjn^1tv__4#?-|3I8&0J*1 zt8Xg5gZFr|YSP|6Ddm12QYqF+r-G(%Z0WqMfr^ImT=|#m<3c?lr_h#LoSJ|}1{^Xx z#>h=2U#}s4C{Bs8v$4S!H+F?57MBZVz}l%X zoCi5d2}q@!KTE(pr*bLXL#)^7~-={);$hpPG$c%+L?W4zQ})eiHbAJ|&F zN$j5O#rGtJvp^*gwT(pE5~|`Y>t7_W>|^oS?eYy|s$h!)l@#MgI=?vXR&~dBn9Ym& zOQrec3^dx_ljXp?C4XU@T(}*q{2o4`&rf{k6S@-4mg=eV+UqH%37@B5F>`RnPflv-in#pqe;WCOV?MQ^v1z-nIP~2&(_cz@CqlX5 zo?NgXLLlu)U~{I(6Onc=ym$6C$$8t5Usb~iFVA#zjlYN*9+{#FQDLNE9zN5P0pGs* z$*R71_#5;+-RNq_91&vl!rn{%+GC;n+eK9{|Tmo89kfV~JiU zQt-W@tM@bqo})A*^Weh|Rnv_5Xz&UWTpopoFGsve6KZdgs-k9iq#4&8_u$u(>WnNjaQ*KW-5`~cx(aM*|CsX!_8m3G5Xto>7R}tyGvvK|BR_g>4Hd!ESJ5i|m zf8h|i;Au77;BTWQ+Dq)qg0AJN3t_?sYoV+<_=Z< zjK@dps}JU5pwm*sG#O<0Oz*6Uh#ZNj0{GUj6tgc-ZY@ixH9sgMs9sRLmg$Y2^xIMf zoYmuDB89kXsm8#___|M@&#B?4`*W2>7j@;ZmTL-jq3|3}+yY<VSepP!tmzgQ}I~nm)70wE~VE+c#Z2)yE zxNuLa?O*8FmR{iarx4;nZlhE(i-nB;zP%@&k?nhBW3ZDR#4p}Xit+{3)JV^sr z=wk>76LNLbQU9|)ah9DT(Gof)&F76Yb8rd~;r*OdoC&{Y@t7-E`FXQY@)eD_xq`CL-4mGNEKbh z&QdDG3Aoz%E)!;yfK1|p@fA5(HA2?I_S>^^$+(#RmT*Te0pHm9*W($g2RGc`k_k^M z&5s->pV&||F_UJCc`0}l&=3V}S8C3VRFP_)TVUoBA+cwNAWNh0(4;2(b%7lTh{{k# z{rCQdy|;?0E7;lvaS0B=-GfVz;O@a4g1fsr1W0h#V8McJ90F`KxVyW%OVD2A^tt!% ze(UGHWAKnL7JIL%`sS>u-;6|Eduu9+fT$reOlEyb$33Z5&(^`KCl0eKG}Sb`7Xqvypm(*vb+3zjh%SpR_ql!|9yBO+?GzeT1 zE_EL=sYSDpGqKB$`fVMh+mI9eub(C8nLdv64b#BHKObAMX<4H~kWG+BgRGTx8R0Z3 z$qcCmLH8|?1ZhyGrXmxELnCHX50&+?G3#VId)g)*5J)MDMUF;HZoZ7+FeZGhQPz0& z1%_EWO9}E*s7t8Vthd3^IP+z%kOPw_w)0|1*zLBk{$B_@2MQ$Lne;~(>PJD;pmsOt zFz@=>zBEXT4b=>8|MtmAELx=beH(9Mww&OEi9<+}44a%#O)17DP%X3iGO^@5m*_DN zHz3AXIB8G{lJVb_^g7iqvo76+~gKkX*=RDhrplU@zl;&UbY;4m+iL4$Zao> zsqiL$^jc}?|7hw;-)ecQ)Gv$@X~$ezTms} z6oyiAJ-a1-l?)#&u%R8O#C1pH`1NV-oFUw1>{o!&9>};gU>_{jY{w>MV)|zogMBLn zfAeTEId{W|>Jrrd1vgT+~s@isTT?#CgoyYnHmMn!1 z+&APKQbJ|5bK0rVmHo?SSYY+@B;;qZ?jWgp+eO5JV#^;EN)^t;SRM`=^=AksCaxrH zwz!X!Qhq#L7o-p`UuR~za>3#S(#~eZInL)wH7$Y%>~@GySH0qyeyBp zVB%qZ7|(|bg&)X8454x{DDu4pjTP{>e9?HduP;G)T%D)Qc{xy(k(EnGvZ7N-(W5zf zsZwA*ZU0af(6VUy$3DQk#rC;RBPfW^x7U#=a{hJKpQrlv>+)k4cye-@f|Hu_0i%~E zn=p+1Fc@xjIWUy1ZG(ld?jo{{W0qqu$OF|fJ`=| z1`H${^bcUIQbH)`v5cC73;D&SmXnDkDm$kP!8(9>VD`nRg2?kIFe+WBD3O-SPpX;! z`~ey>9zE|4Ty+=RdO4z5x6%0YI3!;jrs+1pjWZDZ$O5Yhc3eYpc7B%8zKK*{L=Q9f zc+Q!#@#^LjYLg@!{Ixz$+qWHJ;7b-R#Lsw6(0#xBFjmpb)cA6tae;PxPap2|cTo&v z)s5F_;ed$X`Fy z*7!M?G6a@H<%<0%vi5T?LxV{(>7ICQnpbComqC_An+$xi5p?VC59gi~S6!3gDH&0k zw&AXj`nvt^lu%p!g!g7bgKY6Y^Qf_G$eJ#Hp&AS-bK7ob-Ge5OV{4Q*rFw^!eRi#_ z{`>{+MUONGq#i|DSqT0^9$=ut&!Q@6Qv+vlg(!}CIBrJ~jT7A9&5))a9*+H>(rv?M zdr19U>O@2ZF<>>on^Rtm66-|CR}9x}jFMLBU}w*{Ca-woF~W01x_p6cd#vkQspaeE zw-bHVK5?Mc`iuTo0>FI^1|J)zb5R%>x-(7LXV1D#nHh++Afebz&KKA}5-`fLBn_r{ znOr}fTN_letk7@5y{Voma=={R`Z`bwD2y-R!pBuwD`{P|p~iSd0`;fQIbcxnMSJ6| z5AScGTSR1}V=GM&U<6*P;WDE{=70gvbv(MT7s~~GP2?w z4(EP_U~sgNl=&p# z>vhXcSKgvNj5M?ILoIysndy&pEsDbQm=k$)X& z;CE1e);B^?=UO|ed^aCxuOp1-W3F|_%Sj$`#jtFElWEhkVEph&*U+|Re`0^C64%3R zk@PY`faNSca2+D+Aa^R$_a9spsD#nVu{N%z0bO-VU3K0W%8N#i{Bc`W!V$-{Tsqn9 zDRC>IO+c@>G{a?%Xaya{roz)RwDmS?4!C!+v4}`NQc1>KSc`~$~6RiPCo{f8$VrMtSkx{^}vX{V=lM}uIBe&@MJ0uyom5^lxwT~LuIt@ z&dU8}<^VPkVB<(hU##*M^5z zjiWEM^AITX5B0kqHg!hLCna{?Lek__KhwNeaVG6MA=WPMn0OWK6yX?A+pa(!)(TQD ze&WSQ-CPB5Eib8aL(@mao7Df36ivvGIg7VNauW#5Q;b|dRU$xajCFsON*Z4pNG}5r zBt3;;gW}>L*dfZLVZR4eUurn0lQ|;NRie9{?RQYa03GrvM;zK1-!ZZ#tDvX!r6bFtH1&*> zdbXOHo65?|MEBis2}2dAKijCj@`C~5e;G%^Z^a&o&u^wN+dqK06V44B{?jKWKLM=&o@Ox?;ILnl8$$oDL;v?AaN`r`ET{g@BR&9*nAZ?jQg>v5LGNzyFD^9|{1zrosbJ{`)p> zA6*p&?A8By&Hrxb|Jlv|kJxmeQBu}pe!Xdw+Ww*thA8G~*>qnT@Zl!<=5i1Bm8>Pk z*W5ct@gF9a-U1Z241v!;g42;c=-0U=;`v>w#hRDOrQD^fF2LhfApdM*Wfva+eoO+0 zAqM0A432r$csx$J3F!Z zaY(=SsLjS5pZDAx=Z)Slz7P1~^ZFf*AOA(^B09NuOP-U0qcB2LE6FI!YNf-YnrWM$(pC4hv0kd&Ti?xV!Afc;NYQ2c)^%@f$hTE`5&+L;bNxJitBqU2#7kgRlSCt& zns{-KFDqhkU23b^_~&UULcQFU|8RRNj|&tb`^aEmZEYQ$>U$j8@oO8=nVC;a5{-mb zI~`&B5MlSw%)|bc70Tb=KOu!z^dQHNzue*&Xe~+!RJ8L+Y#y`tXEQ%yd~(|j=6{_G zzvT-GpIOFHp%;vf-<*>C8yJpKA+hpz)#Eg&a{v+NWGG9zL-*}ct)Lw9PYet}P}9&D zI&!E;Y$_08H@^M^pOsup4tYm0KCZ-K5p$ZV{gHOk(+!koVw3d-hQyAmI;wRPn8n3u zYZVX(!=N2P2A@Ymyn^?Q6=*yiqDJA%XCXSo3$_~To#0W2OQ3#9I2H!)&kQAwJ#ZOU z!3J?2c}Uy)$43$IDdSiBH~dO)iwo?-=oidk9+5|Jp35rxlaqrLOK9NUFZg!jhBMeu zLxN}>A8dze!v8|Ogq6j|PxB&N;T}%>oLZAo6r>qyC!`Dk+ zOtc$Zc6H1ISYNrV*DfnZhi3F^Hg%y=PHz=dt8a0xXyir!%3o3-TtT4r-wFU19yN3P zUwsy&C_YbW$O<0;51rl7^q-et_WB>1`Y^|JBX2UxqoN@VwJIWf1oVMY3JSg?u+xw5 z@s8*x1SZ1_yMYXv-1Cy9Ts;1tUji_YtiUCoaTY5{7RcwG)+?&`PGur)>-Wd)_QDQ0YS~Fsj-z;2+-+hyaWYmfKyx!X(9IGH8lvo`dY%?gZLMfWHCP!^Lc^eadH_=CP6 z5`od<|?zGMnjVGx@zKS=GdE zs5SqQz*9@BdS}g7Uj7}%8vss&6ysY4$`AnE4fdfv3Xv~W5D8-1kv^#1T{-sAkrGf& zCvTR&O{oFwgR#!fo|)PJExvy8 zhU-?hc4S3j7ILEv{gC|ndI|O^#gg{Kgv506jra+6(|gO`m!)Q#r0DCMMDxl+2uTbp z442?luRhrj|fv3l**)K5l=;8Fg%LQ?h{jCz-ww z*VfystJE@w=GrYV^J{AI`@yx>Gg{}(8H=YO< zaI&yeo2J;^AayeMhS#v2SP|I4t@NuXs`X2l&&`?!LNfR}`a*-cv#ktstKN;@Vijpe zcL5^xM`;dAYwVkhbMGEkAH8MYcuhJ};sS@zSzSR^i5A~LIX(w}S$l-+6xKvIj#hmN zrDh!%pz9^Q90DF5i>d7{U8e+{5C>{ZXDS|UA_E>Pe6JsBFTn;cLQ{{vn_0jc&pq1X z1OV{AqA&zyNdk+R8o&BOwVnYpYVHd_^W$ilD@c29v_EK!%L-KX`@&jKHnLKW33tZ- z#%*SDcF~&sG?Rc3M$)NwfD#I^YduNXx#t*`-DALQH+*Pm>{b29YLEH1#%$|z@DBke zt*2>}t|mIoK@@{#MJT9{>z7{+-Qf=X8v;uOr8S=^lHfcxuXz@2hzM}!-Z*pN<@;oz7Xce?KsIw;Z%aY7R;{2-5B;5GRkHOQ$#5CJ*a+XZ;Xz z`NHM*)G+%LCM7=bggsZ*PRKj6usr?2X8P|aiO>f(&KY+1Nsjcg7-D-M>r$pDNIxK^ zYPvDQU2@ybY!O2oH*@FPqWj3Jeq6-z%g1dh$fQ=_&F+rHmv#RN0|!j;YSz&Af&QzX ze9CEc-ofDNj1ua#aCx2MpVw-V3I$qBM&)`GZGO7#yE~XgAr;U+^*uL|5pol6&u%PS_FBL-y-dDJGUC>oR3u6OF9{}7%DSJg#|Gq2BNIb~ znLK*PDmi{T)_Tor978Zobi^CSm!B_ z9wDgG=f6MZMNGp2l*kVE?6(@%WraKIRkUucKU|*ddFs!T&}f-w6zz`#(6Th=NgUI2 z1F8W}zx~l9DP%Wu^ByK9Uss>zwiZ48_HkpaL$Lbl>+1&?UhhLLt=p(oEk@3^zsn1G zY1zLFa&L$6opEPvHVQQ>;w9tsP+#_MisjKX}nFj_(HUH|@eo$HHCU^#h6{cn)tctZ>58@msLEGDqN$oHFhGr|8+ZwRqM~#1>C8b zXl3xS+n(Drm>^_S3eyn^(%Uc#BCD7RQj^k=i&AdKXN9&q&j$s?rGEFOMSwS2Bh?mW zH#vRMwj{b5G2~j;tS2-$w_^2O9SGn|m2Ek6U}sgO{e05ER{%wqKjkW@eZ{@8!I{dj z4(`Y`1k1_E)xv~-9XxDlr*jax)BHOIuQ4=^VM68Bg)Ixsr|ac{>&iFVdfJxV_lGv4 z0J+zHzdJW6e&z7`uKju{F4ax{%-?Yiho`u%W*4?u>-~Os@#qgw4lNa$I>}S@!|&o} zO=S(n9xtBD!ln2#&fcl!zL&T_U5ut{rX7tDa}&T(x0T2hyK9!jca$N?^Eo+mx`{`9 zKG^Z3F3F`$I!3Zc?wnsCtt<;SrJt3ZGlBXZK6 zesN!)leSh&?J>uP$PRg5p6f5vdqvy6Y}@Vl5z`C3oCp!|DxlolG>7JGVCWEx*Z)38 z=%aPRd-Y_|z#q)k;&lZTo86YnIQP6i)E97yTA;n70IWN3Z)62EWF{tXRh$U=XZ;+o z^TveN)G*Mb9gpmat6*eEwypoO7Y-7%8$^mcIWKTyN>ob{;!pUNZQ#68vC^76-~Riz zjKl`%B&6a*HR9y*bV1pGCgc|ZY#8^>j*n4u*|!JHC}n~59Nacc%*fhLod_@DW=z6l zxDS{0ulHvpOX1DyJ)AiD`#K&0`e+1PqNH3Vt17=w8+>@xA-jnreGLu6;qe_rU)HT8 zj`U|rZL8Vhox9y41uA?1H>imk(%Nvvd4Ap!vkE0hk=uP=U#iF}=JW;-%iQ8+so2b) zZfR_TEJz6 zeCXWrFK+Umq;kWj^XCr4B@V~oePS(*hX}7F^jRsKJAXO_E~96zb~;8!in__z?z>J5 zrr_M=<&#%qu)CC^ju9V19~{T``{=fb{m_&;9gK+Zt?MtL z`ZzjN$$A!NUD0?i1%yrH;9A6T73=UvHUGJ2ddp!}`e=Yl<;?{;KgT(E#iBo+1uUm(Vqb4IuXf zn9o1BbYD~}mP4_dvo*ZbmC*}eWx6!mA6*b5zj2gS?PG9bp}jtUE-%*tSjL22+KWsK zlUBXY#1`eEq7g7#3<}%5cPCy`xU;=JpTS2nPI?L4-Gk)&7B&aA@vpYS+yAz-UN^1{ z2nDQdfZ@ipkw6+HZT_~As!gWEarUMqKN@Py6^c%0sHMI|Uz$S+0xZL+yW_3tFpmmQ z`l1l3TZ)2@G4J22?_=jK+m*}*p{goQLL@pZ6*`xfhKj#>8-@SrOqFK|Yps!H2>lE_ zK80FGmW)k>gaFsA=C5cH`a&jU6R0D~ExTh-?;(=C7%_=C0wY;hjI*ylmi_62QRmC$ zii6*GKHvneVZk|rJfi$jeK(n-D@BBr*D_^>!8_}@7;W7d0 zZ#KI=)@C~UP{q(akLdkC^XwZ_ijwonqkj~J_s4Lpz(a1%`9+7{^hC(aKc?R6QG=BU7zQz6IST469414C#Wd zT^3~T9x6va#jS71_2>+KeT)~@gwonU9zqVdXs|I|oDQ@ygs1#C zsS(ps)w``Y+b_0R(*X=I?+}RqpyKwP91-PI`_5XK8hU_S)$Qt{W%RvO;Xux~; z*^h5>PC81!NLdd+Tvx@RyUOpWPpL7%s{c*UP0r`9`#U+g__LM(S-tnz&}_Ji%$yuj z@h!N7C6(-V=?P_aQ%EdnBhi3L60}FN%&il1Hjskwih-HR@n;LaDuimB6e?NvE+fws z5vAWPxO;PpoE|U#5jV_siq|be7qaLYxXS z5>&pt#{$VHX%!6K@lG(LRV7jxGC?Mux%IZ&QTmu;6Ac$>^qsN47fReh<*RWdc`WbKhv4*9fnq_kBdC!i%=adx%pvbdf$lOCEe|5jh6M%$ zJV_f3ucB==Q>$nql}qIBr{lkS!v0vHo>*sGlvu3A`|kSOL>vWY9g_lI&MAGMpX{1z z15BQpIb6K2d#-OzE4IU+hYj{}zs%FqI@LE^00m{?*+4l)X#I%WSROKTt ztF^M>bez{;>TEu zMYfNoie#aTFS{EE4p%TCkOx@p&1{+31NPR)+U5Knt7W_x&SXA+VR-0Qh=WGMKV2V@ z(3s4fKKGVq91N)TGT!1B@YhlrOIn-Yz_9>Yo&s25_tIx_nfzgFc**Ge2lO`gF;(%(>Ma$5Q zh!R0LIg6E}Fo94kgf=!X;xlw`RSB8Rv;P`ilkf!h-c7L1t`8`wB z9=HBr8HpQwc=!F?_Y`F(lj}S6ZvoP;io=vL9d9~r5rL|V94P-YBw)$kF5gx&KS*}Z z{YZ$bgIm(dT28d}}@*qb;AU^{o*07griz`*kB z&5!h5+^SMfq9kz{aWoU1Y|w&OZXjsM{&zt7SJfoshhVcok%DaUk_O9NFv$`qQtJS% zm$3%NkOr|?%hQmY`B!rLqSPM~ z-fX5A($R{lB_PQ~+Z{l+Q`*Lt6-54hPc^y zhFa(rL4fI|H{h{`74AoIW!5$u4ukzFW2N_f_C^X$N=9eXvm%+e|FbGu@g5E4K9Jk6 zr6mGNO=d9y_clIX&{a6tsjJ$I*T=}#{W?N$B)VR$eu|SysB#7qZoE`GMU$lbne-WM}G{>m7OZG;m)P21$ZdByRJ(IzK!pmJ z5G^3VV{}|El>Ef9F3c1aS)FzIj#c*=!X|2|Ol{76+Xn^tdz`XW;;|dStQ?fRe8)Ig zLAiNusl>-^QHvrU5=Wt5jr-1194-W+UAV-dbPB2N^XCl4Zna7q98A0qfDl7d_iqVg zL$o5D*JXeBMDuNw7?Ve>ms;NS8}~-SX}vfQJZE1k>oc%BHVi)%Grb}f+2TeOs5t3#uS3=*$W%Ml{X zM&6_zRW>cRsPkuVKJo10P3B$y?#Lq$-@|jt?k7gv{-V}ACwi!_=XD&X!P@fl^9O@` zDo}`>B{XZm)WOc?6_Gvgll*8fe=ox>AMz+;t4o6?9wHbkuI^EMgRRp!vfl-vduAdC zfoA1My3mJMe5DMF1u=Uu6j3cDS-Q6*-}#Y=!L+vyk$G8#YlKW`MT^=ZLZQU0?qfk= zy@ZcB1_YnUJY$yk_YTVz6KFcQp7(zES!g9!hFrzJT85ys!sa8gCSDcesT9P}49SoX1*>&ls)YXn zK3y#peZSL3y;G@ZCOF`|T2ZBiF#>yUq*uvHY$|6KU<{kWV7~Gu%eyNCsi|prSc|#% zB&szgm$W;VSd}Gl-LOn^i&FR*Q+EbP7yE%PBgS!nDe1%Wa467=73AL&cN6= zk!m0ytvj5uom!P%Cz6`UG7LyfSL*95X6}d%$ynKzsSaaO(~(I$QLopG0OM|x>#8aL ztc{)3JUYYAQ7bKI(LL621l%o!>93vFW4iQWeMS1JQ`gx8f7bZM@jIH@OSRN=4`|7o_;k$ z1|35W>5VF+7f)sKUQmw0+ccn3Fq&pT*Ya_V3letWo)9U+lk)m1TT} zo-GT@%;H4oK<7{JQL3OxEJo?13n`ZQEM0!lph}w88F~={v%&f$uNwssnG?T!03&2S zo|Q0O4z{>{!*La|Vz13;lZi_gV-zDx zguV#qPDuW-_bWyvCy8I3r4KnItcMSQY-`BqhB$gU6yY{lgNu$r+W=m=yT_)JuQH^~ zTL!v;h1Yj%3&C)wTnxBEwf*%sMYqfr;~1#f_y-Q2ZLRGlN#ISs_wLS`XIZ%ax}@pK z$0H=zyXI;(hOTlw>-^z|iH*sm8LhQ{F5I%3)Bu!dTENE^QpRV<+zPFe(k*L zCvjX)ldj04BE3AXMoS2eUupBi=kFvN_A7=y@IY<;G)eyg19g25CJA1i!+r={x zWS=9PHH72JlM>L9AsA@eW(Ko9^9Ks%d>`g9{6n`}*i@p`9EZ9M8UufY1ZQY7<#}b7 zAu)l#XVIC5O-2E%j_do?NWPmxp~NHw`^nrA=AHPUIm@HJ^3GEQx><{WU>XYL-EP%Z-F#pHNUAt|G zK*ff6u@-5IqLBQg8cb?7}0Sh3L*->Vq&FnoYe9pp*Y;^UVCc*v>7g_M|TKdB8t z!%3mI=|XqMi=k~fD$Xi!Qj?HfNixQH*1t5DC6BQ;9vs>A%-dwGa(Lbe@n6Igl&Ejf zl$CgTFvqkpAL^=EuvZ>rksaq3XTI!WDE}c{B@x-u$ZLT zxfXm)gQ>Jpb!L7*;Tk^e=}s8T`UZb4vL=iWN+dq>$FNb?CT`?x1DJ?T%g}7>A-XWw zP^1D#v3%7!-yj|4A!g?C^$#4xPTE*~t~$zXAVwWlz9 z?f{4PEQncENMT3Ii;wu;roOFsugFygW7dTV%UZ;ixuITvUJ{bZvNk8 zI$nN5BO%_b>OvpQVnzazHtOkrsY{HujEk;~W)0BK?Wh7r6$ryGG+BFCJB9hOhULmFWsyE!+LSbnb zZM^#Q5lF;uAYoa9>r=kEzjmMGA)uPi}i$=G%B??FH`DZ<$3v)2P1zL;eHqA0)7L#Db)UikgyA0DY z;AzYbcD3f^+ugkAO{ocsrI z?}pV6s+R?3=6(9s64<0mLH_q&z#T}B69aeD@n*GDzNnfrIv0-oUi;~NkNv?XI!#gTDt$oAZs_X#O zxcYUiRihHi54h#OxMlEbva2Gpd}8(YW~TyAhC@JKzCqN<@r^oQZ0m-m_D>2k`HG0d zrsX4RJmIH9lc5MV1kcBP!aXeX;DqHB7l#jr3p>btr>LEb+)1`K&7Q(7-#k+RcfF-b zD(xyA^@Y>QFJC(C#Y;lMx6?i*J9|Qfb8A$1)l)wO*H-8%8$y*7<%6QG@3uXr{ zY^y6mfhcd`D#c{9t7aTrhi6YJaukB6^@iOJnqq)dM?;@c9-=gy(z*;cX5`W><7gQQ zKI1CiZ3?4|-bRXF$VGHHVw1Az0a-^B?5YbjS(SsqtUZrGO5V*;qPn4h18h4Fts1H} zBSP>3rS#%vH>USe*3!vm;1#kUYrTgMJ0K)XT@4JVq?J&#A^4@xmnc`TC#^&X0u31z zb+%TVohyIS_Fe&|A-I+0x)&kiTpUGP@<5v>4y3C0p8{I2v+m0@m5an{qoEHyF=vL)%6dt5>E$<_(!NH`Of8Tf!uvU#gQiHb zS3_-n^98e7w2tq=M6*C#DFLN~xFoiIZ2nq}0cAe>mt&?FVn_eNk7^N+XT9mLbA?9_ z*S>c3d$ON1Mg?7#^2*u#{N4%vlX+Fdjx|#fwDo(~1g6r$UwBGhg`LQTdX}X0r%3R` z>04~raEzMFm+6_y7KX?+`3}D62*WNAI@&1`I#buCU3U#;aZ|3{r$j~bTH0*v7p6Y* zuk%5aEg7-F;I+kJr|}ssr-zm`X2F%0#m0pvU^wPT~3k$iJ0nX%uhoVJ^n+>;V_VKCP}<*VvUCuSP-sOMrXy z{ojXyNj6Q7#ZlZWL{<|K3zF>ncOo4*CBD^K$r#)lal(3)!mI9p3zX7f^vaM-KJ+PS zNSSc{k;4$aH&zNQ5Teclkmls^T-FQqS9(HbCPqLXFKS#-zrs1vlUQ3o#d!#~M}^Xy zhn_LXv59VFs>U-Jq&X&LAuhFhcjFldx>E~8;;ftZLLrg;zFc=ccOiZL5eb=5@_2n~Y|ZA%6mv2Ozmr1kA|Y98*eJ$huKj_ zSO7U*ufeT$;1}<2x;V~oTydRyI zGn@ETsB>r>LZ77%d&v^uy^GZ-kGRVi2*?WKWeg9Cy^6cR)8{xR!`pK&93QofP>Gi# z{^VT@NPR7@xC_Dt!jC|1$Yda+85krn8%d8|R#dISvO!Rq5R5zb{-EEr+_G!aA7)+e zQ)#fa!h4dbu%+W9Fa7j3J8 zwD-+o-~DpY((of7A^+1;}^g#W&$U9qirs@X_qd}?s^Dzl)iM4_q)QttH)`L7GV597aW z%qP8+h=>2#UmkmDd(q&;jv<0+EmimAcQz9(GemZ@0K!l(`xW6H$?#v{B|fl}UDjpb zSHNXH^Y~DK@1tTA_8unSuclGh?_$ABL%sT2 z@l!$f@x7R}NRrGAA420Ezl@9mxCBLZ(;Z7KhgAEFbVL?J*%+(m2}j*w!R!W9x^%_ z1)KdbP5Kdm5~n?+Kg+g)7KbY8Yz~OZova5ff9-Jw!hK8@JIOwq3Rsd1+mQVvhTkWw z1A|Q=CWC%=?;Y*SoA8n9;QLass^wbs#HJN`eA|oWGg?-bJIBhK&@6)%e$fdlKvA?m80lq z%knm>>uu^LJ5Cj#R9RwjZT7Pj&FeNu&*XOu2}fCV0q# zKGBqKTYsgl=`CF(hf~q(YjET&pji1=1s^h`b^=b8yX8Qogk{_LiDyUCLdb2ce|l2J z0RJt8BTE$*D!m%vTfA{uyCkEaLynpKRBe`YV`v&?%6|&0qx;%Jgx`DWu%zgr`$B|3 z2j4_!vFbc#7FXu-8pKIqT8UN?@h)xQd&|N*L0>I|+P!fZfFlqX4G+?V?bTy!1sav@ z(=R37a++SWXp8SqIA*`L`)`gs-!gllqqew(;v*f0&5~o|&b&W-uS;nxh3IT@*H~BI zh-fTT1vS6lr1sl+-Zpk-;Vw1ai3mk2fmWK!geI z8alKj>BeGqfBP2U-pC0JD$d97@Et}yUgNKI%R9@aM4j)Xsmp^z+MF+vwDld;#i8No zblvA-yfmXeS1-H~ynq};)SZllWG>n5aUFbSN=z(9-G}*Q)A;T^ibido)~BGgp%tge zz?0EGTMdSTSW^$IIvX+0UVr^Kn)x7 zL82BtT!sDY6~*t!#H3lC1|X#w-BXhm?k*YHz+sr2D?*wT<0(QzTDQImg`p@|d(JP- z)%}1Syiqn)SvznoWUlPNPzUQ zr_0XOk1b|?4nKm?0q+=t723CLxk~Qm@=>#}Z*#@x)zDabJ7LjSJN-8FMSXo&iaZcS zC76(AbrO^uzTEoAJmm|#PI}&cExA?!L^XMQXMBz6@ViWDAu_bDv^XT%pD4(*&u2w} z$_z;o4}&=Pj-?3oYKrM_C^&|qU<{Gdekg}wg5+TV?E{}Qr;hmT3O-zZF0CYpNUYDq z{8`)tvl~YZcMr8TRJ>G@ua2%hQ%GE7=VA(EgGLQJD?|z^FJ_KUd1=QnQg)S1u@VTVr2tZ-!0iB2md~ zVbb=ObZrG2Rzj>nI7n<@K1&9U=Q@SloONSxb_b6=-4}d6binNErIwVY;-XRjoL*UY zqn^+n8?PLuj4%#8wnK9Q)OY0o(WAAG9Gu;Rz3!|CS~-)|;;+ix26F)c2t@`QxQ-F+q)sP)5rE?qC6R)Ep+7u6dSY#VT(lr`G+}vx&oz zp@fZQvC9{RlK`482Bg`ZQUe_*@HVcXmYC@6Q}Nwea-jFH)vF$k$yV+ZS{xPP3&BQcACovH>*sH zJjg7J?3kXJbA2-)LjXG?7+nqz#TJ6`vke-=(`ItTP<0Jj#8rS5=wl4tfy)Hira*3% zSQzgi(;=?Sn$Dx10<@@ylHV=feV+tV7}^5nXXP{p$N{O12yN#S7&6$UE5u;N3~)Od zU?Y4Ld#KK&S+U;ASOM`4V~QYU7Hz}yV+?J2!!(;ScF0;gcxcqG zz(j@9#qnGxr|qv8o*hLprMSGF90_Ey97v6|UQGoEU=+`M+8=Q@3F!^KKUlZN_TdhB z6{@obSN|0KSQ%aOVo|oiSw)U;dmA-3m zFh!q#P*9suC#2?Bge-c4HapeV61=wd?%G1a+d@dg%{l$rq~)iyRot@|$rb&dJmlQn zhWscfC<=fK^Wx&-a?Nw)xTTL@%Hg8T&dp#dtYox9A_yi0R#r|tvhHhrlW3zr4N3@f z*kI{7A@dUV|3%YTM#a@MT{{c}3GVLh5Zp-!?(Xiv9fEsscMtCF?(Xg`!QJIMxu5S{ zi+>C=J#?S0?y6na=8Ma1%ZK)A2#$df9VqTW%@MWl*BK?YRwc#FU(k-=poww7E$N71W!o%i~!BuGL$RIH)N2AvHZ1q+Mn z%8+mI1f3We<7O)|17@o>z@m>Jzq(x>w-PJ|YzOt6)T-KQr}X`Ei8oyX<+NwVZREXqM4=iD(C zp(#*RpvtavWXxM$+`qV~N`i`?wlk4>Z>M{RY};o#%-AIh5HT_q#TmIA+)Godt?S!r zhcYsC6DWZrs99n3zNkh6rB|sC=)2Q`%Nemu|JWTE(6xMFt@c`vG~1w6CDgP{*w+JUamTH-Ae z6h5%VMr&r+=o}(IC@d=)u6Gt*05fyKj8Gp)aM7|bTf%`tDN?!yL!%?;`48#>PVS=z z@Q-ybw%5Ranm8wGt4#fndkn2!94QFD<945lokK_2A^bX3&^mUNu8FlJH^-_x8q?-a zBe19WE5e}68_wlzJkBQRUn;%+>I*BB{ryPvkv?c00T#xU7OczzbC0lZ3}NFJSk)$( z9uz1JQd)VFw!236ZDOTVZ{Y|hp%%XZX2w-tSXP$$w(H)IEJs^7)|HH0WMe$qN3J~` zx2?P)UASwd!ay0bjF;7WkHi9T|D9}09-;(?_OIt%5wfwVSl1H-5qa0o#I)H(NiUDg z^=oHRIZNS{`kE&P+0z~7Q9c7>kzaZ*cJME%kE4;Wn)OsTPjWGsD6})Ku~%QjAz~8@ z-wrMXMQ|Q=zitx~{P;wmrdg4>`^{_#=x_A}L+UbMN#6fa2IJ*&rd+i(bUl~gSC^p{ zUX<6cq*G|G*3PQ;;@iSiqmElX1jY5kTy>2+&4*1}c%`hRKqF;{Y=1#-w}U_;TSB^3 z{^K~uf7Ck0Ck{qR6>LUqHW&-X9m8AKyMFD|J^{ftdZyp2#5DTMjo*Dy8oWUu0G&5kTa>zvJ9Q8W5Z+J`h>nxxVmB9)%P_7m8HJ$QXUMPs&met&ekji%tR5{_wy6k z>g}3HMJkYVU)sq22f;=Jsb!$bp-Z<+}C`A28BZ-8FiE{4}e> zgO7npvH5}3Q@F19l8zH@<0TQ2@&$mU3A=ZjiL%d`W+!H zY9Oq{ReaQxeRVq=ycoLcY}5Hu>Cy(KnXFSZ>3u?IlVB%_-J}t#P6D;TPegtw*$3>S z|G-6KJaZ+nBkEfc`ILofYEsE$1r*1dftg;Yzqlbi?a4)P3}k~u@`qCXC?c@wb?e(- zE6F35)=*j$-Va5F<424$XyTc4;Xrh}UoRRj)#(pNo_&+pOk}wDIQ4XUT^Zb;?h!pZ z2q|YMSrBoYSt7%!DgYW*@^bROoxGv+pfXxHcAFY68?U(H^O{%f;RA?uQ7x`NoAiND zCJ&OWx2dmWa5 zfdeG8g_VX?ain<`7sY$U?bOMk1+2H*KQhYOp0X0$f=$2oXGf2?QGX=gl$0o$ZKLp_n=gQJVLv-*r>p>Vb{t?j?p=qko%470W3c$ z;oC9k^2@Q#(^Gerq^)b!w9%WH7aY4_=suOGMq$wHwLw+rH($%Y9_B%US;t3Toh}`L zJ~JLL3;{tE5>+|`4n22&JuN-ooTT%_DjNr(+WXYZb!U}h+GFD}KjyE8v<_s(8WM#i z6Bk^JX&)`>*D`zJirR6R-;>|=m+j}AEG^9AazFTKat|XCm-MT@I&Z7JeRdy@s*H&qk`xhnFZMW7&TzTl6_=1c zX6;M%ionLmt;~h@UMA}Qi!XVM|LY=JYt38Ub{#Hr$u2IziAgfj%Teje1R8XO^pTaR zw{gP{3TK((01JzY6Zd?ocCk@xgW$oDa81*pA@z*P%J2+qd;4{9YEEq;Fk-N0D6mBq z$1RcWJNL^B|9|8qA{!rekD)x`uNM|K45i4H=9V$qjAtlyxVi_A=~5lmHDpne>?Ml? z7_MoB>#WlE?H#i2RhZuHb{sMV;BAjEBz~H!8stkKGxhsYc#$3NS8uRx()Tfz#WZEH z0rD`hEkw$?ijYu8qKIuyf)(%E^9nJV#Dn}?1vX5TPupz2FwN?(mU~R0W3!N1M2EsP zm8O2DW}cU-JulZD+<**EZwF>U%YtT(MN&F!&9kPHZG1@<5BtwnI!Qdd0*~k4u3#_Y zS~EK&{l5D<#`%-_-97veQugO{nRIx5=~?q}3iNVv3jId#d-JWnQyXgQ zf32-tJF45g;y?@+okOE?Y#$CBgSVh~zB@X}`6ZD4-(6DtOXn@F--C9}?Hl|~HU>Kf zJ8S_72?{B<^x5v<%ZJHZRtJ(D2j^{QPR&Qe+x1|e_ng4F$W3G=AA@)D2NYNBQ}}yM zW}SU(=7Q_C1tYYhA5~FlL7s>}WocR5$g708xvf+Mo2LTDBVs0ns_X&+IMOcXehYOf zRe0?}feKg0fh%QCQ>#Grc>uwKj7rp-@^#Dh!$%sU#e*Iw#Tk8 zzcf;;ify}X`>hP{>s!n$jMpaY{hjWx>mE-=vGZvyY<|P0#38|z%i~5}^twf=)vRTP zN`c!JehkR3a64}{=SilW*JYW$U0%r*?z=BGLj#oT5}Foi>~p(8gTpz92*Z*O?PO;#r5hP1k!L~^Fx&!`?gLu1xpyzj?;IXj9>$<;M zc{C}eC>=LYe-Yt%z4OK>G2Kk!$@1ABOQ*x-<3`(;c|)`Mw$^m~@H0p0)XG8DL{dDh z0>tdt$$OJb!>g5KqlwzgInX~|{+(mVCk=7TvcA#zBeGkX&sr|rN8N*x-}7+xIf!sG z$MtGy3sq&r3F$fQf@bS-m`;!*W_tsmy)z%@OdmXYe;w}Sh-5%S6`xh@_ zrQ!JyAl{QfOl@XONQBzYMIBq-Nl!fO0>^zByQD7!%x-|0RkOSo>!aw3JNw7U&Hl#W z`Fv1sW#OB~7Pgx02S}9JIqgOe9QSB(8T|ZbL@*^K6P?@u!?D;6^UrfCn2;~i1OyN` zYMR!M`|v9)w__QTL+PytL$~v@PUz5)e#jw&Ej;h`!4QE?u4GkjPwrVsQC@E`ypIF^ zD`5~(OkUxg_dAOaL8K!)HuQ;|>yI2?xLiUtJ?29JVQ3S|2e{3?lBMe9ryi2!rc)6U^Meiqf`;Oh{_ zEo(3sJ7lB7Dc-+&ywH;w&ql=}tFezKV^_I7Ia$>giF!EiZWTuAJpPjGR zC<lxl zz(RW4dT3${M%j3|sS!9i(L07$B)9&N?yP0y{j=5MA@1TB`%^ByC*Pm(S(}s!o8)M) zzxH2iffePM(VKbkZ3Gh#@~NYz#A0pSxODJM+RR0uw^nIjvNn$nRyb0JY?E6|o0Vq} z>X#8bJs`gY7cbXpPDmJk%YXmW<^LrY<8u&ATMcYy^^{SY0Z&1Q&1)8JK1t#C%{#-@ z*bBOtV1BP5;5WGZR9^31D}n}zjr(R`mlI75d^h~ zcwQP1L-#NNjm-+sM}tAy(>ctczoVvBHlB3Tq8g?9AFnDaPnY993881vdgu!g=%Kl$ zV0ar=caeH}sfxN^5ghvlVIjBDwWD^S;e>X=(ivW^;B6ZNO9qf%$R&CHTu z#Zhk64eQ!3|KbdvXtc`84C_B$>Fdwy%Duu~Z<z*4sLP)ejb!Q0N&1@>7W582}Af{tzq#GbohmXK>!sa*^<| zcV2j&sGIScK$cH#K8OxyA_<#kK&7qd`X_?Xd)N5wVaA_vSit6`koW5~GE4|UOxF3c zIi+Z7{$b6a7xlVH4v%gMtO+&|bc(&F3x6bp_-3 z_=O|tV$2J7`ORtir+XCoz5zsjkR!us^HhkcgXdt>mjb&hW9UlkjuhduMp;^}S1U?Q$iR z`{o?7akZM0KcmpeXsKrvjUMZ^p9ep2Ysc4w8UGqVfwW!p+l>J2qUyFJ?*xAfRbu~a z3Y&K+wK*#`6cT1{GKzgB%zWHS$_q<%$=+=d1fn;P^_;c27C;3v6d(N5`DvEjl+W(y z&bzu-kGaC>hQD_fvKLWD`eughv+`B5!PVW}d@R<@+a(^$=XU86jhp4#+Z#|NjrGB= z=T~fwC=uLpE0;wt+`Fb#(%T9|8xOV7o9u3Z_P=!tZ6rt%%S6| zGbOrZ$4NrNwp!hGdSfj+MnSvBAU_X&|9)CL2z?4>vX!Y9914WO=22LuD`2C8}5Cn&V2D{DSh zB{nnNN|!!5clD)?WV`!%LYj~Jo!;uAa62;Ckd{Ky!S`_A-ScjaG^Z+dgrc-xN?pDx+IO@Zo225=zFYVf+a|zgV9dVCCB9j9zOiomPCsMiGAK$_Kkt5{ApoWO5Gmu^giI=aV~dB3K1o?h{8fv zFnb97%i-VNq5MsY>Jj9k)IH8oFijdq>SS$@zKFdjFkzu};%cp9OTSwidf}Gq*j{k? z)WvEoyO>)&K9z>+v^AYNnVFx3SvQ}&!!CfC>7p$-xZ><`=Cb&_e#XX|6U2cAxnB%l z2*GUzaZLPp7Q+|im@0g{(&@_?zvWRgXE%S>#@8+~!i6@3G!~X0$$Dd$e#OhqD&zO+ zpO?+OSqw?{jNrB%%RFklo+4|s+ALkediS$%P-bb(M>yaxp;5Q^rnno)ccVYnkB3Z3 zmN*pGc&aIKKB1sIlY-W}tws^u%F1d5HkgrYFI#8|_K$ey(>X`Kz?MSNW&WfA7g*k? z+0Qek06Kw@26vP5@%Ss%sHCYPIj*a}F!*3$EJV;ShV5IvpKFe~h1X?5KNXjMEYVOd zW}zJw{AzU{a54!21SI&b-?q;J<;`^Tw6d`j?~;)PFToxCf>e-nZ*4w~tje_mi3#|~ zG@zpNbX}dtVG9h9<)F<$UQ}CFZ^sV0^R(dDFT`R6l_&y#dTU{AkXU~od=`NW2i*|< zbinq1>hRb|`E*Tu?@aHsG3#7;nH!dk{o@0-|KIFbX`P7hgo|1ZvPE22{+k_J=^c%Z zzZ@J+hQBAl3jE-yqvfFp%AZcvpWV8p#Llm@%3*>j7uw>z&aK!R)K||d-q?@r3Ptm( z1B@VIDM;mD&w>w8A*oG5vi}Be5H6CW|7cGDrmD(1Hh=1ioj{OZl@U1#x-k87so zd^da%P*CJ&)5!i)&<*_M$zTDz1~9zOL;6Ft0U_bXg5wHcLvG2h8_ufwn@D(XGoX}G zZSC_9LsI3zp=EsYV5WX9qY#sfMllSn=l`DtzJx+AtB+cqUwQeY*3KZNJ-==xb~`<6g4p;aVWA> zf&*FMYz33;c)JK719#VVC30?W8nrAvB3uU;xYGkfUlFneg#v$T-uV5k6kn8?Nwd{P z?6hx66d+3TaqFHFxK)dW$<}*D;ijW2x73MNfX#2Tz%*|{`4Dh71tVB=I(z9%PMMCi zOi4*04#Bsa(Mw}D7E8Owu(r|Fy?A^6<(_6xA~9V%hYf-pid;qjqA&BI38Qx8AQ!hPd_4PJrg4Y5jDMmo+(#inI}l zP>Ot!x7Ls_9#U{{^w1gan1H`#P$lLZmMPt1yqDPa*Cp?;IdhU+OPDvSoh4u>sps(A zLuQ@rWO?&;jkSWj(>?LMW5|T$>J;DOZ6FDhBI~891lR!CpV?K}KVbFP$_DxeT%JAd zBGlH5CUGhj1q_#(SL6=ou$nw!y}pi2UInUlW*fW?mg6lRnyOCP{o<}5^J+L0Xdb;g zA|&jugR+iPKX>z-JXDZB-4msZVP`3Sv9_ALImm^~>7f zMx3gbm!3k!elD}s%i*5T;qA=rG^lk(r1tk=S!dor&8mjAcyel+{+`o>FCKcXbyp(OT*NGxnDnw{IJmJAu&=`q%jq z8_xRBm3$Q9tUMTH!3RbhP_1o`K%4s6quleHUJq(g1AVG2Z)K4t^R59Ai((~Go z<$3tA0jo90<9logUOP|g?>)}@;|ZF1BuN33VyOOFykza|J@zu}i)~ls97-#6S6d;X z->z5A-Ho#hW>={aEtZ}si3Tj(3k`dJ{yJQC9C*?7%F2tK#@+M~+W&}W!h#WsnN(?B zCCJy70cJ@q3!IwTnEdtnHB8TKk$h&0^v8T>bflq%&86LgT9&v@CP% zepvq6TQvSH->TmJIaknhrX;71t9{e;GvnLAjONbnepve7i~HYhq}D33F9v!)!MWnc z0yS@66*gw=$lX@lx%2z%$kBPcn=XQnZ92_m9HMIeo;I^h7N0UbyzTHiXpa%FaLo*q z+jlh1lIc0h!D*~&W)MJG#EKHceUD>ibAyaLAe>=$2`rgu|o8PE)!p8aR9E*%g{?fxj_US8fHmIm}s zR=Vln$4N6~J>8fp>3kPAdOc{|RO6c%F#X_ zxs~doG;Ch&^31ubY&6zLs@hkEiYm<*h{yRs)krSZcge@>NrqXWYFf z1zgrBzV7OJgh#z!h)Uo+)sVZ0^obKUCPDp-YP*DH|1e?63n1O@l?Z@ zcD%=OGB#+c^h@x`l9lZ#mhQ=KpG7nsa_0=ML`Zxl9e%Ir>o{W+UxvXBQXHp7a~LJ= zYA(!3RpBR^?R>g~s9+3`<$xj3+I0LaH*AD|xoc91x?v8txWT#DlVA5S%w)rjmS)UWG$(Eww_5xV`@LE_>ix4*NfWUwD%SPlG=H*( zDq2AW%u-RY%0dc;A=C6C{;VOmXp;%e(h&tGQc*j`v3`j5i+?J|X$v|Qcn=7M`n6fk z`;W0*G#pa&Uohq|3|{ufUG=U>RW4fI9xu5)Tp#)J#$_B*-y2*%!Y*s7#TBZDU+l8K zXdJ{R@ZbVwA3txRUai$|R4xBrWU_TKqPWv`dXo$acH;Uctm64^4DPx7J;#37j>NY3 zm4$PCG7|ZvP4SK5wf9w-0$LI(Ug7*Y|6`{ijbm6s>pGS)sc3DEZ&3!FZt6?*P?9yU z#m_1VDC`x?ga{4C2Wy@;h3t;Rd*&DJ zN`#x3*9_{-R&o>{7JTeobmp0%s_#WAO}cJ$R4{APMzTKHQ~d4w#XTc+_~ITfm*SHs z$cy!C=DiN-X+4`Cm2kyPYJ8tPSTz`U-S2g>FH6L=XM@{5a6CDbGh&1V?pzeQa{Nn` zU5}@psJco0TnT!H88yeHd zqd*^a@YB+wZvgfLsknbYqJ&4G$fyywu08*6*4KhV!=%b^JiQ z%;>jSdmI1{4A#stx42F$34uE4Y;#elWh&umK8!UZdOtTL;9^q{s`YCRX$#*c5L>cJ zT(Z*X{@FY{ENzR4*(x)zOF!0uq!lR|KZ@K*n`h_jD;N|x7&tOhLbR2&4SVz)8sJLW zl>jY6?8>?Uiwr|o!7j)wFa|-V-i8VcUM-_T zJ%c(?2~PrER!By;fu(1+X?PP?{zqQM zMZTP4k85ygST$pPK}5wy3beaIwQbw_y?(AY?Nk53jbjfve1d_D6+*k81WzpxNCBpa z9E}%_kB{&Cg4B?Zju#9}iWAIGhCKE9&pdX1FBFicpDo!(w02(sm<4881+$sSEtOX3 zh=ZC8$6~e!y;g)zpS^ggb%`_Ee9=GP@;7L#X&>|y)4U0hZ}V(ub4&belz<|#%`JoY zplZlb>^X8Y|2|CLE=VUm4gkdyRhz!^zdLWrTSQ2mN&+QS9SCk79%9IxX$>t~-o8Xm z`?KtEA);{6Ub9kDJvZ`@*1jGIBKq6XWqU@(MCf52#$ z3MeM!u+b)T1uPUp0flPl>>VU7I*W)h@kr^I9p8QtGQiOl+|esdrJ+9^w+2eVWO|yQ z3`ll8@HTKJkx_VeASJ`vz_tPh2S9>z9LIghpiJytX&Bx2gIP&p9zH8>ak{y}9=12&Fvxt4Q5LxVxUNEL7KBve~f z{HF-$4G9Un0l+EI?hnP*yy@-yGkNDhMgmtE8OEs}_DU5GQMAoJ1$whlr+nCsTf2Sj zMWOnMC^#_=LuGkvRb+%g&%{K`RWmd=w6~;aB(euB_HP7n z?CKnVRZ92Kvw!$J3JRcL{psB`s;#U3Qw-D&Ek@xdxJjhM_7$N46Q=_JL)(7xEoh(+ za@UjoJuoQy!`iqZG12xY&APX+7z{{tfXs!PqWHkTOuIPIZ%aDlY(@Du~Tw4NV+QQ zNI!F14ZRJ=%B+6$5${$7)U&2jo|CTwkAH+~R!NP=T@jzWp45sPhgyg*Ox4-%bLWRS zVz zwWzct{>;?j4Y}WkPul~f9KSpoZy3BD@SPjO;Q|y25DVkvH^f^L-bjJ42Wpbj*37kpZ7(t!H{f!%<2ANKgDX8EvVdVga zFHC%y71Zt9?do`EdX@ju8T5LxczL_&75PT_5Iar;^Bauzj+X{!jRWCzuYY%ldT3x> z@*I~;SO*9ZfK_>F0S?4V5LQFlnB`Op{K036Bd)Og^kiimKQdH zoLc#N!tH9K@sbQ?tGTcM#Q7!!8I)6UgZpNJhR7WB5I?qfDch0=xO3Is+9P*}?r`}~ zgE|8Thsa;1dCxY_DwFMX*JGAVA$L(8ua}Y8r6w1?J}Uj%bRSp!^?C5t-63{X)`i{s z&lYr01DU`sAw)q01c)sAdquX~@KAA+L7bNLx%BMS z%INcWRtu7V>y7{tE_$#)`yKDb$|Lu2T8ncFw+_kf7#1#G@IWHuT>Tu5s2Q%<^VNAh zG#qr#&R2!Oh6_An$(PiBWbomqVHL!AE1)A*-%J z28-b|?rj;!J5~efcW=AyWHN4FwPtnoufaafZ@%M-P7*qoAwj0m(=ciDTWME1Wz~$- zfL#tP3u!=)XkXs9klRIct3>a_`cy-U4aL50sK>U@h?iFuSa*h?8uK~y_DTw=SOrOThv5z!3tV_Li;!w>4&S^gwBMq~O50cG zh8_k|AfS(>$z~b+@qkqJ+YUZCyYp%e{n3$8ef|b5jXuQU$yzV?Xq|eyYF4FqS2cvh zafJ}SgI2wpn=3IJuXG@pMk(okZ~P$*K|vuiz4r0hy19Cv0*T?;K`(UXvu44=5&^1 z&s)N6x7%Sw#ih>l z0FE5ZTsIcAH};IF<~;}vT!p<%d?_G50une1 zE>-G+;_B4YLUge$`5V2U4#f*mp=a!U_#IzzA4dv1eRpa|_9NEaJsL-GY{9`~p{0KS zkR!>&1ml%5UzMyw@ve6WGEWu2KOLo>arAtOEL`w=ZCP_IusrcMuo0%1hd8iE4MleA z5-}U5C|*?u8dtc}wlfaI`tS@fs)hMG@vRFi-80OZ z+|^(5%773^y{9yV#0XhUmPN z#!21W8g#MKSR)#>&KJ_UhUP^~%&Q&@Gngjr;*2I&xaL5vCih{_iaZP_Vz6H1Xn1Mr z6)Pu~pC}CSW&Lj&IH|&jWjyUXr)8yCNrV(;{qVp!rj{E&C_{Mn(wPW}FvEwE@Co5u z+gac~Ybi{({T5bhq}41q<(A9HUX7;3`bM3gHeXh{B`>y4{lulb|9^Y#A5(d1T$mVp@|@{{UiU84QTR0=pld0Tu!f~L;X`~Zg@xz$30e_FNCf<5r+mfIG2E|e)A zFzQkoa+US!{?CJ@aDX6B27CcU;?xk`H*d;F%!4QlS1oXyoI47ipJ4OjHHyjV6z=sV zj^nDZF-z3kI7ITuL?*v=$96ZL*35(XlK|! zfynE@zOKQjue8~P8Ap>zV<;nmT5&unPf)~4GW%!|t!T4$jG(g{Mh~b585?Uq^ccP< zP6!0{XC65=DkD%3q@__~a zw98f1OckCr_=*MTofp^VqB%y9NdGt67k8;Nr`eM0&oG?9g6^g~fP!&t;e}?d-#NPd zv%;KSHV|`(zViy0fns9>cOd$I8#~B?YTm<4>G~f89!P%#{MdTp@QQL_?+h$Rq(}mb zMd6nk)lRkBoLMP#y)G(Jk}Z$ik0GnRcVE9HS`-t--s=Ut~Gv=zLXK)77u7O zBryo?MnSrm+siYj3D5ph?NQZx$&E>bgpCo%=yhEB@5o|Dae|iVb#=(;_O}Vv43b`T zmnnacjov0VCN*zdqw$2aVEpUMA9QHgYb5yc4yrTXn2M@@*89Wel)D8yBO@b7nU08 zT7QMEw_}&8sT8?M&7NQz3^|%r;e3MxM4FRNEU0YwF#!*#Y2Rh+eBQ6EB);IKQ*nkC zurG#qI)36#$|NCm<9t86A}pDcKB?DSo@~3v-Alr9;P8%3lo=hJg;lMG*b^}W7A_Of zZ}ZylDz*6OeIQD)m5giFT&2L5V8vSQl~^7Dm0WnW4o$r8dI~#Au0OrEqH4b<3Ay4L zqX0R%vUK;>uSURW>vBxM%fFt7Ur@5N`nu$o`!AtzHF3d;hk-ivU%JF2ht?1bNjg3O zB+VA*stzag#eF5IN4{)>L4$Zwg700EQ{Q3iM=HB!zS<#R#LRNV2wEE9bzDs~UKOKf zsi3!c-OVuC@*k$7lGk`0*A2k|*^KOp{DtQ*eAj-c)eYknI;iUdB><5LHq1?w*7&u) zH9odJusqBsZ??p|k67OF5>T))qvb zx83jZ5$;{)j$IG`%nmJmgjye=1-1cpESdT@d4oV!i?_i3YNLIRzdc06O{|L^MvI}f zh@CbfRR2+Ml0&e_8>_^;>;6a2;RFSd&eFgFWI8?nzNI{8rwmL0M9@|>TU2SCrkilH z>$8o(pzrL9Bh49nHSZSKRS%%uZMKsWog}kbc-FRH_FR)98_Yiv`x^QkbS^IFysX>g zg^@Cys|4-quE133RHRFVqlu%Wqjkgg2~eez^zJ$nBOReagaq@szuo#=(K{pi_~Qh1 z9o;gQeCip4sl7TsXuJY$3|qg1t1I@CAoc)xXA4{$GoY?E?9OO05!)8)(uUTeZVgYo z#|V$aJ!GKKEGu-?%NOS^@4$gy>KM9gT8ADgM;Vr~Ws?2SVQc-i)rEn4d3V*H5q93K z80Cow*fl+#jmjF%cEBUq{2@r|vl54Q2)SEOG+d$MD~ALufr_fY?DoVqRAKlGT2ln0 zZJ6rPG)9PRZ&Gk8&AK-9x=j(tnB13mmDp9bzK2w^A9Erh7ao5jqGre5^t#=wGMQdq zax>!by18bwBNr$YMSSB$XFQ!RS@W{-3@kmuytz2e z*4E_EcTvi%qG>Kt{v*Zx*@kgPL6V(sdJN%Ica?f*Z8#hDHPn>p>g@%GEnw(*5cnYN z`_Z=*Wy2TTDBuPTFAblFxZ{)AOERl0AG9XnnKuUge`6Gl%Yk_OovXKdg-9;@+I0s^&tLHv%!(T_ z-(M!}UVG%%&Hf(l2haUxHA>L2Z;~=7yGN+hTcHbP1g9kP^@2%f9rNqgs;s9H&r_X^PcUmMa{;~AWATLftMxMHn!bqma;zYMwck_*u zrew4nyT{s-yrTS14xYm1n!L2c75~G^h3)Oy_Bpkz4(~*r`a0`l07Sa1V2O3egYtK% zy-^0uO$bmuk{=|!g}iPAq@}Kr>l!hHQSYeQnP!pFQOO16LO*ejElMx)<-^@zVWKzA zrpF_-(xQsc7otF~+jJQS?d~9sP=#%KY-rgy*h= zECb`fKeV-yladH*6gr)OHz$AWy7y?QI#GLTpdIAt574%E?>GXEQBf#LR|a2h?y zd?^+HnTPy+JU8|BF|JiH*xk5$d@Jo_wzjh!K6Tw02b}7YDyp6gLyB z2kM%o;huo-y4Il=L0WK#2i*CeKpn!V#5yg}JINLA1 zartfLV3g?fpxW7koVp4KOvyKoBHD`)C> zBn%X5j`?=JcKKB;|DIG2W%EI6)HjQ*(^pzf@BOl0PEW_=u5NDI9mWs4bNg|RYNcIw z8wcKh^6Nk6&WNmPqD2C?u&J=F`V!vsSp!{6BRbzT9MjrxpS%@3j2P=gY`&^QW+O%@ zr06n;SRD)9b8E++0+wV~`co?2aOO;0Ka_eq5*R^ovqimvk>JmIDDhv8(QT z&tXOZr`3JsUg06)a=^vGom!rT0Yczf=k+&_^K(X`smhbYbUvqpC15}tsGO=E8tC-N z>8xi5s2=l-U@(lgSJ%Eq)=Y~4{VTV}P==IKT}JI$qgxg;x?+)b$zk@IHu82TOKm5a7*JEG9=rQ0s*(2 z-0bJnFXT<0w;9oD-`P>B5`B#D)LL)A(Gt-9$g^nVhnOmShbCK(|9fX~qSO7r|8a5L z#BH|w0k9Tq*Is^)IPo&;hen8YGHWSYC4ANCZ!P2NE)#dKeTaRDhq@PG2+|z^CHe|*we>l@u0E1IrzO7Qv$ETOi zf?v_ZP$%Yxxp#=#c+}a($d1cRGpmnm`AfobAS$qz>yuS`K*P< z9SQ?M*Ke;%$uz9@30_Jte4!*&B3>oo`;1hAc~m}n{?A%PW9iQ;+^cd6=AF^#fQMmU z{{V3(lP>o7y&!gkS)I86+lp-Pz2PW@$YT`2GmHpDM9qE`UKoT{qF`LZmDyV}A`{(L z{Mk6u=CaIt6-%3Q2B$$w{ef7`f}O=1&1gVH2AvmxsF+#m@AiVkMuAh1Zu=`H${s7& z$5oT`JL$YY-p0=WzV{PRq!<)mpQ=x-)Z_!k9 z{!^;4?|K-ph{u6G@vCWR4W9slISq@i&fcGYJL|unsWd?ccNU}Fh07x;vH-Gs{_#Z> z6eLDxU1s1KR30#V9!ZIWj1*EV&Z1BMSfP`p?%U6YQV<4J=a(Q#z`+6%(RYu*_d}8WIL*5S*gn+-a3%=AzYg_*RnbVeO4rKw6ZfybHZ|RT1T|; z#M$pPG{v>)X}eK6IO3+yFo;+oUB8cFhJNGuzBRYXFDh7&O;gkUbM}n(0fSLuP23fG z|0eZ2t1n*NSw}z-`kN?FVB*O=xSSwjB6VW-^AxHAp4qW=6WbsupvD9g3B|hZpv4s4 z_?CH)QPIL+VB)l8zh#*3T~%VKIY5>r>V-u>gi8xrGt&*Jg~MYmd^d|P!b#!YGy?sAzsu9lrijB@PARcS0;X*OBFcakBeQBKq-R9B8 zbP?jF_GtV9bRXU95XA4tZx1`k-c!TM0^eFqRPeg}KvlbcW-I~vvf%?${v^*~e4_x# zPo~&ps&pEB6&6|-CrQx2EOw;dh3cTf1v{kpo-RR0yb>VUUut#7}{X-f)$ zh>5`(q|g&=&Md(rx2bUGgU$6BW=*1tV+xZQn|b{MWst&PCpARvea?RVX|6@FQKyd- znCccXB5%q2pMefQoq;IOaS2bf3tXER;wTSl!ETN#ZWek>JS|EZQ(&Ot^h}NG7$NcA zVMW4rX9GfHj1Y`bmr)5rX~XgOxVjW6x8c!%x$jXLE6ztfYW%yI<&pH;ULc#NO>;67 zF#0kQB3%D{8}V!#Boo9m)A{;SUPn*?FuBAKw_OMsjG|N6V9Rs$OolK3pS%86!`+br zzu(Rj6nOY$dSfOs`Y75O4oZj5N44YiOAVq4Z7&-q4VLI8UuY-)t?H0T86b!6;H%tq z06KNMk)u0_uN!~77bM%qB3P5@7+ZjVG9a6LWFai5rr5%3z`(M#O&py&+QZY5Y>N(* zv2s=k^ah>2?#hsVA4rG0qrxq3;BQ1Pj#IqaVnOU-9IAYIl+5ymIq_;h&~-bZy}Z3w zN~@teJE7y_VuWcz@vU8y-PKUeT&W}XR$Ri9?Lzxbl$Fzt3%|7S<8j3pS}gl5Jbsze z(Mq7_wKp0k^WbZXJhCV15MO#&AtwJiy^*E^F84|cik<1$MAzw{NvVVACRgahy-eA2 zcZv0`vdoV{QNxK{2cR3|Y=}QT0@*>dE45=C^#%s;e$0m6Q`PMs?lDYGLONtc&Q6Qq=Q74gH#2kft>96z6jI+b81Sh-VVX-HN`cq|H=n-DFS1GW@M=w`{&Gm+YYj?){W^d1R(X2C zPa&H}cGm5U7ZBBn7$l8bFrhyCP|yuHjf!=VfZQEia&qbk!h#tU5fH-|3@8hc^`L!c zryLAKQK7G3`@fI>UjOxkibu;C4unHIde9-MZyI#CHZz9k2ESjs_iAZdm0@CpZ@`z| zR~n8T(2^a^5Z%+-ty$%N-Dk-Fral=H(Z8F6oRXr?eoCV+HWXxEei#C)o>Mg(T8})0 zfuDi$*JID-R$@pipvEsMJr1NW=zK*py&*KS)z)xTz+33V$D;1l^50~MBI1T}s@9@okFkuD;X z*sO%)h&!i82(_YuZGO^y&C2~4q)BLhYpXtfDXqG8rN=SEicOzW_f;0z^PU(s<7P+&ODhB!=)gH%$ll6eE zy_JOPM3_3AkMZx-as>bud2vf6f4RM8z(pN-_R3pu=^&u~N`Ub^akp~nsSJCiAe&yU zOvR-68Pw=K1tBBDG}uR(w~B_>^VgGEwf9(e4F7T)>RN5xVq|uea@z8+_-lqQ3v;Y% z89L@X2{rwRE2Rv1`yKEg@g!Pm`89R*wtsKx0(s3G0+WJnFS#f17@KXHHFr@UH~O>W z_8&o(aZd>PA@3Vp9*LL{OweO5;)T)CBX!$a&t;-SeS3aublSs0O)h^_gdK$?6v(RnD}XgB~0sTvuFtkF2B z-?QHqWaxQ~0VW%>^i3B-LZsN0tCnkNr1on6=u}+JLz~qLsYpZ>G?oIoo)ZF4eTBGH z@?67q5+bx|4cLew*bO6QYWOoaso27@NojK#&FBDT;qaVY`kxhs5`g6C?OewEAhJe* zLk$$ts|7tY+{#Wqa&yzN(^%9Uw(T}mV*=BOn>0=QZpkMT+TWL4P+6GKX=?e%1PT@` zPb$INLdh=jmOVxJM#N1qD*Dv+Pw~eovpa)rjR(dJd70#R^( z=@m6dgQbL~$a#@1lI1Wrfb?KPb<@w=<;IX*gr5RrXQOj5+R!{=wfkreyx^(iPiD%` z>2u_UY?JiA(y(MruAf>y&leqo%TB}?`t```{VFMNaYH|fzi{;h#p(?Gql?~0qnvqP zl+aEVe~8qIsRT(aDH>P8Tgx}#3vS$2xEf3fZ4_keFx&}^tk}U*m~|!(CmNv;A4n3; z%;b0>w|n^0f!iHVyGRO&uefm8l>DY_{Kgb#a|kE!6CY?ouTx;M@llE%dz<7#)FjAi z19Rx!PUeo3R%kUilW{OARMM8F2UBahI)!=tW?zWJpDe|_v7{@fT3YD>K%bv%81m1w zO@8MoS043iB{AxAdsh)9z`k-5WHG(DY|uY~-503McHb%Z-2LY@JgzA$5HL{55 zKT3a?zWr;HHyaDZAVSMa*pR?D`l~+$%T@8ZU7)Pm;Rv zW|48TTRo6d$_?7zZ60A$*a^dNJ%px_b;1Sq{Sa~gW`P{UZA2@6jQ z_r5S;1t&CMBDOW^FHxl9)H`XUaqIzUsuP7AkW3}2a@MOQ=o4flXXVCaM+*Sax|XXx z5+1*6MtI_FT_N4-;V4r<22R||GGM2wsJ%8Ms&M{t;3(BDIJPqY%WLixTzK$vIt2Ow zl3VrSE1E$CYlq`br=|eZp4Q|c#3;+i!dOC`NduU23zkZ$n zvj2z-oxSb%WZ<@C{~SV3Q&*S|lPaBKbT#qDm62EINr?5k&+5SHu}<+>Qq-nRuwZ49 zc7!3IDOu0HtpO1PL3DTz-QS!ApH8JP%*KNK_DA2<*4_%jX|{ur+BuUcqM^f?<+I8k z79RD5V&3D?M)mNtuac4oL=Y7Olo?mXCI+-Ujq<`UNKlXAeYqJhg%}V?aM+2ey10TT zlUfZuvF$lIL#Yr$u~GKX+f#&3+!V+K@%{a8Dkfd+Q^eQTXlY-zmJ{kfagfjvX^8Ml zdNywl{9DjI9@uH=peLS4l+F~ZzM6h1QPqr2)qs=%Z`2BkhnLrKK+=OH05aJJvB87+ zasLmW5_Gvq>mj?(e}D*Dk^?13fdi``G?u~jLGwa$`?F}9cD?TKRv8l&=!^R8>TjCN zCy^T4W>Zt5rmK=l^p&sHT?G6l@OXUIDUEz?hOHeP+^Z$t&EuoKNHXl$eMd%!BZzzS zgP@8%iseZ)rW-&Ph^LZ9K)>OHVMM7(*;|MQA3m;et4aIaZBCnG2f=&Bu=M+ZANswC zql1p$!#Ty{f-EVUUT-lT&jN%S-20=aSNL+gs*61S<^w;xvySxJbpc03N@_~AE|)P* zO>_Nm`kES!o_zh!R2aoRaz4y$SCgOLi(hzZ4XDKVU!{w3y`S`=q6{|1GMWqJ=qg}$ zog|Hl%`PMuWegnxO&iDlLR1ANs3`xU)5V(l%C|BL&iou-k0((aebHWSGXp^xN8do=LpLoZDfAZ|gE)xs9AaM?Pt7(NJ=Nid}VR2#FuG z%UPs>OiXLIjHir=2pkY{w(M)%b?nS>@I92xtY^sq$*2>Hb|eAjb1Iu3P~0u|c(?@< z;NAq4k9birzya8Xg#ogGy*woV2C#4{m(~nU-}ms9&val|w5DA|fRK}upZz>kUMcVT zV|G;@yFZj)pZa77an{f@qBvj9$K}Ah=5>uZz^PMSKtXg~59rr2y6$UQ6C_A{qJA}= zO#H553U^L^8Xdm_EJ>UvpuNK;^5Cy`TbMaP%womfC5|F1MznLsGeS$bz60j5+Q6&8 zqyuK-UsEzB?7&LF3L&nbLL|YCTb+$(;O>1cGbdMK%~=1XkTeiVs%)~oe{j|%9CoUUVWzS`{qfjkAF;50frD z7F{%2daV{bpo1hJvQ>Q^asw)hJDbr~#y70xkJgA(G5hTqAPi`+%O>+eQPCu^o;K)d zD`QIU=4^C8VzOrXr)CPYMWCh9_X=^Q@;SrD|=Fo=jeTbXzv3s!X-3S&rAC@kb&# z?`+l&XZzz3-4B#1f^K zm0Q>$Xrd~xlUaE?@#ChY*8w)_uk7~Oz9Srv#nzsLh<904yF8B3VSE=^k0{y>!ovNcRq0DMj7nc2N)?YW9n=X)O!?PhCp zSDz?RvQmpX>^5m7g=S8dH2~L#U?O#^kvZVaFA1ODReK!^!xY1x_ zk54Cj!UMgS%%uA5Wq1~5$MLqL%1p9u;UB+AIrr|vZg%$P!uh@e7B`V ztW6l(t{3a$-ado)dtG>v3i?l?s=bu+#I}S3O+fOljXo24|0&7LR zZyX1^Mv+PD!l!Z4&S>Jc(`??-vpr#;rK3hfM>+GjScF4A0&ODqyU+biK;H3@LDv&6 zdKEq6z-WT;qGuiT(MCZHiynfJ@>8JnKBf+Vo<4@?2>!o%T;#oErwBML`bu;enHxg9spVO$TyFv^N42RewI$)cG# zFm-avN!Q%@As|DnBXr0uaZj>Dm>8D?7)nxf>~u`54>6BO=a)07bd4g^akxX z?VLJ5HpyN=#Qk(gAv5>-CFy{06tG~~2a}_TTzPtM!Xl%3>dCL3h(W|P(#uLqQple3 zT2)8DnC3MjN@aRQTy%;xCq8kYPR0Yt>3?6@!O?bIBC%gy{IB;!@nby8m9XaV@LqlDQOu=}WJ@J6rilPyqqYE&C zhA_mcWHb|J5Fc#YeU^0ARjnZRv3US7tIT>Vkl(Le05Ss^wIJ-d{S+UZ4Jr_g;HULz zNrfIuwhwfoqkktbSD57??VXFn3zj~iqs*fMX=#bjobf!2A)Ca-BB1l_JyZOpBBYhk}oVL~f)nkWla zzr>XTkN;bu%7j3eM4S1&o2SF7tk2Yvy*T9~?w=^N1N6y*g=@kOG-ms|$@QurwZl)a zkZU1v+8}!z_}vG3evg`(^DuwKXH+=gh|bFS!y+TDctQrnu89MC`#c3k~*R=J^}q0@1yYCUq&r)GoBJIW?q+nR8oP}Sa7^2@8B8%cS)0mTHr z7FRMrj$g}&IgkX9V%PB5_c$wV0Nj#f1M9Y1Xz`FT2KVZgoggD7>f}3-9Oby_FdP!x z*s>I1QLVpp8N*)?W!qJu`B7OR6@KR#4SK1O!YzMGwLR^w#HL*X8gxbrt*EGr(zb0#HqL4m%P<2_#is5Ka z7<@*#|M$bUWhtMr%V|!$#UwHN`e^LCzJ>BU>5LF`dU}S^6k5-w6V#}_zS5f}Zu?m!JUZYO1i!lQNbP(TEkW)x>-`d~+>aQ+JZNF!|#L9zT{-CqOMsyFIK|O8{2l z_Tb#TsY)(ZPkO2Ds5%#@I|JesZq9wJ&vxQ{GVQz27p@Zqtb4hH74dRb%0o)Ph@SE_ z0zQT~74{?280CdicKVof3}It+jEzBF)0vqW=U+b&e{wchg(--haQXOL({R>vnU58Y zaT!fhv>>Q{Ri^`xqYrA6XpLpn|2_Xb^UCblL+HdNE3%_`Ss!wCpGl-~jcq z_Kq$a4==wOXG+ve6ly|hQ3GqwYR(FW9ug}(0pwLIOiC#Ltl>*bxn9d zxP~romwNxaD&?uLInIk4sbQ&q&d+m(krBOEL%p?f(4=iGO%YAr*_B#jfi$2vgxP10 zZDIG%qqfH1q|V>ajrOl5VjuX7yuZ@07T*2t78+rvwpk_10%b z<{airI|VDkG%o&CWHQG}+e~c(!oy|5!B(XVxaOG!uR(ZQ?n#QsJPQbBT(fRGyyte= zxAKtuT_SWg3sw-y05qATh6!iPDuslkIPdS==Bwro0a#8L<=MRqGnK(VOR9fpM+_#m zvm(^(LU9)9pM<_&m!LIy459fTUF=hZAB{ihVMAK6#G|l(9x&OBU4N9c(qyFu4(J`RyEta`&KN z*%ghZfim|gnOpFX6RaGS9a(E)KzvC?gaRtC_#XGY^gT&&GzuAm?UtHglzr5np|C^{ zDu$=C%1}m-erpXVTF9DML%&l$t91EE2IAabfv#0G(C4f{^yIn?FM+ER^m3gP62Tm^Xz(Z5t z&vBD%<(Dm{-~+?Qo(IkCYoD6}jK>oNN)E3?G^Rqk^3LufJE}G7|K~@b32z2_4+A0r z12&+c0X-D{Yf%TJMo>Yq%*KXpjo7=lO#wy9=Y5Yy7_|Wb5tT_Nu8mMV!O9wWTh@Mf zI%?frYuPy5tHs904jH$~nO2 zH#uqHON@qM$cPlbo)bD(l0cR+9mtF3d3_aySew&(7y9&~Z+{H+G9v4jf*{j@TO(4E zNZz?^^|8`JQc0URw{OJU55UciJQij)6EEUm!wS!z?;B9*W$3--;l>GH!NWvxIkVTbnJ??SPNHUNT`wU8`fcaQEC1AEmti|Yb*D7=^f4s3e3A+ zYZsdDO8h$99pE|mg8KNppStBw?jMz8^$>;$K^v~X1d2=vMOn;8KF$GS6Y0$&+b0N~R-T3wEbWE9Z1=0MU~&pP?_xaIh7N}M+jLA9ejZ4Lo^m8x+Q zgP;bfuop~#KZc^ahRWl$6UQvdQW4}6WT<0TJf6@#wxXSMklsj%_vdEoIs@Zp>e5r4 zBmW6XN&$omz%==yy}PF6f%QAoDwS|8;=)oBZ#^-=Z(YKJJ{I1_B~KG zgJ(3Y>CmDO(cXvA6j_3N?`JwS+DG8T1 z&zkamZPO2rZ-?g@9%;jVdKje z^;j21^P$u6dZp*r%v^1<^K{BjM3*+j-p#mRtu?O64e@5D1ts<|G4dF~#GOdRK)pS> zm5ZALA+Z(ls%`xNN@LQP>mx=jKPS3kwBncND?j%458I3P+@jegBDT#WBFW!Zy(jeV z@la^9bE)_HL6I60>vxXBe8htPV;9dy45CB}4v^BDVZenYLw$1|Yzi(enONd-jHw{) zKkv(O2gy_$qK9_Mc6x==VoiU9P(1^+|MY~eh59{g3++{qlMtG!_dvq+#X?8%vfjO~ z-b_+?Md4JZ{V7JGeMz_$))AtoeLu1b8`Q!ZljytO8!(XcdDWxOurUS=ziMf#CTH_I zmE88Wp@7gpGOn%Ze6+KmSp-_jb(KN@!%qyEeU^JV`P1IGS-16N8@H zR_n?y8+sFe_mn<-X|s{HE3a_4dod$qf3}9~ap6V7M?*Yy@u~UMAhW2xJjqu7)=zV3 z?@+HN7F_HH)=QEg`vkOMr}q3g8DrClp$ zOUgoD!()|9wtUsg0-JZz915ArzoZ!5sMRmc8B9cJe?DiPzd5+D%}pZgFjMG9u9@-k z(EoGReBB~XRF$Wj7dFUA@q@X$q^9yK3Hj|%k>l-T)YoMVfW8#D_TR; ze=C{(RUgot-JeEYN4*O`ws9%Az&0$p< z`J{eUE9DXPBlesSL@GmH1@f zx6Reb)06DKBi|fp)MNTWnBV3#HBA4^TaZTejoXICpM3fGmC*NBTBswjcAK?Kuax=; z=5U>HT((|wzg<$k{9hS}5(Drrex$%4aWHQ@l#!?j0F%-fnoJ1pQPD|=ZPaTj6UnCh zgS;X(E&sTf^S~18CoQK)dLJIYivN$l-&yjQL$~ymTP^s0UNIhVk8`?zu3HW7YtZ1o zsC`!}sBh<5;>F}#N3$NQa?prda?x{f9hQpoR3`IX#If#X9HyNhW_Ol(FJQM(4z4a~ zu$`Z=F<)8k)e>&Tugi6CJhsVAiD}ErAWdKo=O7s_Of`KgEhgv{Vv+g;m#yjz2MEg; zfgsA*fSd{z?n_q)s7B%Pb$4Jc`eZ{0`rM;&h=iJ!8;ZNY9E^@^OP3!q_-Ps?(4278+Mdbk-%0f$fvdu!?hIPM9T1MEqbku zZ%gSjB_qNHce5<$81Dlc1sE^RGrzcaj9yAO>@yb*^+f%gisLk}@1~=i7i&yn+pMZS zeZf?bCrdKbTeA{OrgJtH+~4`yZ^rX$Z1`SV+0K%Uknu}Hn@k&&6UL0Va^sd5Z$H0b zMJlLEb!`=Xb)&36vS25=7ZzpKw=7fiXIY#FYI3vI!}Q1ny{uA=_E-saT?3CpN-q-z z&2f#HNg8Q$J2Hh^KW(hd4F^#-?>>`pN04u+oFl+m4!6JFnzAjQU=(-z8i?busQqAe zm5IClptQJhD}T^ZuxHs7_a!o-UnXcHG9(L7a14lS57=tI*d2teH5loCY^kd5iyWMU zKJxKIY`JRHEN1?(PB&U8F(c^@O8;K&aQJsl-x#1<5A}`-8_kv3gQCz|G~`<9c)16& z>#q4JTdIz0kxL$yooN{{13UYmwiDLg^}eB-K{cvOg8rIdM%2U)JYr&kVl=dOdfr>f zGsoUHZ3Li*lwDFh%Y_NIAMZ^{|Aj5FGRBOj@;CLO=5usI_O3Q9e9g2cZj-r85@Ki! z#~7}0JVxPeJBh((+IZzBYbVSX0h@#rvh6rfg*|cCU?<0Gxpfp99{Ae93G+`mQC~;3 z>RT{Px_HvCc<8n(3>&`qb15j?EWL)8)UTAVml>5P%e0Sq+?8r!PHCZ%7vA&9>J)LR z^p(X8zR!NbgP$SAr^%j$Opflg`FAM%Uny;uF)fwnAFuM zWv{#NNFL&U#5n5Sl1g%s6KJbORKVEz_BSnm$|Ww(dSKeH_jym9PHzoHX!ZJL+tEVC>F?a5 zF=@ve;r)#JmDg6k$LxOi#}k~43JVmk`rb(Iy(SxvYIA1kDbg@>5pn+|e0WJSc{UKO zb93Wpz}8*)LLv^g%9rcu7F|y<({6o*Uov?Lr>WXY(;fZ0{8xh)8jBY3{R<>qIH=elJnqlA-2{ zDCz_32PGa(iccG#mA>HuHA+jkJWg0Diam^F8!c=SIA6qp-iL~hNgC0dzdSEqkS)+xoX&W%h7NZeNA3pzYGWy(CqD?)gnXQt_OU_Q(n}qe zrYJFQeP}i-8|aTp!0zx9SKRfQV14C)fZ}tXN785)W0X=X`G`kS+q0}$F#?+J?+%L3 ztzE;vSU>56z1y3kB?h!RB%13E8T6XNR3c$oEnJ|0#ruGT?G$nZ8i1M$PYZ<%oi+ zWINt;wIC-Mhp@NPc8an47oF7dcc&+sdHQr&T24Fd!dND=#m-?QQ<_RzPKT=G6Z9ja zqe&Y&#m`S2>&Z9gf+}u73RY0sx$$DvlBdj;7+A*qupQlu(aCAe3qz|9YNWV?aqVCE z8O$z=51327zyR`H^zdiDRE?yu;cB}sFFwHbj7*Pea(sR<9(@u(r>e{f<^E#D(?Wg%NS=*lfu4^&IyV+LM zn!49Sm2LBW^{|}#r;lg_fcIcR3sDUiZ7~04C=>ZjC1e}qQ>|^AReR=SnpCvzoPH9h z-gHxktxv1t9FHv0DKY~b<<8ECsD2QXN268z_}=efUsy{&X^^dW)g7qi&29OR8q&%` zqJ`#B*pA{GxG*r}DTQbeOL}Ib6T?j5j$(6leQs6RXyv)eOZZ4Htw70`^rJR3JdjGe z9Lm|HT?~ZMPWpp>WRC|7f6}Ox7(PlSr>YhQxWJZ@1&DS4#vPA<+5ka3=pK?u zD{H9&bywmhC*MV|l;;q;-J+yZ7$`S!kbHx_ZOI~yKYj!f4Y)7qAj{EQ7R+%jHT)I! z7V|pVdL~<-i^gI+l=v&`1eLP{F!Mj@c*KFg!Wc@}b9b-9$MF0r*C3Zpfv34{*T9%o z+BvvD01h%F*~`ns&_=Wf^zvw5ME#1~!{lUd`HfMt%kC%-GhI|)@C?zD?l$fL8$^Be z$~{MS<}3apo{76XVCmfX-P-h&y!RLwlsbz?OGeAbrsrS!u?`stM=R}w{)18dg%a!i zPQnfliVASMwpD=uK^ua^c$k{h_M10-JnFA-N!Uu&gRx)H(FTK}E0l=OX;pq`Jb@$44~k-wc=7Ez}8XXfr9o&dQ}BF^Cyj-4`} zU7UT23)s~Pfq-rLD?%j9xKOU1i^i>PdbZYx1qKJIu@Es~z(1-19hB3s?cH!OjwXlm z?RdZ_`nZHRT)1}K^FjMC6nJnY-o2!#tdCV!#pZT{JPK%9pZ3Glak4_~_Pjk_6;Mb` zOAtCoat>g2Bo{5_FvHk+ZHPmpEw(gYg!Zois?GWUZ{o^`CvVyTO6ojIcfvbd{tVbh z{wU1wJ#~ybejIrDYlgUo$$<@V(07joLd3QI&@>C6bEt0sQt@or=-1$$mFDbo#B}}5 zH11X$tZbgbtf5qafmgjd6jwE8lAaa7{;Y+*cp&C0PbPMON(J&!teU3`V8gIL)DTSd z4i!F1WA?mnxte@w%G%wf*bcaQajy(^r|I*Q-L=OCw|tkDJ>7iPd#5aikKpoNgE#A; z(#52K&|I{HJ-ub|!9~y@+MeiQC-?mo984cO?Eh8K==uIW5nP~?Q6d)N_k3D>u($1m zf|C^_kj5<=c(K9DO7P+m^+64}#i6P{#q5wD^cCH8Qrg;?=eN3~69r(w;rMYMYL}b5%eOma8kCjEZT@>Zs z6*GIveoNLkjE}lExHEWV-23S+qE!#)s)>PWCW4jqaZB)BhFgh*C|g$h*cIP}<&(3( z4NAG@T0o=u4nn#({4{e0fbgEo^iI?C-m zKcv@US{;swU%9)RtpvOA#QS5j?y!zO=w`VMCq|S-%LW|CIb(dmkgxXz?X^R#kC@FDUO9<#gM5yxTS;Nhkr03 zaln#lnoP8ey6HWDJ`~ZKqQlo}x4$R8(q95jfI*~)t}hdc^0RBKvz*k;+Q| zzlX2I4pIrr;<)zwIMTOYg@hes3K_Zn-C?Jyzr{;h_ zZP)n`1#^YHopgyeLQXPx%uzX~As^*!fU0dQjE>#9T~93B_jX|rp}1nWnzDMRbiGf8 zJElMFQGrA^lI8kC_+DEyfOMtb=ceKuHKPK1`YE&ftx~(di`(J=S$a@wjjSM2N#whu z;gi;|qCO660{f|TK4)GR)ZKv)WO^Gwm6O@OWdyyFfrA7AP^xDc zXFzpMC%ATFUi+Zxv3yBq*9RFe>7d)H8{zN!Z1Ga^kl5d}tEp|@o?Ru{IKuQdZ>qwX z--(E!(@MFEb-Z}d9aX(0z7WoQ$E;}xhWQY&6*b2++Mqk~PAtHawxdhD@EMKZAcn`1 z5pfW2s&+c(|DpvABef*4^XV0}c`e*%nI&&8iAa=(V-E}c!cF@gojivluBgA<@@Lpl z5Sb+TB{}rEQUz5#Fu)xtZZAWcPd}()V~z0vx$9P{t9DCBfR^*9VE7r8IYwzXlmf2G z+fAT>K0!~}9H_LSjiLc?z`msc3$TrH-q+5Xq^lT!xxJ>P%+?Q{MlN67*C-hSmpIom zBd~ZPkaKEme8T7?n!WbG<1WTBmeIqWw%CMptK?wp<65P}5U9+eB?MQ9GF!c9snb!& z^b1P}whltL;q4-!01l=wMAnoE=r}~MlY!-j+ufzC@g6yg{8*dN6cgkH8}I^J`VGE9 zyKIXfNM%}}_u^Qu2jZ>bNP!U+=bHDJdAy<@0@VGiNpNtUst)MHk@FgS4pe^z=3P%c zGCx(`_1UOv^{Y+L_Y8octvsN<6*k~9|Kb9WpToAS@82ECdS@ch={Ddu*vA*Fw~$&M z019oZbE~1E>p0&>e4Ftn@m-_Wy9~JXv#lWFI5;Sm8Oh&08w~26*?(>!nyKb~|DkGj zh{M$b{(gWRdhON}}$8@`F0Bcah)-kLWVY zZuJmcpzR3b+cgI-ri-pVN`1uC^XqQMA|_+IUH28-xTt&a-l}rnv&M_P))I|X530>q z7}|#ih8rSOUBGo1m`7AO%V)q{S!HzuJUId?$5Jt=e1WvXlUr@r}e zcZmY>eAXqN={A1;F>Q`81hLVgLv!N?AnBDLR5EH);x=SHC_d zVollhOcLkj@OqdkK7MAWfC-PwxGz>aWzAp%AIkaQc^pCeW<{a3w^FA+9W#B~RKCmRhQZM& zW+nx+irS`7Tng`BTYm3;v{VC9Hx(;&XZI&(9`MDlsZ=!d!+ zPK3s0{9dA8@2|TS7VY)t{$5!@`pcjg)5p6B6794jk|Js+vb);hCH0cIh<4}^q(Sg+ zn}vxK8WV!5bsCzjfmdRc9Nr|{Mh2mvyYKUnYfJnMMMK!uH1p-(9fFac-`{|oc1hl; za@N38UIt%)vV}IG0PtpDc0C1cFzEU!?Q11pz>2&~2-n3TkX=cU{?RPJi}%z##33Wi z?dnyM66+mte?$_TLs%kwaipxOt{okQrt!G~VA|?0%kG z%CuHPfykt{|0V5p91brjAP`Q!pmtoYMwP<4ZR5BrP5F!xEi!~n+*6!L37@>@bRc7{ zI}X3|)j?F=nf1j*zX?}a8zGXSZujrm1T^2Wqx(NmzRB z#13GF1=T5&d_q~k^EJJbw>#CykGPXRr#&&Htju-+AHO>%Jp%cC+1vs~wo(t{wMJ5N zaGN6#&k(=WP76}yK5awOC$97fh1&qY?o5IaR##8`t{b(Z?82eFN?v{ImK2CmZCcWr zYj~Bg0aqu6&?k0P6@Zh=UDPs`!SuuWguY|q-{5pAV;bS0<#m#kl*>R&WHyYG#V+@A z-4(t)U(;y0iynsuPg6^|W?}XEv*uGnt{r!Q>QxT+%%GqIzyJU)$ibIQ|KiCB)Pscc zE$&|{3b|a?uUdcu2JRGq7wx#B(GYzDIFc{R&oHo7ce;u`;1Lo$M+7JMr{i@%H$Z|( zi0B}I92WA{R@_nMjv><+QY(b)*@8%dhEa)xq+yrHoJSUPTZ$5R1fX;ecS*~bzRE49 z_%1086GsXAqbc_@g323|t|jdg4*Su?@RZw8fY$h9N~rc{7mo{18cyz-1)*K9>bj5P zhMEQQe3X=%t3!suxovTBm(7BJ;_I!lmU<_UM|g3waUAOFB90vW6s>soq`@uxvFU_| z*)l5P{U;B0xM}i|FJLyp{_^H@7s^+1IJ9!c=DF}VE-o0`T?Rr0dl*s878n?YznSA+ zgRh_Vp)5iVfb4buVMaaqD`&!7pyUJ{SqxHR>tPuwh*%TN;X~8zKpBE z)kXyf6*}9J;jZ_>-k|K!ym$7+xzQ9@8><%Ieg*>B#gRSk_d=U>;^i- zBiKYk0NzPjK7X}zy4qgqg192$PNh*N*M~o3<47j;8dSkJB9yd{b&3y`hRwpp1}|4c zP)wX2no@x0MJPW)oXdHlH(y)*n9CUD(|RRAAG`XtLtJrBb$Gm2DEB8#j0F-^CMB_S z4qm_y3kJ8wCWRlLAQ=zR5b6?S7au}UC}ASMptGZn(c6M@mPDdPeFyo?Rrg=DoE0J$ zyyz$U@%MakAC(Ir(!SscSTuA&168v%Q>d#hDdTBV3Iz;G|(F^RCX2c@!&)!dgHD1^ng)K5w9BL>13_^hT z;ZDBxt4F4NpfT)d<5yg@VvAzJU$=w-pC1DJAJ_xPI{qkKQWx=@f9Zofr_U&e+dCv` zNdNr+dla&6L5l<$(yi^51kl5xe^rnKNy7(&kLak}{(_vbjCnMba?Y2ll@8Y0%DZBF z_ctKK5BI*!O+@{CoHikG-;M@`nasKKZ&(Bf79Jzx;HSg3bZxQwuHbL^+zwQCQv!d( z+2{7E-J2!myU(p1D=&os^fPmDYh5OqRU) z>uaAOgWK{CJhIGn9*8NE$$X-=<_7B4^n@iNWYQA#ni@WUM5&0cL+ku`L4D0ZQZyNH zYWrAX-xYg(=QFa*$^GxhW#?1HF-oqa0hM@#N8?O?E{!nv1l8uuaRcZ5A(J@?KPdkv zn|a9n3?rhM$zxaP>`^Kr6hwQONt=iNWP^>OjQGiZaX&AJcg3 z9)C}u^XWzO2gA{(jYe-hH+p~lw$qkR(x0-j^H%*P$Q_Fw7yS%ba2)w7zOfeJTy#@- zoqm|aB&Q&6?<aOG)mr7haVg3dLylNQ**m zc1$IkE3O~hY_=BEE>Gd>T^)^iEq~75uVwD1!hRNzX;sHm=FMi2kAfg$mQJNchw8jcAd)^#`>Nb+HDO2>jSg3JG_Mj z6=SMc0XgWY+TBLQ4TX)J8~0jwNJwa5t*UQ);cpBG!zc8aFk!ts>2U8cuFMr-X>0p1 z=h{Z9u$legWX;lOPWe%bXsimzc5ODxuWG$J;|5IM|sk%zD;9-j+nsxd$H?EYiPpp>72tT z14Kt(0V(tL9HU4Fyh(%_^7+ZuW=$vWdjD?aYdAL<@H+}^D(2hq^L~9)ep26-Z*-xx zt1hM&c)ogU;}$~(-KI84#`@a0!c{%Y*!(Iyw85SO?-dk2AIfd$GlmyiUkezHhhpbP z4P)o?LZ*=&ynXeXSJ>rIv~DJB)9jJO=`r7Wn%G}t%xJn4OJsX5xo%m9i#8`YV+T;8 zzWeN=#2|mj#o=9Npju8@>vd!~Z5-!%^qVo@R*{H&MUV2gZ+Sx6XtL%b*tA=S|0X8w zxfEU-kEs9ke764IR03iJV*x{ot7RGoqq-@+lt~uV?3;rPi*i%(c1Os(RE7Jy`C`!` zBH}pC5v{q%V*65u8qL^W|8&)J9G$~?Hn)c39AqEMc11bljT&7s6~8KjA|&;)=k{*n zwz>7AO}~jVuuI4{1^HN}*Icb!hixUc$j-Yj{kL7n48q#4^PfACHN6r!+VjwH*S(k) zV3`eYNM2;SJCTMD!*<9^jC5Olnx+NLl9JQb)pgORu$co`xYqbaju*!#uUG5|T8Qz; zm(j+a2f5!KH@F(iK(iNamFzoUBA)mD$m%VFU^ig%Uj07}D`E{*N?!lnk#v;KZe9H$ z-EX@Ar+wO4>lfbt^avz?&neSAbEE=7Hz~AG>&j@#hS3|5J%E1e4y5Da09m zLj3LIPP1D8OIAli6hrf$%EX~n*=G7a+R;hOZ}M*0z`2f`7qPUgAav+|ZwMe@=QXZ> zcPVk@bvzN5H5K5r&#g}hYgH6^@;`mvVc5RvbMjNmR;~MIB_SH3y^aE2ro?~R9zvtC z%ryQbva!jv-NP+b7Da)^uRIB1d^krV^mY$`f8Ioj9DNU%5BE)+V_`688jQr~u+0C( z_}`~4S_RCaduytCOD1sS|7~1v*XkjgY(H1y|L+GT@&WP1W@HU7Pu!|dtHSHH&80J` zVLLJ*EC0z8CVo$$e4>ouIg4Oby~}s$H*<`rCY_G`zu5nK$4hTWBf)7=%el)WE15Mp z7ozbE-cmEzpdlw=(y(IZ`Q)V<1W*`IRE$fZwO{I=CncOs2(4?fbVLV zwn+jDiiyKBoL5Ny)9p3w{H0%KNZ$ubvt?gnwke<|7gYVA8dnUU*cxgmzmX9JhMnPc-T>OFhcyhc9$!A2Ck z!t#poy8KckT^sSFx9Rw&xv}6#>|B0EG!>RWzFqf4&VY`Altllh!#16VHUcH%zLY5g zd7BBA%ul$>JMF5^F;u-=_OOOssjH#m*=ITyV-{X7S2ytS$nZ{sz=SWDULf3x^)0Q6 zj+d$5$AxErG$+1S@Pnqw% zrJw84>%>VxRe2+S)q5L<;y>}yG4BT)rVmVeT4qR{knN-{jy2%5hjiA*3f*47lrJ+y z%<3fs|4bhBld?;&T{(xJlFhl!=?YQ5)NC(Jn1A<&fldTK^O+<%|+om zH;fSxFY$5eNugST_g1UhUg5Dz__gCWO$@pgwXNC}r%u8%poa|l{ zV70>rBY3yB6SRqz-F3EAZs%?B#KH|lQgX^xUuaf*qxit1G)PvHoCDM3$I>iuE>4G3 z8k}CS#av$6O;5M37nl9Fkv7GZh#RI`I=@fW?C4O7 zb|3L%@3|3&-tBnz(O;Yne}A5Jmdq@Ge?CUxN0s0b$yyCPeK%L6rW~GcHTcSY-_|k_ z7>aEZQFph^5@gV(4!4OR&T?SM)puq8ThoMi3YK-pVxL@$a^pcw?kTDX8CNC-^4<2; zmA^0Z314^N&AMBK<=}6i3t}h)>`cgnyD+eCZws}1-@JNXPh3i7xI2wpdOzwF*zQ^v zK;#%MiD{|ua%v?fJcu(;eX@ah7=pv=YO#kI^n&2_)d4aODF?M)d^_L$#Q&6@1| z3vQq`M+vm@?ZKR7;;me89*d;2ZYW)+4HUdOTNYL9d$)V$RxvS5Ess(@vB+_EK(HQB z4kmx1pO|&ylE44V%hF~!hiQm-R#8eE!5fcK>F$^<(jq44*=_2x={o)PIv=eVc)@Cd zE)3VDLIN#n&bvHw&$kgJr6tE*|Gwp#-oNnsZhp6lEOxbhIxt;*wl%c@5O5U$zI4kTC*x!tj%c75H7}O=0o{^wN5=7 z_Dam?hkKD+p5HXCu3IhiKD+MhXv<+D67YEctvNNGEIB}8DAE@5YM=RJhxCG~^7pq9 zw!7&-fJDj=kS-d}!KryBe;?TVKzsHK*M@4lK`WQIjE`@Z83FM#I#}VokA4*+0yVWu zU-js^80;={Wc@0HXPIfyMH>bQuWsER5lRur$0;=IQi|faxO%Owq~P(4x~8sIf5fAg z4DTV@dkj;w-s$}I3h={2mI7mDjt35FMw0~>F61g+5Sw8P<_FGy*GeSm?$j=iIE+*O zG`%PAyTFd{XKg%WWqOZRN8n6UbS~2TQ=Bp%r>cAEa|&(x+LIHqttGPgOYhfjkFt0l z2hu@LQ}27-849o+DDha~|F^pykp}cnqh0Ov-Y^3ragdacpC>v*mE{E$N^akWlDj!T z)xjLT{A?Rf?&GPv{z{w{TAOb%*y0|j%nE^4vOe^=wVi9sm#Ud8uCo-)1`WmxE&RGfH*_4 z>D`5kjRI7o(C7Ubx$Ue%x{A+?uG+~vr}v&1@K)>;)LXy$Tm`KVdg*e1{61~OKc7g% zE}-e0uT1A}ddx%8?|2XVbj@A5>z=e{4MXxY>CS)s3f;;U7-_oLIts=yh(q8GWXOJh zcMx9kgH>!IYb}}Z+G5W5ZqZutv4M6jLr{8D^)h>n+PQq90k#?UYS~*<)@=h zpEtibnn105v&e_>Po={|Chp06T5F!mzdYeH_S13=@mFFxC~7m7k(GxbB(p*O?U6G@ zu!heIX5HMIV;+&$jklc2ZhIf0Gmc8Zk~$n@c>XhI4Dt5?Q_Ibv_LO2~W{ke$ys0w< ztN&7|FZD^_M7c5@qW#291MZzVgHGefY$q3Pkb4`YA%*W&)@p9$NJ+u z(@q~P?^vcvsA8(AX)X1Ka4G@5{gcT;;Bj$P|6we6@O*KF23@p0+~&?c3g~^<@N`hi z^gvS3>t;nxan~kbTewcSDWu<*FQ6ZXyXLQsYb9}J@iR7`(^xd*h zirrI4FAIL)C&`cHI05Q{)gESccXbz4RjK}dZP3i|TOw;<%tCJ>@%LEi>&$rGxo~le zzSLeZY+QHZe`o#>TCV&_Q_HCMFv6wL!13zHRai^RCvZiC*^Z@V>(~7)oQckUp2Y|k zUdO2S#MrM*#Oaiq?~axPOvJ&ktX3hXuvBGkxp@;^Q6A;VBg0CdyDz)*-&|*J&;Q~H zFFt4X^|4QCYHYSyADN=|IN5vn33>c;*3XcC*53r`W8dG;`-Kx;&%L#y$o=qP^~9^z zxyNgc-?Z*qkg?KasrFZLeFxU028Iiv`D%Qa>>8Fmf=bbU3t5gP@OMN#-G2{a!Ik&( z8Dhy)`b|1lrDaKZ+4mTvnbP5IR z>C<-%i$$luROS)a$QLR)J@0F38#8Fv$YahU=h2`1L^r;$O8a(ismU`VS-e#Eph~Vp z?v?Qnl3av5*2MeCwgqTqT{bme*qv8%hF03pVxVcHoPo&cMY6`bq1si1runW^rM18a z)E&Z4c{Gu4-#{8x9IJNO@$^l||V+ucC|T&kdvv zUHm^8qr4JrhgR&S4vAyE*4x-<9WSV9|9cHMm6>SR?e6`Z zvUd`OYtUv??DlB~sb3HN8w@5=0Q7L?;s03yEINUMHF-c0Dmm%Fw&-@aWiHzeyels| zefz;)O{b#$DPN^;`nO+~348X#XKCt^yI-WZfE5G$4h2LC*ng|2>O#t=cua zFa69P!&%=CS62Px*3Z89$#vGRD_3U6IWwkn02`aha+0hN5A_NDc(_h})y3O4jOO*% z|J-QyuWYfHZuF8foilZhKdwG zl7035k+{xlbx~=ve%-&g|ElHY&sVhlw^|-!Oy>sLRb3}33$;sZN6Di}_x@hoJ?rNi zpQmR|Y&`bs8+ZO`mc7FBXRP4=_n7%zJ6V_;vnz1<_c>v|E|m1nFm*ECXXmW9{j^qD&Sk^3zrR}9{(JOT z>y-YbYogJjZ@adATfe^je50TOw_`%gN>ONe+N2N=8ME?je$Mu$6VIL%*`K+(&3xNS z_45BN&-vyAUt6Lh`b;$IX7K4R*Xw_-uCsgn+-mdpp0>NTK1Y{#ZT%xZE&A!~_3^;L ztvajE%dKWtU(*T>j$CZM?_=Avy26!GucX%`9c4Wlwwv`f_m@Y1^|StZzYYzJyBfn< z&#ZGm_67rUuq`x6vBV#^6I=UddVlse^YrN6>G4|Ezq8lI1FsSe>JHbrwxo6Pn#aJ= zl`qXb@)_@|t4%+83zvSp;rjRG?)pvdUevt|j}Hn7nG%}!T7KKBtLpJr)%X8)G2Umk zIMv~_xbF2k@6TSn^Zxu)>$VN6Sk(^51B>H?gjLYKaN`4R;FTvA*E~<(YBJ}F^+~VI zQ)kRuQP|&`Z8`6fz3a2>i&h-6+V=1%?=|L>{y)!^bq}t#Ok}!uhI8hvFQ=AG%Rbxw zX7}c`J{&CT*G4eyJ8Un?ND0_eOaZpjrim%PcUp?GE<^YuBD(_W%FzrCE7FwvB1ny_++B%$Rm? z$77DAQ3g#v8cjDi9$5x0Km|5zKk$Oxef73YVYlP%Ut6_!o>70?i7H3^|9?-f?b|FY z#rW^0fK+)Dw4uN#%{9Yxt@Co;tDn~EZ8s9)*Mft*!=qHI)4p6W&bL_{_`Uzt6E8njZ-rHx zHh#Nxx~=)D7HsI)!G%Ld_wUvu;pD?c?yIi94h@W(cxKl1DU6!A*N*&3X7UYdzS;Rb z+rwv@{i^O6R=`*_3_j}sE!-Z2u`F7=_^S2p?`vb6ZYpqo5D1^JAQRdz0+wwD?12T= zzrrPA@Ny0~^d)eBjdNBSwABc#?Efe`Fuay`&w@6nfVqZa60n>dK?ddh4{_FoRSv+m z(~1=uSHrHH`mKVDpM>e#Jgeg4Jpze~(pB#k)w%O7?OtH`wJurkn!~)Cna&0eYs~lLv-EaVkuuXOl eM?_4G_z!!%ZTs9+d&{mc0D-5gpUXO@geCw}1plo7 diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index 831532504..1080d78a0 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -560,7 +560,7 @@ information about the boot process:: [2013-05-29 16:18:20,511: DEBUG/MainProcess] | Worker: Building graph... is in init [2013-05-29 16:18:20,511: DEBUG/MainProcess] | Worker: New boot order: - {Hub, Queues (intra), Pool, Autoreloader, Timer, StateDB, + {Hub, Pool, Autoreloader, Timer, StateDB, Autoscaler, InfoStep, Beat, Consumer} [2013-05-29 16:18:20,514: DEBUG/MainProcess] | Consumer: Preparing bootsteps. [2013-05-29 16:18:20,514: DEBUG/MainProcess] | Consumer: Building graph... From 888c0aa79699923473572e0a6909e0dc70610177 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 5 Nov 2015 18:24:32 -0800 Subject: [PATCH 0796/1103] Set release name in whatsnew --- docs/whatsnew-4.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/whatsnew-4.0.rst b/docs/whatsnew-4.0.rst index e88d31df7..737ac7910 100644 --- a/docs/whatsnew-4.0.rst +++ b/docs/whatsnew-4.0.rst @@ -1,7 +1,7 @@ .. _whatsnew-4.0: =========================================== - What's new in Celery 4.0 (TBA) + What's new in Celery 4.0 (0Today8) =========================================== :Author: Ask Solem (ask at celeryproject.org) From 991982583773555f918a236279a06cf36a32cbcf Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 5 Nov 2015 18:34:21 -0800 Subject: [PATCH 0797/1103] Fixes Rdb tests not resetting sys.stdout --- celery/contrib/rdb.py | 20 ++++++--- celery/tests/case.py | 8 ++-- celery/tests/contrib/test_rdb.py | 73 +++++++++++++++++--------------- 3 files changed, 58 insertions(+), 43 deletions(-) diff --git a/celery/contrib/rdb.py b/celery/contrib/rdb.py index bab9c8029..9b0f16c85 100644 --- a/celery/contrib/rdb.py +++ b/celery/contrib/rdb.py @@ -132,13 +132,23 @@ def get_avail_port(self, host, port, search_limit=100, skew=+0): def say(self, m): print(m, file=self.out) + def __enter__(self): + return self + + def __exit__(self, *exc_info): + self._close_session() + def _close_session(self): self.stdin, self.stdout = sys.stdin, sys.stdout = self._prev_handles - self._handle.close() - self._client.close() - self._sock.close() - self.active = False - self.say(SESSION_ENDED.format(self=self)) + if self.active: + if self._handle is not None: + self._handle.close() + if self._client is not None: + self._client.close() + if self._sock is not None: + self._sock.close() + self.active = False + self.say(SESSION_ENDED.format(self=self)) def do_continue(self, arg): self._close_session() diff --git a/celery/tests/case.py b/celery/tests/case.py index 0901c97b4..580ca957f 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -452,11 +452,11 @@ def _teardown_app(self): assert sys.__stdout__ assert sys.__stderr__ this = self._get_test_name() - if isinstance(sys.stdout, LoggingProxy) or \ - isinstance(sys.__stdout__, LoggingProxy): + if isinstance(sys.stdout, (LoggingProxy, Mock)) or \ + isinstance(sys.__stdout__, (LoggingProxy, Mock)): raise RuntimeError(CASE_LOG_REDIRECT_EFFECT.format(this, 'stdout')) - if isinstance(sys.stderr, LoggingProxy) or \ - isinstance(sys.__stderr__, LoggingProxy): + if isinstance(sys.stderr, (LoggingProxy, Mock)) or \ + isinstance(sys.__stderr__, (LoggingProxy, Mock)): raise RuntimeError(CASE_LOG_REDIRECT_EFFECT.format(this, 'stderr')) backend = self.app.__dict__.get('backend') if backend is not None: diff --git a/celery/tests/contrib/test_rdb.py b/celery/tests/contrib/test_rdb.py index 1fa398b81..38ac40fc8 100644 --- a/celery/tests/contrib/test_rdb.py +++ b/celery/tests/contrib/test_rdb.py @@ -8,14 +8,14 @@ debugger, set_trace, ) -from celery.tests.case import Case, Mock, WhateverIO, patch, skip_if_pypy +from celery.tests.case import AppCase, Mock, WhateverIO, patch, skip_if_pypy class SockErr(socket.error): errno = None -class test_Rdb(Case): +class test_Rdb(AppCase): @patch('celery.contrib.rdb.Rdb') def test_debugger(self, Rdb): @@ -37,56 +37,60 @@ def test_rdb(self, get_avail_port): get_avail_port.return_value = (sock, 8000) sock.accept.return_value = (Mock(), ['helu']) out = WhateverIO() - rdb = Rdb(out=out) - self.assertTrue(get_avail_port.called) - self.assertIn('helu', out.getvalue()) - - # set_quit - with patch('sys.settrace') as settrace: - rdb.set_quit() - settrace.assert_called_with(None) - - # set_trace - with patch('celery.contrib.rdb.Pdb.set_trace') as pset: - with patch('celery.contrib.rdb._frame'): - rdb.set_trace() - rdb.set_trace(Mock()) - pset.side_effect = SockErr - pset.side_effect.errno = errno.ENOENT - with self.assertRaises(SockErr): + with Rdb(out=out) as rdb: + self.assertTrue(get_avail_port.called) + self.assertIn('helu', out.getvalue()) + + # set_quit + with patch('sys.settrace') as settrace: + rdb.set_quit() + settrace.assert_called_with(None) + + # set_trace + with patch('celery.contrib.rdb.Pdb.set_trace') as pset: + with patch('celery.contrib.rdb._frame'): rdb.set_trace() + rdb.set_trace(Mock()) + pset.side_effect = SockErr + pset.side_effect.errno = errno.ENOENT + with self.assertRaises(SockErr): + rdb.set_trace() - # _close_session - rdb._close_session() + # _close_session + rdb._close_session() - # do_continue - rdb.set_continue = Mock() - rdb.do_continue(Mock()) - rdb.set_continue.assert_called_with() + # do_continue + rdb.set_continue = Mock() + rdb.do_continue(Mock()) + rdb.set_continue.assert_called_with() - # do_quit - rdb.set_quit = Mock() - rdb.do_quit(Mock()) - rdb.set_quit.assert_called_with() + # do_quit + rdb.set_quit = Mock() + rdb.do_quit(Mock()) + rdb.set_quit.assert_called_with() @patch('socket.socket') @skip_if_pypy def test_get_avail_port(self, sock): out = WhateverIO() sock.return_value.accept.return_value = (Mock(), ['helu']) - Rdb(out=out) + with Rdb(out=out) as rdb: + pass with patch('celery.contrib.rdb.current_process') as curproc: curproc.return_value.name = 'PoolWorker-10' - Rdb(out=out) + with Rdb(out=out) as rdb: + pass err = sock.return_value.bind.side_effect = SockErr() err.errno = errno.ENOENT with self.assertRaises(SockErr): - Rdb(out=out) + with Rdb(out=out) as rdb: + pass err.errno = errno.EADDRINUSE with self.assertRaises(Exception): - Rdb(out=out) + with Rdb(out=out) as rdb: + pass called = [0] def effect(*a, **kw): @@ -97,4 +101,5 @@ def effect(*a, **kw): finally: called[0] += 1 sock.return_value.bind.side_effect = effect - Rdb(out=out) + with Rdb(out=out) as rdb: + pass From a9078b55fb2ecfd3868055cc0572f8a023dbdc80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Fri, 6 Nov 2015 12:28:16 +0100 Subject: [PATCH 0798/1103] that one got lost --- celery/backends/cassandra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index 39c476883..47b1daf83 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -1,6 +1,6 @@ # -* coding: utf-8 -*- """ - celery.backends.new_cassandra + celery.backends.cassandra ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Apache Cassandra result store backend using DataStax driver From 45c0ed347b8b03664a49a16c953a930e9b4e1829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= Date: Fri, 6 Nov 2015 12:30:30 +0100 Subject: [PATCH 0799/1103] fixes formatting --- celery/backends/cassandra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index 47b1daf83..631c104b7 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -1,7 +1,7 @@ # -* coding: utf-8 -*- """ celery.backends.cassandra - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ~~~~~~~~~~~~~~~~~~~~~~~~~ Apache Cassandra result store backend using DataStax driver From a19f08033c5198e0a2dd478caef7394488b4b0e4 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 6 Nov 2015 12:03:54 -0800 Subject: [PATCH 0800/1103] Renames Python2/Python3 -> Python 2/Python 3 --- celery/app/amqp.py | 2 +- celery/canvas.py | 2 +- celery/utils/saferepr.py | 2 +- celery/utils/timeutils.py | 2 +- docs/whatsnew-4.0.rst | 15 +++++++++------ 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 8d94d7f55..3b1c163a8 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -34,7 +34,7 @@ PY3 = sys.version_info[0] == 3 -# json in Python2.7 borks if dict contains byte keys. +# json in Python 2.7 borks if dict contains byte keys. JSON_NEEDS_UNICODE_KEYS = not PY3 and not try_import('simplejson') #: Human readable queue declaration. diff --git a/celery/canvas.py b/celery/canvas.py index 779fe715f..f4c4ade41 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -36,7 +36,7 @@ PY3 = sys.version_info[0] == 3 -# json in Python2.7 borks if dict contains byte keys. +# json in Python 2.7 borks if dict contains byte keys. JSON_NEEDS_UNICODE_KEYS = PY3 and not try_import('simplejson') diff --git a/celery/utils/saferepr.py b/celery/utils/saferepr.py index 57e6cb0b4..090369b9d 100644 --- a/celery/utils/saferepr.py +++ b/celery/utils/saferepr.py @@ -9,7 +9,7 @@ - Sets are represented the Python 3 way: ``{1, 2}`` vs ``set([1, 2])``. - Unicode strings does not have the ``u'`` prefix, even on Python 2. - - Empty set formatted as ``set()`` (Python3), not ``set([])`` (Python2). + - Empty set formatted as ``set()`` (Python 3), not ``set([])`` (Python 2). - Longs do not have the ``L`` suffix. Very slow with no limits, super quick with limits. diff --git a/celery/utils/timeutils.py b/celery/utils/timeutils.py index d1e324c08..708f57a9d 100644 --- a/celery/utils/timeutils.py +++ b/celery/utils/timeutils.py @@ -90,7 +90,7 @@ def tzname(self, dt): def fromutc(self, dt): # The base tzinfo class no longer implements a DST - # offset aware .fromutc() in Python3 (Issue #2306). + # offset aware .fromutc() in Python 3 (Issue #2306). # I'd rather rely on pytz to do this, than port # the C code from cpython's fromutc [asksol] diff --git a/docs/whatsnew-4.0.rst b/docs/whatsnew-4.0.rst index 737ac7910..ddb2cc201 100644 --- a/docs/whatsnew-4.0.rst +++ b/docs/whatsnew-4.0.rst @@ -53,12 +53,15 @@ Important Notes Dropped support for Python 2.6 ------------------------------ -Celery now requires Python 2.7 or later. - -Dropped support for Python 3.3 ------------------------------- - -Celery now requires Python3 3.4 or later. +Celery now requires Python 2.7 or later, +and also drops support for Python 3.3 so supported versions are: + +- CPython 2.7 +- CPython 3.4 +- CPython 3.5 +- PyPy 4.0 (pypy2) +- PyPy 2.4 (pypy3) +- Jython 2.7.0 JSON is now the default serializer ---------------------------------- From b1deab39aad2fdec95f48b9f6e19ca1967285544 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 13 Nov 2015 18:36:56 -0800 Subject: [PATCH 0801/1103] Coverage one point up, long way to go --- .coveragerc | 3 +- celery/app/amqp.py | 36 +-- celery/backends/base.py | 12 +- celery/backends/redis.py | 2 +- celery/canvas.py | 12 +- celery/concurrency/asynpool.py | 2 +- celery/tests/app/test_amqp.py | 134 +++++++++++- celery/tests/app/test_app.py | 14 ++ celery/tests/app/test_builtins.py | 201 ++++------------- celery/tests/bin/test_celery.py | 11 + celery/tests/case.py | 77 +++++-- celery/tests/concurrency/test_eventlet.py | 13 ++ celery/tests/contrib/test_rdb.py | 10 +- celery/tests/tasks/test_canvas.py | 158 +++++++++++++- celery/tests/tasks/test_chord.py | 9 + celery/tests/worker/test_request.py | 255 ++++++++++++++++++---- celery/tests/worker/test_strategy.py | 91 +++++++- celery/utils/functional.py | 4 + celery/worker/request.py | 6 +- celery/worker/strategy.py | 8 +- 20 files changed, 793 insertions(+), 265 deletions(-) diff --git a/.coveragerc b/.coveragerc index 6e1334bda..c26f8646e 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,7 @@ [run] branch = 1 cover_pylib = 0 -omit = celery.utils.debug,celery.tests.*,celery.bin.graph +include=*celery/* +omit = celery.utils.debug,celery.tests.*,celery.bin.graph; [report] omit = */python?.?/*,*/site-packages/*,*/pypy/* diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 3b1c163a8..c87f454e8 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -26,7 +26,7 @@ from celery.local import try_import from celery.utils.saferepr import saferepr from celery.utils.text import indent as textindent -from celery.utils.timeutils import to_utc +from celery.utils.timeutils import maybe_make_aware, to_utc from . import routes as _routes @@ -300,7 +300,6 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, shadow=None, chain=None, now=None, timezone=None): args = args or () kwargs = kwargs or {} - utc = self.utc if not isinstance(args, (list, tuple)): raise TypeError('task args must be a list or tuple') if not isinstance(kwargs, Mapping): @@ -308,22 +307,22 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, if countdown: # convert countdown to ETA now = now or self.app.now() timezone = timezone or self.app.timezone - eta = now + timedelta(seconds=countdown) - if utc: - eta = to_utc(eta).astimezone(timezone) + eta = maybe_make_aware( + now + timedelta(seconds=countdown), tz=timezone, + ) if isinstance(expires, numbers.Real): now = now or self.app.now() timezone = timezone or self.app.timezone - expires = now + timedelta(seconds=expires) - if utc: - expires = to_utc(expires).astimezone(timezone) + expires = maybe_make_aware( + now + timedelta(seconds=expires), tz=timezone, + ) eta = eta and eta.isoformat() expires = expires and expires.isoformat() argsrepr = saferepr(args) kwargsrepr = saferepr(kwargs) - if JSON_NEEDS_UNICODE_KEYS: + if JSON_NEEDS_UNICODE_KEYS: # pragma: no cover if callbacks: callbacks = [utf8dict(callback) for callback in callbacks] if errbacks: @@ -400,7 +399,7 @@ def as_task_v1(self, task_id, name, args=None, kwargs=None, eta = eta and eta.isoformat() expires = expires and expires.isoformat() - if JSON_NEEDS_UNICODE_KEYS: + if JSON_NEEDS_UNICODE_KEYS: # pragma: no cover if callbacks: callbacks = [utf8dict(callback) for callback in callbacks] if errbacks: @@ -462,12 +461,13 @@ def _create_task_sender(self): default_serializer = self.app.conf.task_serializer default_compressor = self.app.conf.result_compression - def publish_task(producer, name, message, - exchange=None, routing_key=None, queue=None, - event_dispatcher=None, retry=None, retry_policy=None, - serializer=None, delivery_mode=None, - compression=None, declare=None, - headers=None, **kwargs): + def send_task_message(producer, name, message, + exchange=None, routing_key=None, queue=None, + event_dispatcher=None, + retry=None, retry_policy=None, + serializer=None, delivery_mode=None, + compression=None, declare=None, + headers=None, **kwargs): retry = default_retry if retry is None else retry headers2, properties, body, sent_event = message if headers: @@ -527,7 +527,7 @@ def publish_task(producer, name, message, if sent_event: evd = event_dispatcher or default_evd exname = exchange or self.exchange - if isinstance(name, Exchange): + if isinstance(exname, Exchange): exname = exname.name sent_event.update({ 'queue': qname, @@ -537,7 +537,7 @@ def publish_task(producer, name, message, evd.publish('task-sent', sent_event, self, retry=retry, retry_policy=retry_policy) return ret - return publish_task + return send_task_message @cached_property def default_queue(self): diff --git a/celery/backends/base.py b/celery/backends/base.py index 3f96fc5b2..ba7f014c5 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -110,13 +110,13 @@ def __init__(self, app, def mark_as_started(self, task_id, **meta): """Mark a task as started""" - return self.store_result(task_id, meta, status=states.STARTED) + return self.store_result(task_id, meta, states.STARTED) def mark_as_done(self, task_id, result, request=None, store_result=True, state=states.SUCCESS): """Mark task as successfully executed.""" if store_result: - self.store_result(task_id, result, status=state, request=request) + self.store_result(task_id, result, state, request=request) if request and request.chord: self.on_chord_part_return(request, state, result) @@ -125,7 +125,7 @@ def mark_as_failure(self, task_id, exc, state=states.FAILURE): """Mark task as executed with failure. Stores the exception.""" if store_result: - self.store_result(task_id, exc, status=state, + self.store_result(task_id, exc, state, traceback=traceback, request=request) if request and request.chord: self.on_chord_part_return(request, state, exc) @@ -134,8 +134,8 @@ def mark_as_revoked(self, task_id, reason='', request=None, store_result=True, state=states.REVOKED): exc = TaskRevokedError(reason) if store_result: - self.store_result(task_id, exc, - status=state, traceback=None, request=request) + self.store_result(task_id, exc, state, + traceback=None, request=request) if request and request.chord: self.on_chord_part_return(request, state, exc) @@ -143,7 +143,7 @@ def mark_as_retry(self, task_id, exc, traceback=None, request=None, store_result=True, state=states.RETRY): """Mark task as being retries. Stores the current exception (if any).""" - return self.store_result(task_id, exc, status=state, + return self.store_result(task_id, exc, state, traceback=traceback, request=request) def chord_error_from_stack(self, callback, exc=None): diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 486a4bbec..3af35cd96 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -17,7 +17,7 @@ from celery.canvas import maybe_signature from celery.exceptions import ChordError, ImproperlyConfigured from celery.five import string_t -from celery.utils import deprecated_property, strtobool +from celery.utils import deprecated_property from celery.utils.functional import dictfilter from celery.utils.log import get_logger from celery.utils.timeutils import humanize_seconds diff --git a/celery/canvas.py b/celery/canvas.py index f4c4ade41..1367a633c 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -27,7 +27,7 @@ from celery.result import GroupResult from celery.utils import abstract from celery.utils.functional import ( - maybe_list, is_list, noop, regen, chunks as _chunks, + maybe_list, is_list, regen, chunks as _chunks, ) from celery.utils.text import truncate @@ -457,7 +457,7 @@ def prepare_steps(self, args, tasks, steps_pop = steps.pop steps_extend = steps.extend - next_step = prev_task = prev_prev_task = None + prev_task = None prev_res = prev_prev_res = None tasks, results = [], [] i = 0 @@ -490,7 +490,7 @@ def prepare_steps(self, args, tasks, prev_res = prev_prev_res task = chord( task, body=prev_task, - task_id=res.task_id, root_id=root_id, app=app, + task_id=prev_res.task_id, root_id=root_id, app=app, ) if is_last_task: # chain(task_id=id) means task id is set for the last task @@ -526,8 +526,8 @@ def prepare_steps(self, args, tasks, tasks.append(task) results.append(res) - prev_prev_task, prev_task, prev_prev_res, prev_res = ( - prev_task, task, prev_res, res, + prev_task, prev_prev_res, prev_res = ( + task, prev_res, res, ) if root_id is None and tasks: @@ -701,7 +701,7 @@ def _prepared(self, tasks, partial_args, group_id, root_id, app, dict=dict, task = from_dict(task, app=app) if isinstance(task, group): # needs yield_from :( - unroll = task_prepared( + unroll = task._prepared( task.tasks, partial_args, group_id, root_id, app, ) for taskN, resN in unroll: diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 9aa819274..781370a16 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -33,7 +33,7 @@ from time import sleep from weakref import WeakValueDictionary, ref -from amqp.utils import promise +from amqp import promise from billiard.pool import RUN, TERMINATE, ACK, NACK, WorkersJoined from billiard import pool as _pool from billiard.compat import buf_t, setblocking, isblocking diff --git a/celery/tests/app/test_amqp.py b/celery/tests/app/test_amqp.py index 883e8603a..200182ba2 100644 --- a/celery/tests/app/test_amqp.py +++ b/celery/tests/app/test_amqp.py @@ -1,10 +1,15 @@ from __future__ import absolute_import +from datetime import datetime, timedelta + from kombu import Exchange, Queue -from celery.app.amqp import Queues +from celery import uuid +from celery.app.amqp import Queues, utf8dict from celery.five import keys -from celery.tests.case import AppCase +from celery.utils.timeutils import to_utc + +from celery.tests.case import AppCase, Mock class test_TaskConsumer(AppCase): @@ -146,6 +151,12 @@ def test_with_max_priority(self): 'x-max-priority': 3, }) + q1 = Queue('moo', queue_arguments=None) + qs1.add(q1) + self.assertEqual(qs1['moo'].queue_arguments, { + 'x-max-priority': 10, + }) + qs2 = Queues(ha_policy='all', max_priority=5) qs2.add('bar') self.assertEqual(qs2['bar'].queue_arguments, { @@ -169,3 +180,122 @@ def test_with_max_priority(self): self.assertEqual(qs3['xyx3'].queue_arguments, { 'x-max-priority': 7, }) + + +class test_AMQP(AppCase): + + def setup(self): + self.simple_message = self.app.amqp.as_task_v2( + uuid(), 'foo', create_sent_event=True, + ) + + def test_Queues__with_ha_policy(self): + x = self.app.amqp.Queues({}, ha_policy='all') + self.assertEqual(x.ha_policy, 'all') + + def test_Queues__with_max_priority(self): + x = self.app.amqp.Queues({}, max_priority=23) + self.assertEqual(x.max_priority, 23) + + def test_send_task_message__no_kwargs(self): + self.app.amqp.send_task_message(Mock(), 'foo', self.simple_message) + + def test_send_task_message__properties(self): + prod = Mock(name='producer') + self.app.amqp.send_task_message( + prod, 'foo', self.simple_message, foo=1, retry=False, + ) + self.assertEqual(prod.publish.call_args[1]['foo'], 1) + + def test_send_task_message__headers(self): + prod = Mock(name='producer') + self.app.amqp.send_task_message( + prod, 'foo', self.simple_message, headers={'x1x': 'y2x'}, + retry=False, + ) + self.assertEqual(prod.publish.call_args[1]['headers']['x1x'], 'y2x') + + def test_send_task_message__queue_string(self): + prod = Mock(name='producer') + self.app.amqp.send_task_message( + prod, 'foo', self.simple_message, queue='foo', retry=False, + ) + kwargs = prod.publish.call_args[1] + self.assertEqual(kwargs['routing_key'], 'foo') + self.assertEqual(kwargs['exchange'], 'foo') + + def test_send_event_exchange_string(self): + evd = Mock(name="evd") + self.app.amqp.send_task_message( + Mock(), 'foo', self.simple_message, retry=False, + exchange='xyz', routing_key='xyb', + event_dispatcher=evd, + ) + self.assertTrue(evd.publish.called) + event = evd.publish.call_args[0][1] + self.assertEqual(event['routing_key'], 'xyb') + self.assertEqual(event['exchange'], 'xyz') + + def test_send_task_message__with_delivery_mode(self): + prod = Mock(name='producer') + self.app.amqp.send_task_message( + prod, 'foo', self.simple_message, delivery_mode=33, retry=False, + ) + self.assertEqual(prod.publish.call_args[1]['delivery_mode'], 33) + + def test_routes(self): + r1 = self.app.amqp.routes + r2 = self.app.amqp.routes + self.assertIs(r1, r2) + + +class test_as_task_v2(AppCase): + + def test_raises_if_args_is_not_tuple(self): + with self.assertRaises(TypeError): + self.app.amqp.as_task_v2(uuid(), 'foo', args='123') + + def test_raises_if_kwargs_is_not_mapping(self): + with self.assertRaises(TypeError): + self.app.amqp.as_task_v2(uuid(), 'foo', kwargs=(1, 2, 3)) + + def test_countdown_to_eta(self): + now = to_utc(datetime.utcnow()).astimezone(self.app.timezone) + m = self.app.amqp.as_task_v2( + uuid(), 'foo', countdown=10, now=now, + ) + self.assertEqual( + m.headers['eta'], + (now + timedelta(seconds=10)).isoformat(), + ) + + def test_expires_to_datetime(self): + now = to_utc(datetime.utcnow()).astimezone(self.app.timezone) + m = self.app.amqp.as_task_v2( + uuid(), 'foo', expires=30, now=now, + ) + self.assertEqual( + m.headers['expires'], + (now + timedelta(seconds=30)).isoformat(), + ) + + def test_callbacks_errbacks_chord(self): + + @self.app.task + def t(i): + pass + + m = self.app.amqp.as_task_v2( + uuid(), 'foo', + callbacks=[t.s(1), t.s(2)], + errbacks=[t.s(3), t.s(4)], + chord=t.s(5), + ) + _, _, embed = m.body + self.assertListEqual( + embed['callbacks'], [utf8dict(t.s(1)), utf8dict(t.s(2))], + ) + self.assertListEqual( + embed['errbacks'], [utf8dict(t.s(3)), utf8dict(t.s(4))], + ) + self.assertEqual(embed['chord'], utf8dict(t.s(5))) diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index 85f0b3eb6..8d350d880 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -24,6 +24,7 @@ CELERY_TEST_CONFIG, AppCase, Mock, + Case, depends_on_current_app, mask_modules, patch, @@ -75,6 +76,19 @@ def test_bugreport(self): self.assertTrue(_app.bugreport(app=self.app)) +class test_task_join_will_block(Case): + + def test_task_join_will_block(self): + prev, _state._task_join_will_block = _state._task_join_will_block, 0 + try: + self.assertEqual(_state._task_join_will_block, 0) + _state._set_task_join_will_block(True) + print(_state.task_join_will_block) + self.assertTrue(_state.task_join_will_block()) + finally: + _state._task_join_will_block = prev + + class test_App(AppCase): def setup(self): diff --git a/celery/tests/app/test_builtins.py b/celery/tests/app/test_builtins.py index 7f7bac1e8..b6539935a 100644 --- a/celery/tests/app/test_builtins.py +++ b/celery/tests/app/test_builtins.py @@ -2,10 +2,10 @@ from celery import group, chord from celery.app import builtins -from celery.canvas import Signature from celery.five import range -from celery._state import _task_stack -from celery.tests.case import AppCase, Mock, patch +from celery.utils.functional import pass1 + +from celery.tests.case import AppCase, ContextMock, Mock, patch class BuiltinsCase(AppCase): @@ -32,6 +32,18 @@ def test_run(self): self.assertTrue(self.app.backend.cleanup.called) +class test_accumulate(BuiltinsCase): + + def setup(self): + self.accumulate = self.app.tasks['celery.accumulate'] + + def test_with_index(self): + self.assertEqual(self.accumulate(1, 2, 3, 4, index=0), 1) + + def test_no_index(self): + self.assertEqual(self.accumulate(1, 2, 3, 4), (1, 2, 3, 4)) + + class test_map(BuiltinsCase): def test_run(self): @@ -78,46 +90,42 @@ def chunks_mul(l): class test_group(BuiltinsCase): def setup(self): + self.maybe_signature = self.patch('celery.canvas.maybe_signature') + self.maybe_signature.side_effect = pass1 + self.app.producer_or_acquire = Mock() + self.app.producer_or_acquire.attach_mock(ContextMock(), 'return_value') + self.app.conf.task_always_eager = True self.task = builtins.add_group_task(self.app) super(test_group, self).setup() def test_apply_async_eager(self): - self.task.apply = Mock() - self.app.conf.task_always_eager = True + self.task.apply = Mock(name='apply') self.task.apply_async((1, 2, 3, 4, 5)) self.assertTrue(self.task.apply.called) - def test_apply(self): - x = group([self.add.s(4, 4), self.add.s(8, 8)]) - res = x.apply() - self.assertEqual(res.get(), [8, 16]) + def mock_group(self, *tasks): + g = group(*tasks, app=self.app) + result = g.freeze() + for task in g.tasks: + task.clone = Mock(name='clone') + task.clone.attach_mock(Mock(), 'apply_async') + return g, result + + @patch('celery.app.builtins.get_current_worker_task') + def test_task(self, get_current_worker_task): + g, result = self.mock_group(self.add.s(2), self.add.s(4)) + self.task(g.tasks, result, result.id, (2,)).results + g.tasks[0].clone().apply_async.assert_called_with( + group_id=result.id, producer=self.app.producer_or_acquire(), + add_to_parent=False, + ) + get_current_worker_task().add_trail.assert_called_with(result) - def test_apply_async(self): - x = group([self.add.s(4, 4), self.add.s(8, 8)]) - x.apply_async() - - def test_apply_empty(self): - x = group(app=self.app) - x.apply() - res = x.apply_async() - self.assertFalse(res) - self.assertFalse(res.results) - - def test_apply_async_with_parent(self): - _task_stack.push(self.add) - try: - self.add.push_request(called_directly=False) - try: - assert not self.add.request.children - x = group([self.add.s(4, 4), self.add.s(8, 8)]) - res = x() - self.assertTrue(self.add.request.children) - self.assertIn(res, self.add.request.children) - self.assertEqual(len(self.add.request.children), 1) - finally: - self.add.pop_request() - finally: - _task_stack.pop() + @patch('celery.app.builtins.get_current_worker_task') + def test_task__disable_add_to_parent(self, get_current_worker_task): + g, result = self.mock_group(self.add.s(2, 2), self.add.s(4, 4)) + self.task(g.tasks, result, result.id, None, add_to_parent=False) + self.assertFalse(get_current_worker_task().add_trail.called) class test_chain(BuiltinsCase): @@ -126,126 +134,9 @@ def setup(self): BuiltinsCase.setup(self) self.task = builtins.add_chain_task(self.app) - def test_apply_async(self): - c = self.add.s(2, 2) | self.add.s(4) | self.add.s(8) - result = c.apply_async() - self.assertTrue(result.parent) - self.assertTrue(result.parent.parent) - self.assertIsNone(result.parent.parent.parent) - - def test_group_to_chord__freeze_parent_id(self): - def using_freeze(c): - c.freeze(parent_id='foo', root_id='root') - return c._frozen[0] - self.assert_group_to_chord_parent_ids(using_freeze) - - def assert_group_to_chord_parent_ids(self, freezefun): - c = ( - self.add.s(5, 5) | - group([self.add.s(i, i) for i in range(5)], app=self.app) | - self.add.si(10, 10) | - self.add.si(20, 20) | - self.add.si(30, 30) - ) - tasks = freezefun(c) - self.assertEqual(tasks[-1].parent_id, 'foo') - self.assertEqual(tasks[-1].root_id, 'root') - self.assertEqual(tasks[-2].parent_id, tasks[-1].id) - self.assertEqual(tasks[-2].root_id, 'root') - self.assertEqual(tasks[-2].body.parent_id, tasks[-2].tasks.id) - self.assertEqual(tasks[-2].body.parent_id, tasks[-2].id) - self.assertEqual(tasks[-2].body.root_id, 'root') - self.assertEqual(tasks[-2].tasks.tasks[0].parent_id, tasks[-1].id) - self.assertEqual(tasks[-2].tasks.tasks[0].root_id, 'root') - self.assertEqual(tasks[-2].tasks.tasks[1].parent_id, tasks[-1].id) - self.assertEqual(tasks[-2].tasks.tasks[1].root_id, 'root') - self.assertEqual(tasks[-2].tasks.tasks[2].parent_id, tasks[-1].id) - self.assertEqual(tasks[-2].tasks.tasks[2].root_id, 'root') - self.assertEqual(tasks[-2].tasks.tasks[3].parent_id, tasks[-1].id) - self.assertEqual(tasks[-2].tasks.tasks[3].root_id, 'root') - self.assertEqual(tasks[-2].tasks.tasks[4].parent_id, tasks[-1].id) - self.assertEqual(tasks[-2].tasks.tasks[4].root_id, 'root') - self.assertEqual(tasks[-3].parent_id, tasks[-2].body.id) - self.assertEqual(tasks[-3].root_id, 'root') - self.assertEqual(tasks[-4].parent_id, tasks[-3].id) - self.assertEqual(tasks[-4].root_id, 'root') - - def test_group_to_chord(self): - c = ( - self.add.s(5) | - group([self.add.s(i, i) for i in range(5)], app=self.app) | - self.add.s(10) | - self.add.s(20) | - self.add.s(30) - ) - c._use_link = True - tasks, results = c.prepare_steps((), c.tasks) - - self.assertEqual(tasks[-1].args[0], 5) - self.assertIsInstance(tasks[-2], chord) - self.assertEqual(len(tasks[-2].tasks), 5) - self.assertEqual(tasks[-2].parent_id, tasks[-1].id) - self.assertEqual(tasks[-2].root_id, tasks[-1].id) - self.assertEqual(tasks[-2].body.args[0], 10) - self.assertEqual(tasks[-2].body.parent_id, tasks[-2].id) - - self.assertEqual(tasks[-3].args[0], 20) - self.assertEqual(tasks[-3].root_id, tasks[-1].id) - self.assertEqual(tasks[-3].parent_id, tasks[-2].body.id) - - self.assertEqual(tasks[-4].args[0], 30) - self.assertEqual(tasks[-4].parent_id, tasks[-3].id) - self.assertEqual(tasks[-4].root_id, tasks[-1].id) - - self.assertTrue(tasks[-2].body.options['link']) - self.assertTrue(tasks[-2].body.options['link'][0].options['link']) - - c2 = self.add.s(2, 2) | group(self.add.s(i, i) for i in range(10)) - c2._use_link = True - tasks2, _ = c2.prepare_steps((), c2.tasks) - self.assertIsInstance(tasks2[0], group) - - def test_group_to_chord__protocol_2(self): - c = ( - group([self.add.s(i, i) for i in range(5)], app=self.app) | - self.add.s(10) | - self.add.s(20) | - self.add.s(30) - ) - c._use_link = False - tasks, _ = c.prepare_steps((), c.tasks) - self.assertIsInstance(tasks[-1], chord) - - c2 = self.add.s(2, 2) | group(self.add.s(i, i) for i in range(10)) - c2._use_link = False - tasks2, _ = c2.prepare_steps((), c2.tasks) - self.assertIsInstance(tasks2[0], group) - def test_apply_options(self): - - class static(Signature): - - def clone(self, *args, **kwargs): - return self - - def s(*args, **kwargs): - return static(self.add, args, kwargs, type=self.add, app=self.app) - - c = s(2, 2) | s(4, 4) | s(8, 8) - r1 = c.apply_async(task_id='some_id') - self.assertEqual(r1.id, 'some_id') - - c.apply_async(group_id='some_group_id') - self.assertEqual(c.tasks[-1].options['group_id'], 'some_group_id') - - c.apply_async(chord='some_chord_id') - self.assertEqual(c.tasks[-1].options['chord'], 'some_chord_id') - - c.apply_async(link=[s(32)]) - self.assertListEqual(c.tasks[-1].options['link'], [s(32)]) - - c.apply_async(link_error=[s('error')]) - for task in c.tasks: - self.assertListEqual(task.options['link_error'], [s('error')]) + def test_not_implemented(self): + with self.assertRaises(NotImplementedError): + self.task() class test_chord(BuiltinsCase): diff --git a/celery/tests/bin/test_celery.py b/celery/tests/bin/test_celery.py index 4139750a2..196e4a4a5 100644 --- a/celery/tests/bin/test_celery.py +++ b/celery/tests/bin/test_celery.py @@ -41,6 +41,17 @@ def test_main(self): mpc.assert_called_with() main.assert_called_with() + def test_main__multi(self): + with patch('celery.__main__.maybe_patch_concurrency') as mpc: + with patch('celery.bin.celery.main') as main: + prev, sys.argv = sys.argv, ['foo', 'multi'] + try: + __main__.main() + self.assertFalse(mpc.called) + main.assert_called_with() + finally: + sys.argv = prev + class test_Command(AppCase): diff --git a/celery/tests/case.py b/celery/tests/case.py index 580ca957f..731159836 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -309,6 +309,12 @@ def alive_threads(): class Case(unittest.TestCase): + def patch(self, *path, **options): + manager = patch(".".join(path), **options) + patched = manager.start() + self.addCleanup(manager.stop) + return patched + def assertWarns(self, expected_warning): return _AssertWarnsContext(expected_warning, self, None) @@ -420,6 +426,8 @@ def setUp(self): self._threads_at_setup = self.threads_at_startup() from celery import _state from celery import result + self._prev_res_join_block = result.task_join_will_block + self._prev_state_join_block = _state.task_join_will_block result.task_join_will_block = \ _state.task_join_will_block = lambda: False self._current_app = current_app() @@ -446,12 +454,16 @@ class NonTLS(object): raise def _teardown_app(self): + from celery import _state + from celery import result from celery.utils.log import LoggingProxy assert sys.stdout assert sys.stderr assert sys.__stdout__ assert sys.__stderr__ this = self._get_test_name() + result.task_join_will_block = self._prev_res_join_block + _state.task_join_will_block = self._prev_state_join_block if isinstance(sys.stdout, (LoggingProxy, Mock)) or \ isinstance(sys.__stdout__, (LoggingProxy, Mock)): raise RuntimeError(CASE_LOG_REDIRECT_EFFECT.format(this, 'stdout')) @@ -839,7 +851,49 @@ def _inner(*args, **kwargs): return _inner -def task_message_from_sig(app, sig, utc=True): +def TaskMessage(name, id=None, args=(), kwargs={}, callbacks=None, + errbacks=None, chain=None, shadow=None, utc=None, **options): + from celery import uuid + from kombu.serialization import dumps + id = id or uuid() + message = Mock(name='TaskMessage-{0}'.format(id)) + message.headers = { + 'id': id, + 'task': name, + 'shadow': shadow, + } + embed = {'callbacks': callbacks, 'errbacks': errbacks, 'chain': chain} + message.headers.update(options) + message.content_type, message.content_encoding, message.body = dumps( + (args, kwargs, embed), serializer='json', + ) + message.payload = (args, kwargs, embed) + return message + + +def TaskMessage1(name, id=None, args=(), kwargs={}, callbacks=None, + errbacks=None, chain=None, **options): + from celery import uuid + from kombu.serialization import dumps + id = id or uuid() + message = Mock(name='TaskMessage-{0}'.format(id)) + message.headers = {} + message.payload = { + 'task': name, + 'id': id, + 'args': args, + 'kwargs': kwargs, + 'callbacks': callbacks, + 'errbacks': errbacks, + } + message.payload.update(options) + message.content_type, message.content_encoding, message.body = dumps( + message.payload, + ) + return message + + +def task_message_from_sig(app, sig, utc=True, TaskMessage=TaskMessage): sig.freeze() callbacks = sig.options.pop('link', None) errbacks = sig.options.pop('link_error', None) @@ -862,6 +916,8 @@ def task_message_from_sig(app, sig, utc=True): errbacks=[dict(s) for s in errbacks] if errbacks else None, eta=eta, expires=expires, + utc=utc, + **sig.options ) @@ -878,22 +934,3 @@ def restore_logging(): sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__ = outs root.level = level root.handlers[:] = handlers - - -def TaskMessage(name, id=None, args=(), kwargs={}, callbacks=None, - errbacks=None, chain=None, **options): - from celery import uuid - from kombu.serialization import dumps - id = id or uuid() - message = Mock(name='TaskMessage-{0}'.format(id)) - message.headers = { - 'id': id, - 'task': name, - } - embed = {'callbacks': callbacks, 'errbacks': errbacks, 'chain': chain} - message.headers.update(options) - message.content_type, message.content_encoding, message.body = dumps( - (args, kwargs, embed), serializer='json', - ) - message.payload = (args, kwargs, embed) - return message diff --git a/celery/tests/concurrency/test_eventlet.py b/celery/tests/concurrency/test_eventlet.py index d9447f46c..9761a84db 100644 --- a/celery/tests/concurrency/test_eventlet.py +++ b/celery/tests/concurrency/test_eventlet.py @@ -1,5 +1,6 @@ from __future__ import absolute_import +import os import sys from celery.app.defaults import is_pypy @@ -43,6 +44,18 @@ def test_aaa_is_patched(self): maybe_patch_concurrency(['x', '-P', 'eventlet']) monkey_patch.assert_called_with() + @patch('eventlet.debug.hub_blocking_detection', create=True) + @patch('eventlet.monkey_patch', create=True) + def test_aaa_blockdetecet(self, monkey_patch, hub_blocking_detection): + os.environ['EVENTLET_NOBLOCK'] = "10.3" + try: + from celery import maybe_patch_concurrency + maybe_patch_concurrency(['x', '-P', 'eventlet']) + monkey_patch.assert_called_with() + hub_blocking_detection.assert_called_with(10.3, 10.3) + finally: + os.environ.pop('EVENTLET_NOBLOCK', None) + eventlet_modules = ( 'eventlet', diff --git a/celery/tests/contrib/test_rdb.py b/celery/tests/contrib/test_rdb.py index 38ac40fc8..26b3a5498 100644 --- a/celery/tests/contrib/test_rdb.py +++ b/celery/tests/contrib/test_rdb.py @@ -74,22 +74,22 @@ def test_rdb(self, get_avail_port): def test_get_avail_port(self, sock): out = WhateverIO() sock.return_value.accept.return_value = (Mock(), ['helu']) - with Rdb(out=out) as rdb: + with Rdb(out=out): pass with patch('celery.contrib.rdb.current_process') as curproc: curproc.return_value.name = 'PoolWorker-10' - with Rdb(out=out) as rdb: + with Rdb(out=out): pass err = sock.return_value.bind.side_effect = SockErr() err.errno = errno.ENOENT with self.assertRaises(SockErr): - with Rdb(out=out) as rdb: + with Rdb(out=out): pass err.errno = errno.EADDRINUSE with self.assertRaises(Exception): - with Rdb(out=out) as rdb: + with Rdb(out=out): pass called = [0] @@ -101,5 +101,5 @@ def effect(*a, **kw): finally: called[0] += 1 sock.return_value.bind.side_effect = effect - with Rdb(out=out) as rdb: + with Rdb(out=out): pass diff --git a/celery/tests/tasks/test_canvas.py b/celery/tests/tasks/test_canvas.py index 9a22515af..6855aad82 100644 --- a/celery/tests/tasks/test_canvas.py +++ b/celery/tests/tasks/test_canvas.py @@ -1,5 +1,6 @@ from __future__ import absolute_import +from celery._state import _task_stack from celery.canvas import ( Signature, chain, @@ -210,6 +211,128 @@ def test_repr(self): repr(x), '%s(2, 2) | %s(2)' % (self.add.name, self.add.name), ) + def test_apply_async(self): + c = self.add.s(2, 2) | self.add.s(4) | self.add.s(8) + result = c.apply_async() + self.assertTrue(result.parent) + self.assertTrue(result.parent.parent) + self.assertIsNone(result.parent.parent.parent) + + def test_group_to_chord__freeze_parent_id(self): + def using_freeze(c): + c.freeze(parent_id='foo', root_id='root') + return c._frozen[0] + self.assert_group_to_chord_parent_ids(using_freeze) + + def assert_group_to_chord_parent_ids(self, freezefun): + c = ( + self.add.s(5, 5) | + group([self.add.s(i, i) for i in range(5)], app=self.app) | + self.add.si(10, 10) | + self.add.si(20, 20) | + self.add.si(30, 30) + ) + tasks = freezefun(c) + self.assertEqual(tasks[-1].parent_id, 'foo') + self.assertEqual(tasks[-1].root_id, 'root') + self.assertEqual(tasks[-2].parent_id, tasks[-1].id) + self.assertEqual(tasks[-2].root_id, 'root') + self.assertEqual(tasks[-2].body.parent_id, tasks[-2].tasks.id) + self.assertEqual(tasks[-2].body.parent_id, tasks[-2].id) + self.assertEqual(tasks[-2].body.root_id, 'root') + self.assertEqual(tasks[-2].tasks.tasks[0].parent_id, tasks[-1].id) + self.assertEqual(tasks[-2].tasks.tasks[0].root_id, 'root') + self.assertEqual(tasks[-2].tasks.tasks[1].parent_id, tasks[-1].id) + self.assertEqual(tasks[-2].tasks.tasks[1].root_id, 'root') + self.assertEqual(tasks[-2].tasks.tasks[2].parent_id, tasks[-1].id) + self.assertEqual(tasks[-2].tasks.tasks[2].root_id, 'root') + self.assertEqual(tasks[-2].tasks.tasks[3].parent_id, tasks[-1].id) + self.assertEqual(tasks[-2].tasks.tasks[3].root_id, 'root') + self.assertEqual(tasks[-2].tasks.tasks[4].parent_id, tasks[-1].id) + self.assertEqual(tasks[-2].tasks.tasks[4].root_id, 'root') + self.assertEqual(tasks[-3].parent_id, tasks[-2].body.id) + self.assertEqual(tasks[-3].root_id, 'root') + self.assertEqual(tasks[-4].parent_id, tasks[-3].id) + self.assertEqual(tasks[-4].root_id, 'root') + + def test_group_to_chord(self): + c = ( + self.add.s(5) | + group([self.add.s(i, i) for i in range(5)], app=self.app) | + self.add.s(10) | + self.add.s(20) | + self.add.s(30) + ) + c._use_link = True + tasks, results = c.prepare_steps((), c.tasks) + + self.assertEqual(tasks[-1].args[0], 5) + self.assertIsInstance(tasks[-2], chord) + self.assertEqual(len(tasks[-2].tasks), 5) + self.assertEqual(tasks[-2].parent_id, tasks[-1].id) + self.assertEqual(tasks[-2].root_id, tasks[-1].id) + self.assertEqual(tasks[-2].body.args[0], 10) + self.assertEqual(tasks[-2].body.parent_id, tasks[-2].id) + + self.assertEqual(tasks[-3].args[0], 20) + self.assertEqual(tasks[-3].root_id, tasks[-1].id) + self.assertEqual(tasks[-3].parent_id, tasks[-2].body.id) + + self.assertEqual(tasks[-4].args[0], 30) + self.assertEqual(tasks[-4].parent_id, tasks[-3].id) + self.assertEqual(tasks[-4].root_id, tasks[-1].id) + + self.assertTrue(tasks[-2].body.options['link']) + self.assertTrue(tasks[-2].body.options['link'][0].options['link']) + + c2 = self.add.s(2, 2) | group(self.add.s(i, i) for i in range(10)) + c2._use_link = True + tasks2, _ = c2.prepare_steps((), c2.tasks) + self.assertIsInstance(tasks2[0], group) + + def test_group_to_chord__protocol_2(self): + c = ( + group([self.add.s(i, i) for i in range(5)], app=self.app) | + self.add.s(10) | + self.add.s(20) | + self.add.s(30) + ) + c._use_link = False + tasks, _ = c.prepare_steps((), c.tasks) + self.assertIsInstance(tasks[-1], chord) + + c2 = self.add.s(2, 2) | group(self.add.s(i, i) for i in range(10)) + c2._use_link = False + tasks2, _ = c2.prepare_steps((), c2.tasks) + self.assertIsInstance(tasks2[0], group) + + def test_apply_options(self): + + class static(Signature): + + def clone(self, *args, **kwargs): + return self + + def s(*args, **kwargs): + return static(self.add, args, kwargs, type=self.add, app=self.app) + + c = s(2, 2) | s(4, 4) | s(8, 8) + r1 = c.apply_async(task_id='some_id') + self.assertEqual(r1.id, 'some_id') + + c.apply_async(group_id='some_group_id') + self.assertEqual(c.tasks[-1].options['group_id'], 'some_group_id') + + c.apply_async(chord='some_chord_id') + self.assertEqual(c.tasks[-1].options['chord'], 'some_chord_id') + + c.apply_async(link=[s(32)]) + self.assertListEqual(c.tasks[-1].options['link'], [s(32)]) + + c.apply_async(link_error=[s('error')]) + for task in c.tasks: + self.assertListEqual(task.options['link_error'], [s('error')]) + def test_reverse(self): x = self.add.s(2, 2) | self.add.s(2) self.assertIsInstance(signature(x), chain) @@ -255,13 +378,12 @@ def test_root_id_parent_id(self): self.assert_sent_with_ids(tasks[-3], tasks[-1].id, tasks[-2].id) self.assert_sent_with_ids(tasks[-4], tasks[-1].id, tasks[-3].id) - def assert_sent_with_ids(self, task, rid, pid, **options): self.app.amqp.send_task_message = Mock(name='send_task_message') self.app.backend = Mock() self.app.producer_or_acquire = ContextMock() - res = task.apply_async(**options) + task.apply_async(**options) self.assertTrue(self.app.amqp.send_task_message.called) message = self.app.amqp.send_task_message.call_args[0][2] self.assertEqual(message.headers['parent_id'], pid) @@ -306,6 +428,38 @@ def test_maybe_group_sig(self): _maybe_group(self.add.s(2, 2), self.app), [self.add.s(2, 2)], ) + def test_apply(self): + x = group([self.add.s(4, 4), self.add.s(8, 8)]) + res = x.apply() + self.assertEqual(res.get(), [8, 16]) + + def test_apply_async(self): + x = group([self.add.s(4, 4), self.add.s(8, 8)]) + x.apply_async() + + def test_apply_empty(self): + x = group(app=self.app) + x.apply() + res = x.apply_async() + self.assertFalse(res) + self.assertFalse(res.results) + + def test_apply_async_with_parent(self): + _task_stack.push(self.add) + try: + self.add.push_request(called_directly=False) + try: + assert not self.add.request.children + x = group([self.add.s(4, 4), self.add.s(8, 8)]) + res = x() + self.assertTrue(self.add.request.children) + self.assertIn(res, self.add.request.children) + self.assertEqual(len(self.add.request.children), 1) + finally: + self.add.pop_request() + finally: + _task_stack.pop() + def test_from_dict(self): x = group([self.add.s(2, 2), self.add.s(4, 4)]) x['args'] = (2, 2) diff --git a/celery/tests/tasks/test_chord.py b/celery/tests/tasks/test_chord.py index e458213a6..d5e243101 100644 --- a/celery/tests/tasks/test_chord.py +++ b/celery/tests/tasks/test_chord.py @@ -79,6 +79,15 @@ class AlwaysReady(TSR): # did not retry self.assertFalse(retry.call_count) + def test_deps_ready_fails(self): + GroupResult = Mock(name='GroupResult') + GroupResult.return_value.ready.side_effect = KeyError('foo') + unlock_chord = self.app.tasks['celery.chord_unlock'] + + with self.assertRaises(KeyError): + unlock_chord('groupid', Mock(), result=[Mock()], + GroupResult=GroupResult, result_from_tuple=Mock()) + def test_callback_fails(self): class AlwaysReady(TSR): diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index 72ab9c7ce..01a0941f2 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -36,7 +36,9 @@ from celery.signals import task_revoked from celery.utils import uuid from celery.worker import request as module -from celery.worker.request import Request, logger as req_logger +from celery.worker.request import ( + Request, create_request_cls, logger as req_logger, +) from celery.worker.state import revoked from celery.tests.case import ( @@ -51,6 +53,39 @@ ) +class RequestCase(AppCase): + + def setup(self): + self.app.conf.result_serializer = 'pickle' + + @self.app.task(shared=False) + def add(x, y, **kw_): + return x + y + self.add = add + + @self.app.task(shared=False) + def mytask(i, **kwargs): + return i ** i + self.mytask = mytask + + @self.app.task(shared=False) + def mytask_raising(i): + raise KeyError(i) + self.mytask_raising = mytask_raising + + def xRequest(self, name=None, id=None, args=None, kwargs=None, + on_ack=None, on_reject=None, Request=Request, **head): + args = [1] if args is None else args + kwargs = {'f': 'x'} if kwargs is None else kwargs + on_ack = on_ack or Mock(name='on_ack') + on_reject = on_reject or Mock(name='on_reject') + message = TaskMessage( + name or self.mytask.name, id, args=args, kwargs=kwargs, **head + ) + return Request(message, app=self.app, + on_ack=on_ack, on_reject=on_reject) + + class test_mro_lookup(Case): def test_order(self): @@ -125,7 +160,7 @@ def test_retry_semipredicate(self): self.assertEqual(ret.exc, exc) -class test_trace_task(AppCase): +class test_trace_task(RequestCase): def setup(self): @@ -162,7 +197,7 @@ def test_execute_jail_success(self): def test_marked_as_started(self): _started = [] - def store_result(tid, meta, state, **kwars): + def store_result(tid, meta, state, **kwargs): if state == states.STARTED: _started.append(tid) self.mytask.backend.store_result = Mock(name='store_result') @@ -207,25 +242,7 @@ def send(self, event, **fields): self.sent.append(event) -class test_Request(AppCase): - - def setup(self): - self.app.conf.result_serializer = 'pickle' - - @self.app.task(shared=False) - def add(x, y, **kw_): - return x + y - self.add = add - - @self.app.task(shared=False) - def mytask(i, **kwargs): - return i ** i - self.mytask = mytask - - @self.app.task(shared=False) - def mytask_raising(i): - raise KeyError(i) - self.mytask_raising = mytask_raising +class test_Request(RequestCase): def get_request(self, sig, Request=Request, **kwargs): return Request( @@ -239,6 +256,12 @@ def get_request(self, sig, Request=Request, **kwargs): **kwargs ) + def test_shadow(self): + self.assertEqual( + self.get_request(self.add.s(2, 2).set(shadow='fooxyz')).name, + 'fooxyz', + ) + def test_invalid_eta_raises_InvalidTaskError(self): with self.assertRaises(InvalidTaskError): self.get_request(self.add.s(2, 2).set(eta='12345')) @@ -358,18 +381,6 @@ def test_tzlocal_is_cached(self): req._tzlocal = 'foo' self.assertEqual(req.tzlocal, 'foo') - def xRequest(self, name=None, id=None, args=None, kwargs=None, - on_ack=None, on_reject=None, **head): - args = [1] if args is None else args - kwargs = {'f': 'x'} if kwargs is None else kwargs - on_ack = on_ack or Mock(name='on_ack') - on_reject = on_reject or Mock(name='on_reject') - message = TaskMessage( - name or self.mytask.name, id, args=args, kwargs=kwargs, **head - ) - return Request(message, app=self.app, - on_ack=on_ack, on_reject=on_reject) - def test_task_wrapper_repr(self): self.assertTrue(repr(self.xRequest())) @@ -414,6 +425,23 @@ def test_compat_properties(self): job.task_name = 'NAME' self.assertEqual(job.name, 'NAME') + def test_terminate__pool_ref(self): + pool = Mock() + signum = signal.SIGTERM + job = self.get_request(self.mytask.s(1, f='x')) + job._apply_result = Mock(name='_apply_result') + with assert_signal_called( + task_revoked, sender=job.task, request=job, + terminated=True, expired=False, signum=signum): + job.time_start = monotonic() + job.worker_pid = 314 + job.terminate(pool, signal='TERM') + job._apply_result().terminate.assert_called_with(signum) + + job._apply_result = Mock(name='_apply_result2') + job._apply_result.return_value = None + job.terminate(pool, signal='TERM') + def test_terminate__task_started(self): pool = Mock() signum = signal.SIGTERM @@ -627,6 +655,8 @@ def test_from_message_invalid_kwargs(self): def test_on_timeout(self, warn, error): job = self.xRequest() + job.acknowledge = Mock(name='ack') + job.task.acks_late = True job.on_timeout(soft=True, timeout=1337) self.assertIn('Soft time limit', warn.call_args[0][0]) job.on_timeout(soft=False, timeout=1337) @@ -634,6 +664,7 @@ def test_on_timeout(self, warn, error): self.assertEqual( self.mytask.backend.get_status(job.id), states.FAILURE, ) + job.acknowledge.assert_called_with() self.mytask.ignore_result = True job = self.xRequest() @@ -642,6 +673,12 @@ def test_on_timeout(self, warn, error): self.mytask.backend.get_status(job.id), states.PENDING, ) + job = self.xRequest() + job.acknowledge = Mock(name='ack') + job.task.acks_late = False + job.on_timeout(soft=True, timeout=1335) + self.assertFalse(job.acknowledge.called) + def test_fast_trace_task(self): from celery.app import trace setup_worker_optimizations(self.app) @@ -874,23 +911,163 @@ def apply_async(self, target, args=None, kwargs=None, self.assertEqual(p.args[1], tid) self.assertEqual(p.args[3], job.message.body) - def _test_on_failure(self, exception): + def _test_on_failure(self, exception, **kwargs): tid = uuid() job = self.xRequest(id=tid, args=[4]) job.send_event = Mock(name='send_event') + job.task.backend.mark_as_failure = Mock(name='mark_as_failure') try: raise exception - except Exception: + except type(exception): exc_info = ExceptionInfo() - job.on_failure(exc_info) + job.on_failure(exc_info, **kwargs) self.assertTrue(job.send_event.called) + return job def test_on_failure(self): self._test_on_failure(Exception('Inside unit tests')) - def test_on_failure_unicode_exception(self): + def test_on_failure__unicode_exception(self): self._test_on_failure(Exception('Бобры атакуют')) - def test_on_failure_utf8_exception(self): + def test_on_failure__utf8_exception(self): self._test_on_failure(Exception( from_utf8('Бобры атакуют'))) + + def test_on_failure__WorkerLostError(self): + exc = WorkerLostError() + job = self._test_on_failure(exc) + job.task.backend.mark_as_failure.assert_called_with( + job.id, exc, request=job, store_result=True, + ) + + def test_on_failure__return_ok(self): + self._test_on_failure(KeyError(), return_ok=True) + + def test_reject(self): + job = self.xRequest(id=uuid()) + job.on_reject = Mock(name='on_reject') + job.acknowleged = False + job.reject(requeue=True) + job.on_reject.assert_called_with( + req_logger, job.connection_errors, True, + ) + self.assertTrue(job.acknowledged) + job.on_reject.reset_mock() + job.reject(requeue=True) + self.assertFalse(job.on_reject.called) + + def test_group(self): + gid = uuid() + job = self.xRequest(id=uuid(), group=gid) + self.assertEqual(job.group, gid) + + +class test_create_request_class(RequestCase): + + def setup(self): + RequestCase.setup(self) + self.task = Mock(name='task') + self.pool = Mock(name='pool') + self.eventer = Mock(name='eventer') + + def create_request_cls(self, **kwargs): + return create_request_cls( + Request, self.task, self.pool, 'foo', self.eventer, **kwargs + ) + + def zRequest(self, Request=None, revoked_tasks=None, ref=None, **kwargs): + return self.xRequest( + Request=Request or self.create_request_cls( + ref=ref, + revoked_tasks=revoked_tasks, + ), + **kwargs) + + def test_on_success(self): + self.zRequest(id=uuid()).on_success((False, "hey", 3.1222)) + + def test_on_success__SystemExit(self, + errors=(SystemExit, KeyboardInterrupt)): + for exc in errors: + einfo = None + try: + raise exc() + except exc: + einfo = ExceptionInfo() + with self.assertRaises(exc): + self.zRequest(id=uuid()).on_success((True, einfo, 1.0)) + + def test_on_success__calls_failure(self): + job = self.zRequest(id=uuid()) + einfo = Mock(name='einfo') + job.on_failure = Mock(name='on_failure') + job.on_success((True, einfo, 1.0)) + job.on_failure.assert_called_with(einfo, return_ok=True) + + def test_on_success__acks_late_enabled(self): + self.task.acks_late = True + job = self.zRequest(id=uuid()) + job.acknowledge = Mock(name='ack') + job.on_success((False, 'foo', 1.0)) + job.acknowledge.assert_called_with() + + def test_on_success__acks_late_disabled(self): + self.task.acks_late = False + job = self.zRequest(id=uuid()) + job.acknowledge = Mock(name='ack') + job.on_success((False, 'foo', 1.0)) + self.assertFalse(job.acknowledge.called) + + def test_on_success__no_events(self): + self.eventer = None + job = self.zRequest(id=uuid()) + job.send_event = Mock(name='send_event') + job.on_success((False, 'foo', 1.0)) + self.assertFalse(job.send_event.called) + + def test_on_success__with_events(self): + job = self.zRequest(id=uuid()) + job.send_event = Mock(name='send_event') + job.on_success((False, 'foo', 1.0)) + job.send_event.assert_called_with( + 'task-succeeded', result='foo', runtime=1.0, + ) + + def test_execute_using_pool__revoked(self): + tid = uuid() + job = self.zRequest(id=tid, revoked_tasks={tid}) + job.revoked = Mock() + job.revoked.return_value = True + with self.assertRaises(TaskRevokedError): + job.execute_using_pool(self.pool) + + def test_execute_using_pool__expired(self): + tid = uuid() + job = self.zRequest(id=tid, revoked_tasks=set()) + job.expires = 1232133 + job.revoked = Mock() + job.revoked.return_value = True + with self.assertRaises(TaskRevokedError): + job.execute_using_pool(self.pool) + + def test_execute_using_pool(self): + from celery.app.trace import trace_task_ret as trace + weakref_ref = Mock(name='weakref.ref') + job = self.zRequest(id=uuid(), revoked_tasks=set(), ref=weakref_ref) + job.execute_using_pool(self.pool) + self.pool.apply_async.assert_called_with( + trace, + args=(job.type, job.id, job.request_dict, job.body, + job.content_type, job.content_encoding), + accept_callback=job.on_accepted, + timeout_callback=job.on_timeout, + callback=job.on_success, + error_callback=job.on_failure, + soft_timeout=self.task.soft_time_limit, + timeout=self.task.time_limit, + correlation_id=job.id, + ) + self.assertTrue(job._apply_result) + weakref_ref.assert_called_with(self.pool.apply_async()) + self.assertIs(job._apply_result, weakref_ref()) diff --git a/celery/tests/worker/test_strategy.py b/celery/tests/worker/test_strategy.py index 6e34f3841..143bed25c 100644 --- a/celery/tests/worker/test_strategy.py +++ b/celery/tests/worker/test_strategy.py @@ -5,13 +5,57 @@ from kombu.utils.limits import TokenBucket +from celery.exceptions import InvalidTaskError from celery.worker import state +from celery.worker.strategy import proto1_to_proto2 from celery.utils.timeutils import rate -from celery.tests.case import AppCase, Mock, patch, task_message_from_sig +from celery.tests.case import ( + AppCase, Mock, TaskMessage, TaskMessage1, patch, task_message_from_sig, +) -class test_default_strategy(AppCase): +class test_proto1_to_proto2(AppCase): + + def setup(self): + self.message = Mock(name='message') + self.body = { + 'args': (1,), + 'kwargs': {'foo': 'baz'}, + 'utc': False, + 'taskset': '123', + } + + def test_message_without_args(self): + self.body.pop('args') + with self.assertRaises(InvalidTaskError): + proto1_to_proto2(self.message, self.body) + + def test_message_without_kwargs(self): + self.body.pop('kwargs') + with self.assertRaises(InvalidTaskError): + proto1_to_proto2(self.message, self.body) + + def test_message_kwargs_not_mapping(self): + self.body['kwargs'] = (2,) + with self.assertRaises(InvalidTaskError): + proto1_to_proto2(self.message, self.body) + + def test_message_no_taskset_id(self): + self.body.pop('taskset') + self.assertTrue(proto1_to_proto2(self.message, self.body)) + + def test_message(self): + body, headers, decoded, utc = proto1_to_proto2(self.message, self.body) + self.assertTupleEqual(body, ((1,), {'foo': 'baz'}, { + 'callbacks': None, 'errbacks': None, 'chord': None, 'chain': None, + })) + self.assertDictEqual(headers, dict(self.body, group='123')) + self.assertTrue(decoded) + self.assertFalse(utc) + + +class test_default_strategy_proto2(AppCase): def setup(self): @self.app.task(shared=False) @@ -20,6 +64,12 @@ def add(x, y): self.add = add + def get_message_class(self): + return TaskMessage + + def prepare_message(self, message): + return message + class Context(object): def __init__(self, sig, s, reserved, consumer, message): @@ -29,10 +79,12 @@ def __init__(self, sig, s, reserved, consumer, message): self.consumer = consumer self.message = message - def __call__(self, **kwargs): + def __call__(self, callbacks=[], **kwargs): return self.s( - self.message, None, - self.message.ack, self.message.reject, [], **kwargs + self.message, + (self.message.payload + if not self.message.headers.get('id') else None), + self.message.ack, self.message.reject, callbacks, **kwargs ) def was_reserved(self): @@ -76,7 +128,10 @@ def _context(self, sig, s = sig.type.start_strategy(self.app, consumer, task_reserved=reserved) self.assertTrue(s) - message = task_message_from_sig(self.app, sig, utc=utc) + message = task_message_from_sig( + self.app, sig, utc=utc, TaskMessage=self.get_message_class(), + ) + message = self.prepare_message(message) yield self.Context(sig, s, reserved, consumer, message) def test_when_logging_disabled(self): @@ -94,6 +149,14 @@ def test_task_strategy(self): C.consumer.on_task_request.assert_called_with(req) self.assertTrue(C.event_sent()) + def test_callbacks(self): + with self._context(self.add.s(2, 2)) as C: + callbacks = [Mock(name='cb1'), Mock(name='cb2')] + C(callbacks=callbacks) + req = C.get_request() + for callback in callbacks: + callback.assert_called_with(req) + def test_when_events_disabled(self): with self._context(self.add.s(2, 2), events=False) as C: C() @@ -136,3 +199,19 @@ def test_when_revoked(self): C.get_request() finally: state.revoked.discard(task.id) + + +class test_default_strategy_proto1(test_default_strategy_proto2): + + def get_message_class(self): + return TaskMessage1 + + +class test_default_strategy_proto1__no_utc(test_default_strategy_proto2): + + def get_message_class(self): + return TaskMessage1 + + def prepare_message(self, message): + message.payload['utc'] = False + return message diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 1af2914e5..31ebbfed1 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -210,6 +210,10 @@ def noop(*args, **kwargs): pass +def pass1(arg, *args, **kwargs): + return arg + + def evaluate_promises(it): for value in it: if isinstance(value, promise): diff --git a/celery/worker/request.py b/celery/worker/request.py index b3cb81ad0..1c01d5a79 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -81,6 +81,7 @@ class Request(object): 'on_ack', 'body', 'hostname', 'eventer', 'connection_errors', 'task', 'eta', 'expires', 'request_dict', 'on_reject', 'utc', 'content_type', 'content_encoding', 'argsrepr', 'kwargsrepr', + '_decoded', '__weakref__', '__dict__', ) @@ -99,6 +100,7 @@ def __init__(self, message, on_ack=noop, self.message = message self.body = body self.utc = utc + self._decoded = decoded if decoded: self.content_type = self.content_encoding = None else: @@ -111,7 +113,7 @@ def __init__(self, message, on_ack=noop, self.root_id = headers.get('root_id') self.parent_id = headers.get('parent_id') if 'shadow' in headers: - self.name = headers['shadow'] + self.name = headers['shadow'] or self.name if 'timelimit' in headers: self.time_limits = headers['timelimit'] self.argsrepr = headers.get('argsrepr', '') @@ -460,7 +462,7 @@ def correlation_id(self): @cached_property def _payload(self): - return self.message.payload + return self.body if self._decoded else self.message.payload @cached_property def chord(self): diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index a753e78dc..d087743e6 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -50,7 +50,13 @@ def proto1_to_proto2(message, body): body['group'] = body['taskset'] except KeyError: pass - return (args, kwargs), body, True, body.get('utc', True) + embed = { + 'callbacks': body.get('callbacks'), + 'errbacks': body.get('errbacks'), + 'chord': body.get('chord'), + 'chain': None, + } + return (args, kwargs, embed), body, True, body.get('utc', True) def default(task, app, consumer, From f399d076a102d9022ef61cbdc5dfce9795496b1b Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Thu, 12 Nov 2015 13:29:09 -0500 Subject: [PATCH 0802/1103] Replace use of 'if [[...]]' not supported in sh The use of 'if [[...]]' breaks on Ubuntu that uses dash as its implementation of sh. Replace wildcard matching with grep. --- extra/generic-init.d/celeryd | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/extra/generic-init.d/celeryd b/extra/generic-init.d/celeryd index 873dd9f52..9dd43e9b7 100755 --- a/extra/generic-init.d/celeryd +++ b/extra/generic-init.d/celeryd @@ -37,9 +37,14 @@ if [ $(id -u) -ne 0 ]; then exit 1 fi +origin_is_runlevel_dir () { + set +e + dirname $0 | grep -q "/etc/rc.\.d" + echo $? +} # Can be a runlevel symlink (e.g. S02celeryd) -if [[ `dirname $0` == /etc/rc*.d ]]; then +if [ $(origin_is_runlevel_dir) -eq 0 ]; then SCRIPT_FILE=$(readlink "$0") else SCRIPT_FILE="$0" From 0c801b90702bb12cddd65f6483c5ba6cc844546f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 17 Nov 2015 19:30:59 -0800 Subject: [PATCH 0803/1103] 92% coverage --- .coveragerc | 15 ++- celery/app/base.py | 3 +- celery/app/defaults.py | 2 +- celery/app/task.py | 2 +- celery/app/trace.py | 4 +- celery/app/utils.py | 2 +- celery/backends/cache.py | 2 +- celery/backends/database/__init__.py | 2 +- celery/backends/database/session.py | 3 +- celery/events/dumper.py | 2 +- celery/schedules.py | 18 +-- celery/tests/app/test_app.py | 118 +++++++++++++++++ celery/tests/app/test_loaders.py | 16 +++ celery/tests/app/test_log.py | 1 + celery/tests/app/test_routes.py | 8 +- celery/tests/app/test_schedules.py | 74 ++++++++++- celery/tests/backends/test_database.py | 71 +++++++++- celery/tests/backends/test_rpc.py | 3 + celery/tests/case.py | 13 +- celery/tests/concurrency/test_concurrency.py | 47 ++++++- celery/tests/concurrency/test_eventlet.py | 111 +++++++++------- celery/tests/concurrency/test_gevent.py | 128 +++++++++---------- celery/tests/concurrency/test_prefork.py | 114 ++++++++++++++--- celery/tests/fixups/test_django.py | 21 +-- celery/tests/security/test_certificate.py | 5 + celery/tests/security/test_security.py | 10 ++ celery/tests/tasks/test_tasks.py | 63 ++++++++- celery/tests/tasks/test_trace.py | 115 ++++++++++++++++- celery/tests/utils/test_debug.py | 98 ++++++++++++++ celery/tests/utils/test_mail.py | 32 ++++- celery/tests/utils/test_text.py | 5 + celery/tests/utils/test_utils.py | 76 ++++++++++- celery/tests/worker/test_autoscale.py | 2 +- celery/tests/worker/test_consumer.py | 49 +++++++ celery/tests/worker/test_control.py | 81 ++++++++++-- celery/tests/worker/test_loops.py | 35 ++++- celery/tests/worker/test_worker.py | 47 +------ celery/utils/abstract.py | 4 +- celery/utils/debug.py | 8 +- celery/worker/control.py | 12 +- 40 files changed, 1177 insertions(+), 245 deletions(-) create mode 100644 celery/tests/utils/test_debug.py diff --git a/.coveragerc b/.coveragerc index c26f8646e..39ff403db 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,6 +2,17 @@ branch = 1 cover_pylib = 0 include=*celery/* -omit = celery.utils.debug,celery.tests.*,celery.bin.graph; +omit = celery.tests.* [report] -omit = */python?.?/*,*/site-packages/*,*/pypy/* +omit = + */python?.?/* + */site-packages/* + */pypy/* + */celery/bin/graph.py + *celery/bin/logtool.py + *celery/task/base.py + *celery/five.py + *celery/contrib/sphinx.py + *celery/backends/couchdb.py + *celery/backends/couchbase.py + *celery/backends/cassandra.py diff --git a/celery/app/base.py b/celery/app/base.py index 1d34f08ea..3774b9cce 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -19,7 +19,7 @@ from amqp import starpromise try: from billiard.util import register_after_fork -except ImportError: +except ImportError: # pragma: no cover register_after_fork = None from kombu.clocks import LamportClock from kombu.common import oid_from @@ -771,7 +771,6 @@ def mail_admins(self, subject, body, fail_silently=False): def select_queues(self, queues=None): """Select a subset of queues, where queues must be a list of queue names to keep.""" - return self.amqp.queues.select(queues) def either(self, default_key, *values): diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 9f44884e6..a4d158d20 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -335,7 +335,7 @@ def flatten(d, root='', keyfilter=_flatten_keys): _OLD_SETTING_KEYS = set(keys(_TO_NEW_KEY)) -def find_deprecated_settings(source): +def find_deprecated_settings(source): # pragma: no cover from celery.utils import warn_deprecated for name, opt in flatten(NAMESPACES): if (opt.deprecate_by or opt.remove_by) and getattr(source, name, None): diff --git a/celery/app/task.py b/celery/app/task.py index bf2bd449f..bbd1d85e6 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -477,7 +477,7 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, """ try: check_arguments = self.__header__ - except AttributeError: + except AttributeError: # pragma: no cover pass else: check_arguments(*(args or ()), **(kwargs or {})) diff --git a/celery/app/trace.py b/celery/app/trace.py index d337373a9..5634a867f 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -390,12 +390,12 @@ def trace_task(uuid, args, kwargs, request=None): else: sigs.append(sig) for group_ in groups: - group.apply_async( + group_.apply_async( (retval,), parent_id=uuid, root_id=root_id, ) if sigs: - group(sigs).apply_async( + group(sigs, app=app).apply_async( (retval,), parent_id=uuid, root_id=root_id, ) diff --git a/celery/app/utils.py b/celery/app/utils.py index 1775e94a5..9078294a8 100644 --- a/celery/app/utils.py +++ b/celery/app/utils.py @@ -141,7 +141,7 @@ def table(self, with_defaults=False, censored=True): return filt({ k: v for k, v in items( self if with_defaults else self.without_defaults()) - if k.isupper() and not k.startswith('_') + if not k.startswith('_') }) def humanize(self, with_defaults=False, censored=True): diff --git a/celery/backends/cache.py b/celery/backends/cache.py index 8736d6765..9d8f7c97e 100644 --- a/celery/backends/cache.py +++ b/celery/backends/cache.py @@ -45,7 +45,7 @@ def import_best_memcache(): import memcache # noqa except ImportError: raise ImproperlyConfigured(REQUIRES_BACKEND) - if PY3: + if PY3: # pragma: no cover memcache_key_t = bytes_to_str _imp[0] = (is_pylibmc, memcache, memcache_key_t) return _imp[0] diff --git a/celery/backends/database/__init__.py b/celery/backends/database/__init__.py index 508f3413f..bbd570a71 100644 --- a/celery/backends/database/__init__.py +++ b/celery/backends/database/__init__.py @@ -25,7 +25,7 @@ try: from sqlalchemy.exc import DatabaseError, InvalidRequestError from sqlalchemy.orm.exc import StaleDataError -except ImportError: +except ImportError: # pragma: no cover raise ImproperlyConfigured( 'The database result backend requires SQLAlchemy to be installed.' 'See http://pypi.python.org/pypi/SQLAlchemy') diff --git a/celery/backends/database/session.py b/celery/backends/database/session.py index 036b84300..17cdc8982 100644 --- a/celery/backends/database/session.py +++ b/celery/backends/database/session.py @@ -10,7 +10,7 @@ try: from billiard.util import register_after_fork -except ImportError: +except ImportError: # pragma: no cover register_after_fork = None from sqlalchemy import create_engine @@ -24,6 +24,7 @@ class SessionManager(object): + def __init__(self): self._engines = {} self._sessions = {} diff --git a/celery/events/dumper.py b/celery/events/dumper.py index 3c20186e6..672670b97 100644 --- a/celery/events/dumper.py +++ b/celery/events/dumper.py @@ -48,7 +48,7 @@ def say(self, msg): # need to flush so that output can be piped. try: self.out.flush() - except AttributeError: + except AttributeError: # pragma: no cover pass def on_event(self, ev): diff --git a/celery/schedules.py b/celery/schedules.py index 6b03e59d1..52c366128 100644 --- a/celery/schedules.py +++ b/celery/schedules.py @@ -589,7 +589,10 @@ def __eq__(self, other): return NotImplemented def __ne__(self, other): - return not self.__eq__(other) + res = self.__eq__(other) + if res is NotImplemented: + return True + return not res def maybe_schedule(s, relative=False, app=None): @@ -691,12 +694,8 @@ def __init__(self, event, lat, lon, nowfun=None, app=None): self.method = self._methods[event] self.use_center = self._use_center_l[event] - def now(self): - return (self.nowfun or self.app.now)() - def __reduce__(self): - return (self.__class__, ( - self.event, self.lat, self.lon), None) + return self.__class__, (self.event, self.lat, self.lon) def __repr__(self): return ''.format( @@ -715,7 +714,7 @@ def remaining_estimate(self, last_run_at): self.ephem.Sun(), start=last_run_at_utc, use_center=self.use_center, ) - except self.ephem.CircumpolarError: + except self.ephem.CircumpolarError: # pragma: no cover """Sun will not rise/set today. Check again tomorrow (specifically, after the next anti-transit).""" next_utc = ( @@ -750,4 +749,7 @@ def __eq__(self, other): return NotImplemented def __ne__(self, other): - return not self.__eq__(other) + res = self.__eq__(other) + if res is NotImplemented: + return True + return not res diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index 8d350d880..ad5c5fbcd 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -9,6 +9,7 @@ from amqp import promise +from celery import Celery from celery import shared_task, current_app from celery import app as _app from celery import _state @@ -19,12 +20,14 @@ from celery.loaders.base import BaseLoader, unconfigured from celery.platforms import pyimplementation from celery.utils.serialization import pickle +from celery.utils.timeutils import timezone from celery.tests.case import ( CELERY_TEST_CONFIG, AppCase, Mock, Case, + ContextMock, depends_on_current_app, mask_modules, patch, @@ -128,6 +131,12 @@ def fun(): task = app.task(fun) self.assertEqual(task.name, app.main + '.fun') + def test_task_too_many_args(self): + with self.assertRaises(TypeError): + self.app.task(Mock(name='fun'), True) + with self.assertRaises(TypeError): + self.app.task(Mock(name='fun'), True, 1, 2) + def test_with_config_source(self): with self.Celery(config_source=ObjectConfig) as app: self.assertEqual(app.conf.FOO, 1) @@ -235,6 +244,18 @@ def lazy_list(): self.assertEqual(prom.fun, self.app._autodiscover_tasks) self.assertEqual(prom.args[0](), [1, 2, 3]) + def test_autodiscover_tasks__no_packages(self): + fixup1 = Mock(name='fixup') + fixup2 = Mock(name='fixup') + self.app._autodiscover_tasks_from_names = Mock(name='auto') + self.app._fixups = [fixup1, fixup2] + fixup1.autodiscover_tasks.return_value = ['A', 'B', 'C'] + fixup2.autodiscover_tasks.return_value = ['D', 'E', 'F'] + self.app.autodiscover_tasks(force=True) + self.app._autodiscover_tasks_from_names.assert_called_with( + ['A', 'B', 'C', 'D', 'E', 'F'], related_name='tasks', + ) + @with_environ('CELERY_BROKER_URL', '') def test_with_broker(self): with self.Celery(broker='foo://baribaz') as app: @@ -739,6 +760,86 @@ def test_after_fork(self): self.assertIsNone(self.app._pool) self.app._after_fork(self.app) + def test_global_after_fork(self): + app = Mock(name='app') + prev, _state._apps = _state._apps, [app] + try: + obj = Mock(name='obj') + _appbase._global_after_fork(obj) + app._after_fork.assert_called_with(obj) + finally: + _state._apps = prev + + @patch('multiprocessing.util', create=True) + def test_global_after_fork__raises(self, util): + app = Mock(name='app') + prev, _state._apps = _state._apps, [app] + try: + obj = Mock(name='obj') + exc = app._after_fork.side_effect = KeyError() + _appbase._global_after_fork(obj) + util._logger.info.assert_called_with( + 'after forker raised exception: %r', exc, exc_info=1) + util._logger = None + _appbase._global_after_fork(obj) + finally: + _state._apps = prev + + def test_ensure_after_fork__no_multiprocessing(self): + prev, _appbase.register_after_fork = ( + _appbase.register_after_fork, None) + try: + _appbase._after_fork_registered = False + _appbase._ensure_after_fork() + self.assertTrue(_appbase._after_fork_registered) + finally: + _appbase.register_after_fork = prev + + def test_canvas(self): + self.assertTrue(self.app.canvas.Signature) + + def test_signature(self): + sig = self.app.signature('foo', (1, 2)) + self.assertIs(sig.app, self.app) + + def test_timezone__none_set(self): + self.app.conf.timezone = None + tz = self.app.timezone + self.assertEqual(tz, timezone.get_timezone('UTC')) + + def test_compat_on_configure(self): + on_configure = Mock(name='on_configure') + + class CompatApp(Celery): + + def on_configure(self, *args, **kwargs): + on_configure(*args, **kwargs) + + with CompatApp(set_as_current=False) as app: + app.loader = Mock() + app.loader.conf = {} + app._load_config() + on_configure.assert_called_with() + + def test_add_periodic_task(self): + + @self.app.task + def add(x, y): + pass + assert not self.app.configured + self.app.add_periodic_task( + 10, self.app.signature('add', (2, 2)), + name='add1', expires=3, + ) + self.assertTrue(self.app._pending_periodic_tasks) + assert not self.app.configured + + sig2 = add.s(4, 4) + self.assertTrue(self.app.configured) + self.app.add_periodic_task(20, sig2, name='add2', expires=4) + self.assertIn('add1', self.app.conf.beat_schedule) + self.assertIn('add2', self.app.conf.beat_schedule) + def test_pool_no_multiprocessing(self): with mask_modules('multiprocessing.util'): pool = self.app.pool @@ -747,6 +848,18 @@ def test_pool_no_multiprocessing(self): def test_bugreport(self): self.assertTrue(self.app.bugreport()) + def test_send_task__connection_provided(self): + connection = Mock(name='connection') + router = Mock(name='router') + router.route.return_value = {} + self.app.amqp = Mock(name='amqp') + self.app.amqp.Producer.attach_mock(ContextMock(), 'return_value') + self.app.send_task('foo', (1, 2), connection=connection, router=router) + self.app.amqp.Producer.assert_called_with(connection) + self.app.amqp.send_task_message.assert_called_with( + self.app.amqp.Producer(), 'foo', + self.app.amqp.create_task_message()) + def test_send_task_sent_event(self): class Dispatcher(object): @@ -799,6 +912,11 @@ def test_error_mail_disabled(self): x.send(Mock(), Mock()) self.assertFalse(task.app.mail_admins.called) + def test_select_queues(self): + self.app.amqp = Mock(name='amqp') + self.app.select_queues({'foo', 'bar'}) + self.app.amqp.queues.select.assert_called_with({'foo', 'bar'}) + class test_defaults(AppCase): diff --git a/celery/tests/app/test_loaders.py b/celery/tests/app/test_loaders.py index 9d80e08f8..6c27c8785 100644 --- a/celery/tests/app/test_loaders.py +++ b/celery/tests/app/test_loaders.py @@ -184,6 +184,22 @@ class ConfigModule(ModuleType): if prevconfig: sys.modules[configname] = prevconfig + def test_read_configuration_ImportError(self): + sentinel = object() + prev, os.environ['CELERY_CONFIG_MODULE'] = ( + os.environ.get('CELERY_CONFIG_MODULE', sentinel), 'daweqew.dweqw', + ) + try: + l = default.Loader(app=self.app) + with self.assertRaises(ImportError): + l.read_configuration(fail_silently=False) + l.read_configuration(fail_silently=True) + finally: + if prev is not sentinel: + os.environ['CELERY_CONFIG_MODULE'] = prev + else: + os.environ.pop('CELERY_CONFIG_MODULE', None) + def test_import_from_cwd(self): l = default.Loader(app=self.app) old_path = list(sys.path) diff --git a/celery/tests/app/test_log.py b/celery/tests/app/test_log.py index 2920d97a2..944c27252 100644 --- a/celery/tests/app/test_log.py +++ b/celery/tests/app/test_log.py @@ -199,6 +199,7 @@ def test_get_default_logger(self): def test_configure_logger(self): logger = self.app.log.get_default_logger() self.app.log._configure_logger(logger, sys.stderr, None, '', False) + self.app.log._configure_logger(None, sys.stderr, None, '', False) logger.handlers[:] = [] def test_setup_logging_subsystem_colorize(self): diff --git a/celery/tests/app/test_routes.py b/celery/tests/app/test_routes.py index 7eed424f2..9730aab05 100644 --- a/celery/tests/app/test_routes.py +++ b/celery/tests/app/test_routes.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -from kombu import Exchange +from kombu import Exchange, Queue from kombu.utils.functional import maybe_evaluate from celery.app import routes @@ -121,6 +121,12 @@ def test_expand_destination_string(self): dest = x.expand_destination('foo') self.assertEqual(dest['queue'].name, 'foo') + def test_expand_destination__Queue(self): + queue = Queue('foo') + x = Router(self.app, {}, self.app.amqp.queues) + dest = x.expand_destination({'queue': queue}) + self.assertIs(dest['queue'], queue) + def test_lookup_paths_traversed(self): set_queues( self.app, foo=self.a_queue, bar=self.b_queue, diff --git a/celery/tests/app/test_schedules.py b/celery/tests/app/test_schedules.py index 90f49125b..576c0e162 100644 --- a/celery/tests/app/test_schedules.py +++ b/celery/tests/app/test_schedules.py @@ -7,8 +7,10 @@ from pickle import dumps, loads from celery.five import items -from celery.schedules import ParseException, crontab, crontab_parser -from celery.tests.case import AppCase, SkipTest +from celery.schedules import ( + ParseException, crontab, crontab_parser, schedule, solar, +) +from celery.tests.case import AppCase, Mock, SkipTest @contextmanager @@ -21,6 +23,73 @@ def patch_crontab_nowfun(cls, retval): cls.nowfun = prev_nowfun +class test_solar(AppCase): + + def setup(self): + try: + import ephem # noqa + except ImportError: + raise SkipTest('ephem module not installed') + self.s = solar('sunrise', 60, 30, app=self.app) + + def test_reduce(self): + fun, args = self.s.__reduce__() + self.assertEqual(fun(*args), self.s) + + def test_eq(self): + self.assertEqual(self.s, solar('sunrise', 60, 30, app=self.app)) + self.assertNotEqual(self.s, solar('sunset', 60, 30, app=self.app)) + self.assertNotEqual(self.s, schedule(10)) + + def test_repr(self): + self.assertTrue(repr(self.s)) + + def test_is_due(self): + self.s.remaining_estimate = Mock(name='rem') + self.s.remaining_estimate.return_value = timedelta(seconds=0) + self.assertTrue(self.s.is_due(datetime.utcnow()).is_due) + + def test_is_due__not_due(self): + self.s.remaining_estimate = Mock(name='rem') + self.s.remaining_estimate.return_value = timedelta(hours=10) + self.assertFalse(self.s.is_due(datetime.utcnow()).is_due) + + def test_remaining_estimate(self): + self.s.cal = Mock(name='cal') + self.s.cal.next_rising().datetime.return_value = datetime.utcnow() + self.s.remaining_estimate(datetime.utcnow()) + + def test_coordinates(self): + with self.assertRaises(ValueError): + solar('sunrise', -120, 60) + with self.assertRaises(ValueError): + solar('sunrise', 120, 60) + with self.assertRaises(ValueError): + solar('sunrise', 60, -200) + with self.assertRaises(ValueError): + solar('sunrise', 60, 200) + + def test_invalid_event(self): + with self.assertRaises(ValueError): + solar('asdqwewqew', 60, 60) + + +class test_schedule(AppCase): + + def test_ne(self): + s1 = schedule(10, app=self.app) + s2 = schedule(12, app=self.app) + s3 = schedule(10, app=self.app) + self.assertEqual(s1, s3) + self.assertNotEqual(s1, s2) + + def test_pickle(self): + s1 = schedule(10, app=self.app) + fun, args = s1.__reduce__() + s2 = fun(*args) + self.assertEqual(s1, s2) + + class test_crontab_parser(AppCase): def crontab(self, *args, **kwargs): @@ -182,6 +251,7 @@ def test_eq(self): ) self.assertFalse(object() == self.crontab(minute='1')) self.assertFalse(self.crontab(minute='1') == object()) + self.assertNotEqual(crontab(month_of_year='1'), schedule(10)) class test_crontab_remaining_estimate(AppCase): diff --git a/celery/tests/backends/test_database.py b/celery/tests/backends/test_database.py index c7d5f8fbe..5c2fcba6e 100644 --- a/celery/tests/backends/test_database.py +++ b/celery/tests/backends/test_database.py @@ -10,8 +10,10 @@ from celery.tests.case import ( AppCase, + Mock, SkipTest, depends_on_current_app, + patch, skip_if_pypy, skip_if_jython, ) @@ -21,7 +23,11 @@ except ImportError: DatabaseBackend = Task = TaskSet = retry = None # noqa else: - from celery.backends.database import DatabaseBackend, retry + from celery.backends.database import ( + DatabaseBackend, retry, session_cleanup, + ) + from celery.backends.database import session + from celery.backends.database.session import SessionManager from celery.backends.database.models import Task, TaskSet @@ -31,6 +37,23 @@ def __init__(self, data): self.data = data +class test_session_cleanup(AppCase): + + def test_context(self): + session = Mock(name='session') + with session_cleanup(session): + pass + session.close.assert_called_with() + + def test_context_raises(self): + session = Mock(name='session') + with self.assertRaises(KeyError): + with session_cleanup(session): + raise KeyError() + session.rollback.assert_called_with() + session.close.assert_called_with() + + class test_DatabaseBackend(AppCase): @skip_if_pypy @@ -188,3 +211,49 @@ def test_Task__repr__(self): def test_TaskSet__repr__(self): self.assertIn('foo', repr(TaskSet('foo', None))) + + +class test_SessionManager(AppCase): + + def test_after_fork(self): + s = SessionManager() + self.assertFalse(s.forked) + s._after_fork() + self.assertTrue(s.forked) + + @patch('celery.backends.database.session.create_engine') + def test_get_engine_forked(self, create_engine): + s = SessionManager() + s._after_fork() + engine = s.get_engine('dburi', foo=1) + create_engine.assert_called_with('dburi', foo=1) + self.assertIs(engine, create_engine()) + engine2 = s.get_engine('dburi', foo=1) + self.assertIs(engine2, engine) + + @patch('celery.backends.database.session.sessionmaker') + def test_create_session_forked(self, sessionmaker): + s = SessionManager() + s.get_engine = Mock(name='get_engine') + s._after_fork() + engine, session = s.create_session('dburi', short_lived_sessions=True) + sessionmaker.assert_called_with(bind=s.get_engine()) + self.assertIs(session, sessionmaker()) + sessionmaker.return_value = Mock(name='new') + engine, session2 = s.create_session('dburi', short_lived_sessions=True) + sessionmaker.assert_called_with(bind=s.get_engine()) + self.assertIsNot(session2, session) + sessionmaker.return_value = Mock(name='new2') + engine, session3 = s.create_session( + 'dburi', short_lived_sessions=False) + sessionmaker.assert_called_with(bind=s.get_engine()) + self.assertIs(session3, session2) + + def test_coverage_madness(self): + prev, session.register_after_fork = ( + session.register_after_fork, None, + ) + try: + SessionManager() + finally: + session.register_after_fork = prev diff --git a/celery/tests/backends/test_rpc.py b/celery/tests/backends/test_rpc.py index 60c3aaa5c..2b0ccb86b 100644 --- a/celery/tests/backends/test_rpc.py +++ b/celery/tests/backends/test_rpc.py @@ -43,6 +43,9 @@ def test_destination_for(self): with self.assertRaises(RuntimeError): self.b.destination_for('task_id', None) + def test_rkey(self): + self.assertEqual(self.b.rkey('id1'), 'id1') + def test_binding(self): queue = self.b.binding self.assertEqual(queue.name, self.b.oid) diff --git a/celery/tests/case.py b/celery/tests/case.py index 731159836..f40c92939 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -34,7 +34,7 @@ from nose import SkipTest from kombu import Queue from kombu.log import NullHandler -from kombu.utils import nested, symbol_by_name +from kombu.utils import symbol_by_name from celery import Celery from celery.app import current_app @@ -54,7 +54,7 @@ 'skip_if_environ', 'todo', 'skip', 'skip_if', 'skip_unless', 'mask_modules', 'override_stdouts', 'mock_module', 'replace_module_value', 'sys_platform', 'reset_modules', - 'patch_modules', 'mock_context', 'mock_open', 'patch_many', + 'patch_modules', 'mock_context', 'mock_open', 'assert_signal_called', 'skip_if_pypy', 'skip_if_jython', 'task_message_from_sig', 'restore_logging', ] @@ -315,6 +315,11 @@ def patch(self, *path, **options): self.addCleanup(manager.stop) return patched + def mock_modules(self, *modules): + manager = mock_module(*modules) + manager.__enter__() + self.addCleanup(partial(manager.__exit__, None, None, None)) + def assertWarns(self, expected_warning): return _AssertWarnsContext(expected_warning, self, None) @@ -815,10 +820,6 @@ def mock_open(typ=WhateverIO, side_effect=None): yield val -def patch_many(*targets): - return nested(*[patch(target) for target in targets]) - - @contextmanager def assert_signal_called(signal, **expected): handler = Mock() diff --git a/celery/tests/concurrency/test_concurrency.py b/celery/tests/concurrency/test_concurrency.py index dd845de1f..0ea7d6567 100644 --- a/celery/tests/concurrency/test_concurrency.py +++ b/celery/tests/concurrency/test_concurrency.py @@ -5,7 +5,8 @@ from itertools import count from celery.concurrency.base import apply_target, BasePool -from celery.tests.case import AppCase, Mock +from celery.exceptions import WorkerShutdown, WorkerTerminate +from celery.tests.case import AppCase, Mock, patch class test_BasePool(AppCase): @@ -47,6 +48,47 @@ def callback(*args): {'target': (3, (8, 16)), 'callback': (4, (42,))}) + def test_apply_target__propagate(self): + target = Mock(name='target') + target.side_effect = KeyError() + with self.assertRaises(KeyError): + apply_target(target, propagate=(KeyError,)) + + def test_apply_target__raises(self): + target = Mock(name='target') + target.side_effect = KeyError() + with self.assertRaises(KeyError): + apply_target(target) + + def test_apply_target__raises_WorkerShutdown(self): + target = Mock(name='target') + target.side_effect = WorkerShutdown() + with self.assertRaises(WorkerShutdown): + apply_target(target) + + def test_apply_target__raises_WorkerTerminate(self): + target = Mock(name='target') + target.side_effect = WorkerTerminate() + with self.assertRaises(WorkerTerminate): + apply_target(target) + + def test_apply_target__raises_BaseException(self): + target = Mock(name='target') + callback = Mock(name='callback') + target.side_effect = BaseException() + apply_target(target, callback=callback) + self.assertTrue(callback.called) + + @patch('celery.concurrency.base.reraise') + def test_apply_target__raises_BaseException_raises_else(self, reraise): + target = Mock(name='target') + callback = Mock(name='callback') + reraise.side_effect = KeyError() + target.side_effect = BaseException() + with self.assertRaises(KeyError): + apply_target(target, callback=callback) + self.assertFalse(callback.called) + def test_does_not_debug(self): x = BasePool(10) x._does_debug = False @@ -67,6 +109,9 @@ def test_interface_on_apply(self): def test_interface_info(self): self.assertDictEqual(BasePool(10).info, {}) + def test_interface_flush(self): + self.assertIsNone(BasePool(10).flush()) + def test_active(self): p = BasePool(10) self.assertFalse(p.active) diff --git a/celery/tests/concurrency/test_eventlet.py b/celery/tests/concurrency/test_eventlet.py index 9761a84db..46828f0b9 100644 --- a/celery/tests/concurrency/test_eventlet.py +++ b/celery/tests/concurrency/test_eventlet.py @@ -3,29 +3,20 @@ import os import sys -from celery.app.defaults import is_pypy from celery.concurrency.eventlet import ( apply_target, Timer, TaskPool, ) -from celery.tests.case import ( - AppCase, Mock, SkipTest, mock_module, patch, patch_many, skip_if_pypy, -) +from celery.tests.case import AppCase, Mock, patch, skip_if_pypy class EventletCase(AppCase): @skip_if_pypy def setup(self): - if is_pypy: - raise SkipTest('mock_modules not working on PyPy1.9') - try: - self.eventlet = __import__('eventlet') - except ImportError: - raise SkipTest( - 'eventlet not installed, skipping related tests.') + self.mock_modules(*eventlet_modules) @skip_if_pypy def teardown(self): @@ -68,46 +59,80 @@ def test_aaa_blockdetecet(self, monkey_patch, hub_blocking_detection): class test_Timer(EventletCase): + def setup(self): + EventletCase.setup(self) + self.spawn_after = self.patch('eventlet.greenthread.spawn_after') + self.GreenletExit = self.patch('greenlet.GreenletExit') + def test_sched(self): - with mock_module(*eventlet_modules): - with patch_many('eventlet.greenthread.spawn_after', - 'greenlet.GreenletExit') as (spawn_after, - GreenletExit): - x = Timer() - x.GreenletExit = KeyError - entry = Mock() - g = x._enter(1, 0, entry) - self.assertTrue(x.queue) - - x._entry_exit(g, entry) - g.wait.side_effect = KeyError() - x._entry_exit(g, entry) - entry.cancel.assert_called_with() - self.assertFalse(x._queue) - - x._queue.add(g) - x.clear() - x._queue.add(g) - g.cancel.side_effect = KeyError() - x.clear() + x = Timer() + x.GreenletExit = KeyError + entry = Mock() + g = x._enter(1, 0, entry) + self.assertTrue(x.queue) + + x._entry_exit(g, entry) + g.wait.side_effect = KeyError() + x._entry_exit(g, entry) + entry.cancel.assert_called_with() + self.assertFalse(x._queue) + + x._queue.add(g) + x.clear() + x._queue.add(g) + g.cancel.side_effect = KeyError() + x.clear() + + def test_cancel(self): + x = Timer() + tref = Mock(name='tref') + x.cancel(tref) + tref.cancel.assert_called_with() + x.GreenletExit = KeyError + tref.cancel.side_effect = KeyError() + x.cancel(tref) class test_TaskPool(EventletCase): + def setup(self): + EventletCase.setup(self) + self.GreenPool = self.patch('eventlet.greenpool.GreenPool') + self.greenthread = self.patch('eventlet.greenthread') + def test_pool(self): - with mock_module(*eventlet_modules): - with patch_many('eventlet.greenpool.GreenPool', - 'eventlet.greenthread') as (GreenPool, - greenthread): - x = TaskPool() - x.on_start() - x.on_stop() - x.on_apply(Mock()) - x._pool = None - x.on_stop() - self.assertTrue(x.getpid()) + x = TaskPool() + x.on_start() + x.on_stop() + x.on_apply(Mock()) + x._pool = None + x.on_stop() + self.assertTrue(x.getpid()) @patch('celery.concurrency.eventlet.base') def test_apply_target(self, base): apply_target(Mock(), getpid=Mock()) self.assertTrue(base.apply_target.called) + + def test_grow(self): + x = TaskPool(10) + x._pool = Mock(name='_pool') + x.grow(2) + self.assertEqual(x.limit, 12) + x._pool.resize.assert_called_with(12) + + def test_shrink(self): + x = TaskPool(10) + x._pool = Mock(name='_pool') + x.shrink(2) + self.assertEqual(x.limit, 8) + x._pool.resize.assert_called_with(8) + + def test_get_info(self): + x = TaskPool(10) + x._pool = Mock(name='_pool') + self.assertDictEqual(x._get_info(), { + 'max-concurrency': 10, + 'free-threads': x._pool.free(), + 'running-threads': x._pool.running(), + }) diff --git a/celery/tests/concurrency/test_gevent.py b/celery/tests/concurrency/test_gevent.py index c4a61db6d..d99bffca4 100644 --- a/celery/tests/concurrency/test_gevent.py +++ b/celery/tests/concurrency/test_gevent.py @@ -6,9 +6,7 @@ apply_timeout, ) -from celery.tests.case import ( - AppCase, Mock, SkipTest, mock_module, patch, patch_many, skip_if_pypy, -) +from celery.tests.case import AppCase, Mock, patch, skip_if_pypy gevent_modules = ( 'gevent', @@ -23,80 +21,78 @@ class GeventCase(AppCase): @skip_if_pypy def setup(self): - try: - self.gevent = __import__('gevent') - except ImportError: - raise SkipTest( - 'gevent not installed, skipping related tests.') + self.mock_modules(*gevent_modules) class test_gevent_patch(GeventCase): def test_is_patched(self): - with mock_module(*gevent_modules): - with patch('gevent.monkey.patch_all', create=True) as patch_all: - import gevent - gevent.version_info = (1, 0, 0) - from celery import maybe_patch_concurrency - maybe_patch_concurrency(['x', '-P', 'gevent']) - self.assertTrue(patch_all.called) + with patch('gevent.monkey.patch_all', create=True) as patch_all: + import gevent + gevent.version_info = (1, 0, 0) + from celery import maybe_patch_concurrency + maybe_patch_concurrency(['x', '-P', 'gevent']) + self.assertTrue(patch_all.called) + +class test_Timer(GeventCase): -class test_Timer(AppCase): + def setup(self): + GeventCase.setup(self) + self.greenlet = self.patch('gevent.greenlet') + self.GreenletExit = self.patch('gevent.greenlet.GreenletExit') def test_sched(self): - with mock_module(*gevent_modules): - with patch_many('gevent.greenlet', - 'gevent.greenlet.GreenletExit') as (greenlet, - GreenletExit): - greenlet.Greenlet = object - x = Timer() - greenlet.Greenlet = Mock() - x._Greenlet.spawn_later = Mock() - x._GreenletExit = KeyError - entry = Mock() - g = x._enter(1, 0, entry) - self.assertTrue(x.queue) - - x._entry_exit(g) - g.kill.assert_called_with() - self.assertFalse(x._queue) - - x._queue.add(g) - x.clear() - x._queue.add(g) - g.kill.side_effect = KeyError() - x.clear() - - g = x._Greenlet() - g.cancel() - - -class test_TaskPool(AppCase): + self.greenlet.Greenlet = object + x = Timer() + self.greenlet.Greenlet = Mock() + x._Greenlet.spawn_later = Mock() + x._GreenletExit = KeyError + entry = Mock() + g = x._enter(1, 0, entry) + self.assertTrue(x.queue) + + x._entry_exit(g) + g.kill.assert_called_with() + self.assertFalse(x._queue) + + x._queue.add(g) + x.clear() + x._queue.add(g) + g.kill.side_effect = KeyError() + x.clear() + + g = x._Greenlet() + g.cancel() + + +class test_TaskPool(GeventCase): + + def setup(self): + GeventCase.setup(self) + self.spawn_raw = self.patch('gevent.spawn_raw') + self.Pool = self.patch('gevent.pool.Pool') def test_pool(self): - with mock_module(*gevent_modules): - with patch_many('gevent.spawn_raw', 'gevent.pool.Pool') as ( - spawn_raw, Pool): - x = TaskPool() - x.on_start() - x.on_stop() - x.on_apply(Mock()) - x._pool = None - x.on_stop() - - x._pool = Mock() - x._pool._semaphore.counter = 1 - x._pool.size = 1 - x.grow() - self.assertEqual(x._pool.size, 2) - self.assertEqual(x._pool._semaphore.counter, 2) - x.shrink() - self.assertEqual(x._pool.size, 1) - self.assertEqual(x._pool._semaphore.counter, 1) - - x._pool = [4, 5, 6] - self.assertEqual(x.num_processes, 3) + x = TaskPool() + x.on_start() + x.on_stop() + x.on_apply(Mock()) + x._pool = None + x.on_stop() + + x._pool = Mock() + x._pool._semaphore.counter = 1 + x._pool.size = 1 + x.grow() + self.assertEqual(x._pool.size, 2) + self.assertEqual(x._pool._semaphore.counter, 2) + x.shrink() + self.assertEqual(x._pool.size, 1) + self.assertEqual(x._pool._semaphore.counter, 1) + + x._pool = [4, 5, 6] + self.assertEqual(x.num_processes, 3) class test_apply_timeout(AppCase): diff --git a/celery/tests/concurrency/test_prefork.py b/celery/tests/concurrency/test_prefork.py index b48629c9d..bd405eb03 100644 --- a/celery/tests/concurrency/test_prefork.py +++ b/celery/tests/concurrency/test_prefork.py @@ -1,14 +1,16 @@ from __future__ import absolute_import import errno +import os import socket -import time from itertools import cycle +from celery.app.defaults import DEFAULTS +from celery.datastructures import AttributeDict from celery.five import items, range from celery.utils.functional import noop -from celery.tests.case import AppCase, Mock, SkipTest, patch +from celery.tests.case import AppCase, Mock, SkipTest, patch, restore_logging try: from celery.concurrency import prefork as mp from celery.concurrency import asynpool @@ -54,6 +56,67 @@ def get(self): return self.value +class test_process_initializer(AppCase): + + @patch('celery.platforms.signals') + @patch('celery.platforms.set_mp_process_title') + def test_process_initializer(self, set_mp_process_title, _signals): + with restore_logging(): + from celery import signals + from celery._state import _tls + from celery.concurrency.prefork import ( + process_initializer, WORKER_SIGRESET, WORKER_SIGIGNORE, + ) + + def on_worker_process_init(**kwargs): + on_worker_process_init.called = True + on_worker_process_init.called = False + signals.worker_process_init.connect(on_worker_process_init) + + def Loader(*args, **kwargs): + loader = Mock(*args, **kwargs) + loader.conf = {} + loader.override_backends = {} + return loader + + with self.Celery(loader=Loader) as app: + app.conf = AttributeDict(DEFAULTS) + process_initializer(app, 'awesome.worker.com') + _signals.ignore.assert_any_call(*WORKER_SIGIGNORE) + _signals.reset.assert_any_call(*WORKER_SIGRESET) + self.assertTrue(app.loader.init_worker.call_count) + self.assertTrue(on_worker_process_init.called) + self.assertIs(_tls.current_app, app) + set_mp_process_title.assert_called_with( + 'celeryd', hostname='awesome.worker.com', + ) + + with patch('celery.app.trace.setup_worker_optimizations') as S: + os.environ['FORKED_BY_MULTIPROCESSING'] = "1" + try: + process_initializer(app, 'luke.worker.com') + S.assert_called_with(app, 'luke.worker.com') + finally: + os.environ.pop('FORKED_BY_MULTIPROCESSING', None) + + os.environ['CELERY_LOG_FILE'] = 'worker%I.log' + app.log.setup = Mock(name='log_setup') + try: + process_initializer(app, 'luke.worker.com') + finally: + os.environ.pop('CELERY_LOG_FILE', None) + + +class test_process_destructor(AppCase): + + @patch('celery.concurrency.prefork.signals') + def test_process_destructor(self, signals): + mp.process_destructor(13, -3) + signals.worker_process_shutdown.send.assert_called_with( + sender=None, pid=13, exitcode=-3, + ) + + class MockPool(object): started = False closed = False @@ -284,6 +347,39 @@ def test_start(self): pool.terminate() self.assertTrue(_pool.terminated) + def test_restart(self): + pool = TaskPool(10) + pool._pool = Mock(name='pool') + pool.restart() + pool._pool.restart.assert_called_with() + pool._pool.apply_async.assert_called_with(mp.noop) + + def test_did_start_ok(self): + pool = TaskPool(10) + pool._pool = Mock(name='pool') + self.assertIs(pool.did_start_ok(), pool._pool.did_start_ok()) + + def test_register_with_event_loop(self): + pool = TaskPool(10) + pool._pool = Mock(name='pool') + loop = Mock(name='loop') + pool.register_with_event_loop(loop) + pool._pool.register_with_event_loop.assert_called_with(loop) + + def test_on_close(self): + pool = TaskPool(10) + pool._pool = Mock(name='pool') + pool._pool._state = mp.RUN + pool.on_close() + pool._pool.close.assert_called_with() + + def test_on_close__pool_not_running(self): + pool = TaskPool(10) + pool._pool = Mock(name='pool') + pool._pool._state = mp.CLOSE + pool.on_close() + self.assertFalse(pool._pool.close.called) + def test_apply_async(self): pool = TaskPool(10) pool.start() @@ -320,17 +416,3 @@ def test_num_processes(self): pool = TaskPool(7) pool.start() self.assertEqual(pool.num_processes, 7) - - def test_restart(self): - raise SkipTest('functional test') - - def get_pids(pool): - return {p.pid for p in pool._pool._pool} - - tp = self.TaskPool(5) - time.sleep(0.5) - tp.start() - pids = get_pids(tp) - tp.restart() - time.sleep(0.5) - self.assertEqual(pids, get_pids(tp)) diff --git a/celery/tests/fixups/test_django.py b/celery/tests/fixups/test_django.py index c2dffd41c..0249a5c95 100644 --- a/celery/tests/fixups/test_django.py +++ b/celery/tests/fixups/test_django.py @@ -12,7 +12,7 @@ ) from celery.tests.case import ( - AppCase, Mock, patch, patch_many, patch_modules, mask_modules, + AppCase, Mock, patch, patch_modules, mask_modules, ) @@ -63,15 +63,16 @@ def se(name): def test_install(self): self.app.loader = Mock() + self.cw = self.patch('os.getcwd') + self.p = self.patch('sys.path') + self.sigs = self.patch('celery.fixups.django.signals') with self.fixup_context(self.app) as (f, _, _): - with patch_many('os.getcwd', 'sys.path', - 'celery.fixups.django.signals') as (cw, p, sigs): - cw.return_value = '/opt/vandelay' - f.install() - sigs.worker_init.connect.assert_called_with(f.on_worker_init) - self.assertEqual(self.app.loader.now, f.now) - self.assertEqual(self.app.loader.mail_admins, f.mail_admins) - p.append.assert_called_with('/opt/vandelay') + self.cw.return_value = '/opt/vandelay' + f.install() + self.sigs.worker_init.connect.assert_called_with(f.on_worker_init) + self.assertEqual(self.app.loader.now, f.now) + self.assertEqual(self.app.loader.mail_admins, f.mail_admins) + self.p.append.assert_called_with('/opt/vandelay') def test_now(self): with self.fixup_context(self.app) as (f, _, _): @@ -114,7 +115,7 @@ def test_install(self): self.app.conf = {'CELERY_DB_REUSE_MAX': None} self.app.loader = Mock() with self.fixup_context(self.app) as (f, _, _): - with patch_many('celery.fixups.django.signals') as (sigs,): + with patch('celery.fixups.django.signals') as sigs: f.install() sigs.beat_embedded_init.connect.assert_called_with( f.close_database, diff --git a/celery/tests/security/test_certificate.py b/celery/tests/security/test_certificate.py index f9678f947..3cdc596c8 100644 --- a/celery/tests/security/test_certificate.py +++ b/celery/tests/security/test_certificate.py @@ -26,6 +26,11 @@ def test_has_expired(self): raise SkipTest('cert expired') self.assertFalse(Certificate(CERT1).has_expired()) + def test_has_expired_mock(self): + x = Certificate(CERT1) + x._cert = Mock(name='cert') + self.assertIs(x.has_expired(), x._cert.has_expired()) + class test_CertStore(SecurityCase): diff --git a/celery/tests/security/test_security.py b/celery/tests/security/test_security.py index 134efc9bb..ca560c73f 100644 --- a/celery/tests/security/test_security.py +++ b/celery/tests/security/test_security.py @@ -20,6 +20,7 @@ from celery.exceptions import ImproperlyConfigured, SecurityError from celery.five import builtins +from celery.security import disable_untrusted_serializers, setup_security from celery.security.utils import reraise_errors from kombu.serialization import registry @@ -53,6 +54,11 @@ def test_disable_insecure_serializers(self): finally: disable_insecure_serializers(allowed=['json']) + @patch('celery.security._disable_insecure_serializers') + def test_disable_untrusted_serializers(self, disable): + disable_untrusted_serializers(['foo']) + disable.assert_called_with(allowed=['foo']) + def test_setup_security(self): disabled = registry._disabled_content_types self.assertEqual(0, len(disabled)) @@ -62,6 +68,10 @@ def test_setup_security(self): self.assertIn('application/x-python-serialize', disabled) disabled.clear() + @patch('celery.current_app') + def test_setup_security__default_app(self, current_app): + setup_security() + @patch('celery.security.register_auth') @patch('celery.security._disable_insecure_serializers') def test_setup_registry_complete(self, dis, reg, key='KEY', cert='CERT'): diff --git a/celery/tests/tasks/test_tasks.py b/celery/tests/tasks/test_tasks.py index eef8d118a..1a02d9d18 100644 --- a/celery/tests/tasks/test_tasks.py +++ b/celery/tests/tasks/test_tasks.py @@ -6,13 +6,17 @@ from celery import Task -from celery.exceptions import Retry +from celery import group +from celery.app.task import _reprtask +from celery.exceptions import Ignore, Retry from celery.five import items, range, string_t from celery.result import EagerResult from celery.utils import uuid from celery.utils.timeutils import parse_iso8601 -from celery.tests.case import AppCase, depends_on_current_app, patch +from celery.tests.case import ( + AppCase, ContextMock, Mock, depends_on_current_app, patch, +) def return_True(*args, **kwargs): @@ -269,6 +273,20 @@ def xxx(): pass self.assertIs(pickle.loads(pickle.dumps(xxx)), xxx.app.tasks[xxx.name]) + @patch('celery.app.task.current_app') + @depends_on_current_app + def test_bind__no_app(self, current_app): + class XTask(Task): + _app = None + XTask._app = None + XTask.__bound__ = False + XTask.bind = Mock(name='bind') + self.assertIs(XTask.app, current_app) + XTask.bind.assert_called_with(current_app) + + def test_reprtask__no_fmt(self): + self.assertTrue(_reprtask(self.mytask)) + def test_AsyncResult(self): task_id = uuid() result = self.retry_task.AsyncResult(task_id) @@ -375,6 +393,47 @@ def test_regular_task(self): self.mytask.backend.mark_as_done(presult.id, result=None) self.assertTrue(presult.successful()) + def test_send_event(self): + mytask = self.mytask._get_current_object() + mytask.app.events = Mock(name='events') + mytask.app.events.attach_mock(ContextMock(), 'default_dispatcher') + mytask.request.id = 'fb' + mytask.send_event('task-foo', id=3122) + mytask.app.events.default_dispatcher().send.assert_called_with( + 'task-foo', uuid='fb', id=3122, + ) + + def test_replace(self): + sig1 = Mock(name='sig1') + with self.assertRaises(Ignore): + self.mytask.replace(sig1) + + def test_replace__group(self): + c = group([self.mytask.s()], app=self.app) + c.freeze = Mock(name='freeze') + c.delay = Mock(name='delay') + self.mytask.request.id = 'id' + self.mytask.request.group = 'group' + self.mytask.request.root_id = 'root_id', + with self.assertRaises(Ignore): + self.mytask.replace(c) + + def test_send_error_email_enabled(self): + mytask = self.increment_counter._get_current_object() + mytask.send_error_emails = True + mytask.disable_error_emails = False + mytask.ErrorMail = Mock(name='ErrorMail') + context = Mock(name='context') + exc = Mock(name='context') + mytask.send_error_email(context, exc, foo=1) + mytask.ErrorMail.assert_called_with(mytask, foo=1) + mytask.ErrorMail().send.assert_called_with(context, exc) + + def test_add_trail__no_trail(self): + mytask = self.increment_counter._get_current_object() + mytask.trail = False + mytask.add_trail('foo') + def test_repr_v2_compat(self): self.mytask.__v2_compat__ = True self.assertIn('v2 compatible', repr(self.mytask)) diff --git a/celery/tests/tasks/test_trace.py b/celery/tests/tasks/test_trace.py index 037acc4d6..aaaa6986c 100644 --- a/celery/tests/tasks/test_trace.py +++ b/celery/tests/tasks/test_trace.py @@ -1,12 +1,20 @@ from __future__ import absolute_import -from celery import uuid +from kombu.exceptions import EncodeError + +from celery import group, uuid from celery import signals from celery import states -from celery.exceptions import Ignore, Retry +from celery.exceptions import Ignore, Retry, Reject from celery.app.trace import ( TraceInfo, build_tracer, + get_log_policy, + log_policy_reject, + log_policy_ignore, + log_policy_internal, + log_policy_expected, + log_policy_unexpected, trace_task, setup_worker_optimizations, reset_worker_optimizations, @@ -60,6 +68,33 @@ def add_with_success(x, y): self.trace(add_with_success, (2, 2), {}) self.assertTrue(add_with_success.on_success.called) + def test_get_log_policy(self): + einfo = Mock(name='einfo') + einfo.internal = False + self.assertIs( + get_log_policy(self.add, einfo, Reject()), + log_policy_reject, + ) + self.assertIs( + get_log_policy(self.add, einfo, Ignore()), + log_policy_ignore, + ) + self.add.throws = (TypeError,) + self.assertIs( + get_log_policy(self.add, einfo, KeyError()), + log_policy_unexpected, + ) + self.assertIs( + get_log_policy(self.add, einfo, TypeError()), + log_policy_expected, + ) + einfo2 = Mock(name='einfo2') + einfo2.internal = True + self.assertIs( + get_log_policy(self.add, einfo2, KeyError()), + log_policy_internal, + ) + def test_trace_after_return(self): @self.app.task(shared=False, after_return=Mock()) @@ -134,6 +169,74 @@ def ignored(): retval, info = self.trace(ignored, (), {}) self.assertEqual(info.state, states.IGNORED) + def test_when_Reject(self): + + @self.app.task(shared=False) + def rejecting(): + raise Reject() + + retval, info = self.trace(rejecting, (), {}) + self.assertEqual(info.state, states.REJECTED) + + @patch('celery.canvas.maybe_signature') + def test_callbacks__scalar(self, maybe_signature): + sig = Mock(name='sig') + request = {'callbacks': [sig], 'root_id': 'root'} + maybe_signature.return_value = sig + retval, _ = self.trace(self.add, (2, 2), {}, request=request) + sig.apply_async.assert_called_with( + (4,), parent_id='id-1', root_id='root', + ) + + @patch('celery.canvas.maybe_signature') + def test_callbacks__EncodeError(self, maybe_signature): + sig = Mock(name='sig') + request = {'callbacks': [sig], 'root_id': 'root'} + maybe_signature.return_value = sig + sig.apply_async.side_effect = EncodeError() + retval, einfo = self.trace(self.add, (2, 2), {}, request=request) + self.assertEqual(einfo.state, states.FAILURE) + + @patch('celery.canvas.maybe_signature') + @patch('celery.app.trace.group.apply_async') + def test_callbacks__sigs(self, group_, maybe_signature): + sig1 = Mock(name='sig') + sig2 = Mock(name='sig2') + sig3 = group([Mock(name='g1'), Mock(name='g2')], app=self.app) + sig3.apply_async = Mock(name='gapply') + request = {'callbacks': [sig1, sig3, sig2], 'root_id': 'root'} + + def passt(s, *args, **kwargs): + return s + maybe_signature.side_effect = passt + retval, _ = self.trace(self.add, (2, 2), {}, request=request) + group_.assert_called_with( + (4,), parent_id='id-1', root_id='root', + ) + sig3.apply_async.assert_called_with( + (4,), parent_id='id-1', root_id='root', + ) + + @patch('celery.canvas.maybe_signature') + @patch('celery.app.trace.group.apply_async') + def test_callbacks__only_groups(self, group_, maybe_signature): + sig1 = group([Mock(name='g1'), Mock(name='g2')], app=self.app) + sig2 = group([Mock(name='g3'), Mock(name='g4')], app=self.app) + sig1.apply_async = Mock(name='gapply') + sig2.apply_async = Mock(name='gapply') + request = {'callbacks': [sig1, sig2], 'root_id': 'root'} + + def passt(s, *args, **kwargs): + return s + maybe_signature.side_effect = passt + retval, _ = self.trace(self.add, (2, 2), {}, request=request) + sig1.apply_async.assert_called_with( + (4,), parent_id='id-1', root_id='root', + ) + sig2.apply_async.assert_called_with( + (4,), parent_id='id-1', root_id='root', + ) + def test_trace_SystemExit(self): with self.assertRaises(SystemExit): self.trace(self.raises, (SystemExit(),), {}) @@ -184,6 +287,14 @@ def test_handle_error_state(self): store_errors=self.add_cast.store_errors_even_if_ignored, ) + @patch('celery.app.trace.ExceptionInfo') + def test_handle_reject(self, ExceptionInfo): + x = self.TI(states.FAILURE) + x._log_error = Mock(name='log_error') + req = Mock(name='req') + x.handle_reject(self.add, req) + x._log_error.assert_called_with(self.add, req, ExceptionInfo()) + class test_stackprotection(AppCase): diff --git a/celery/tests/utils/test_debug.py b/celery/tests/utils/test_debug.py new file mode 100644 index 000000000..739954a66 --- /dev/null +++ b/celery/tests/utils/test_debug.py @@ -0,0 +1,98 @@ +from __future__ import absolute_import, unicode_literals + +from celery.utils import debug + +from celery.tests.case import Case, Mock, patch + + +class test_on_blocking(Case): + + @patch('inspect.getframeinfo') + def test_on_blocking(self, getframeinfo): + frame = Mock(name='frame') + with self.assertRaises(RuntimeError): + debug._on_blocking(1, frame) + getframeinfo.assert_called_with(frame) + + +class test_blockdetection(Case): + + @patch('celery.utils.debug.signals') + def test_context(self, signals): + with debug.blockdetection(10): + signals.arm_alarm.assert_called_with(10) + signals.__setitem__.assert_called_with('ALRM', debug._on_blocking) + signals.__setitem__.assert_called_with('ALRM', signals['ALRM']) + signals.reset_alarm.assert_called_with() + + +class test_sample_mem(Case): + + @patch('celery.utils.debug.mem_rss') + def test_sample_mem(self, mem_rss): + prev, debug._mem_sample = debug._mem_sample, [] + try: + debug.sample_mem() + self.assertIs(debug._mem_sample[0], mem_rss()) + finally: + debug._mem_sample = prev + + +class test_sample(Case): + + def test_sample(self): + x = list(range(100)) + self.assertEqual( + list(debug.sample(x, 10)), + [0, 10, 20, 30, 40, 50, 60, 70, 80, 90], + ) + x = list(range(91)) + self.assertEqual( + list(debug.sample(x, 10)), + [0, 9, 18, 27, 36, 45, 54, 63, 72, 81], + ) + + +class test_hfloat(Case): + + def test_hfloat(self): + self.assertEqual(str(debug.hfloat(10, 5)), "10") + self.assertEqual(str(debug.hfloat(10.45645234234, 5)), "10.456") + + +class test_humanbytes(Case): + + def test_humanbytes(self): + self.assertEqual(debug.humanbytes(2 ** 20), "1MB") + self.assertEqual(debug.humanbytes(4 * 2 ** 20), "4MB") + self.assertEqual(debug.humanbytes(2 ** 16), "64kB") + self.assertEqual(debug.humanbytes(2 ** 16), "64kB") + self.assertEqual(debug.humanbytes(2 ** 8), "256b") + + +class test_mem_rss(Case): + + @patch('celery.utils.debug.ps') + @patch('celery.utils.debug.humanbytes') + def test_mem_rss(self, humanbytes, ps): + ret = debug.mem_rss() + ps.assert_called_with() + ps().get_memory_info.assert_called_with() + humanbytes.assert_called_with(ps().get_memory_info().rss) + self.assertIs(ret, humanbytes()) + ps.return_value = None + self.assertIsNone(debug.mem_rss()) + + +class test_ps(Case): + + @patch('celery.utils.debug.Process') + @patch('os.getpid') + def test_ps(self, getpid, Process): + prev, debug._process = debug._process, None + try: + debug.ps() + Process.assert_called_with(getpid()) + self.assertIs(debug._process, Process()) + finally: + debug._process = prev diff --git a/celery/tests/utils/test_mail.py b/celery/tests/utils/test_mail.py index e4fc9650d..3d9a17c42 100644 --- a/celery/tests/utils/test_mail.py +++ b/celery/tests/utils/test_mail.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -from celery.utils.mail import Message, Mailer, SSLError +from celery.utils.mail import Message, Mailer, SSLError, ErrorMail from celery.tests.case import Case, Mock, patch @@ -51,3 +51,33 @@ def test_send(self, SMTP): client.quit.side_effect = SSLError() mailer._send(msg) client.close.assert_called_with() + + +class test_ErrorMail(Case): + + def setUp(self): + self.task = Mock(name='task') + self.mailer = ErrorMail( + self.task, subject='foo{foo} ', body='bar{bar} ', + ) + + def test_should_send(self): + self.assertTrue(self.mailer.should_send(Mock(), Mock())) + + def test_format_subject(self): + self.assertEqual( + self.mailer.format_subject({'foo': 'FOO'}), + 'fooFOO', + ) + + def test_format_body(self): + self.assertEqual( + self.mailer.format_body({'bar': 'BAR'}), + 'barBAR', + ) + + def test_send(self): + self.mailer.send({'foo': 'FOO', 'bar': 'BAR'}, KeyError()) + self.task.app.mail_admins.assert_called_with( + 'fooFOO', 'barBAR', fail_silently=True, + ) diff --git a/celery/tests/utils/test_text.py b/celery/tests/utils/test_text.py index 1b0ca2805..8258e86b3 100644 --- a/celery/tests/utils/test_text.py +++ b/celery/tests/utils/test_text.py @@ -7,6 +7,7 @@ indent, pretty, truncate, + truncate_bytes, ) from celery.tests.case import AppCase, Case @@ -68,6 +69,10 @@ def test_truncate_text(self): self.assertEqual(truncate('ABCDEFGHI', 3), 'ABC...') self.assertEqual(truncate('ABCDEFGHI', 10), 'ABCDEFGHI') + def test_truncate_bytes(self): + self.assertEqual(truncate_bytes(b'ABCDEFGHI', 3), b'ABC...') + self.assertEqual(truncate_bytes(b'ABCDEFGHI', 10), b'ABCDEFGHI') + def test_abbr(self): self.assertEqual(abbr(None, 3), '???') self.assertEqual(abbr('ABCDEFGHI', 6), 'ABC...') diff --git a/celery/tests/utils/test_utils.py b/celery/tests/utils/test_utils.py index f9244dcbc..2b63252bb 100644 --- a/celery/tests/utils/test_utils.py +++ b/celery/tests/utils/test_utils.py @@ -8,6 +8,8 @@ from celery.utils import ( chunks, + deprecated_property, + isatty, is_iterable, cached_property, warn_deprecated, @@ -22,6 +24,15 @@ def double(x): return x * 2 +class test_isatty(Case): + + def test_tty(self): + fh = Mock(name='fh') + self.assertIs(isatty(fh), fh.isatty()) + fh.isatty.side_effect = AttributeError() + self.assertFalse(isatty(fh)) + + class test_worker_direct(Case): def test_returns_if_queue(self): @@ -29,6 +40,61 @@ def test_returns_if_queue(self): self.assertIs(worker_direct(q), q) +class test_deprecated_property(Case): + + @patch('celery.utils.warn_deprecated') + def test_deprecated(self, warn_deprecated): + + class X(object): + _foo = None + + @deprecated_property(deprecation='1.2') + def foo(self): + return self._foo + + @foo.setter + def foo(self, value): + self._foo = value + + @foo.deleter + def foo(self): + self._foo = None + self.assertTrue(X.foo) + self.assertTrue(X.foo.__set__(None, 1)) + self.assertTrue(X.foo.__delete__(None)) + x = X() + x.foo = 10 + warn_deprecated.assert_called_with( + stacklevel=3, deprecation='1.2', alternative=None, + description='foo', removal=None, + ) + warn_deprecated.reset_mock() + self.assertEqual(x.foo, 10) + warn_deprecated.assert_called_with( + stacklevel=3, deprecation='1.2', alternative=None, + description='foo', removal=None, + ) + warn_deprecated.reset_mock() + del(x.foo) + warn_deprecated.assert_called_with( + stacklevel=3, deprecation='1.2', alternative=None, + description='foo', removal=None, + ) + self.assertIsNone(x._foo) + + def test_deprecated_no_setter_or_deleter(self): + class X(object): + @deprecated_property(deprecation='1.2') + def foo(self): + pass + self.assertTrue(X.foo) + x = X() + with self.assertRaises(AttributeError): + x.foo = 10 + with self.assertRaises(AttributeError): + del(x.foo) + + class test_gen_task_name(Case): def test_no_module(self): @@ -54,8 +120,16 @@ def test_simple(self): self.assertTrue(jsonify(10.3)) self.assertTrue(jsonify('hello')) + unknown_type_filter = Mock() + obj = object() + self.assertIs( + jsonify(obj, unknown_type_filter=unknown_type_filter), + unknown_type_filter.return_value, + ) + unknown_type_filter.assert_called_with(obj) + with self.assertRaises(ValueError): - jsonify(object()) + jsonify(obj) class test_chunks(Case): diff --git a/celery/tests/worker/test_autoscale.py b/celery/tests/worker/test_autoscale.py index 21226ab6d..774d89b61 100644 --- a/celery/tests/worker/test_autoscale.py +++ b/celery/tests/worker/test_autoscale.py @@ -134,7 +134,7 @@ def test_shrink_raises_exception(self): x.scale_up(3) x._last_action = monotonic() - 10000 x.pool.shrink_raises_exception = True - x.scale_down(1) + x._shrink(1) @patch('celery.worker.autoscale.debug') def test_shrink_raises_ValueError(self, debug): diff --git a/celery/tests/worker/test_consumer.py b/celery/tests/worker/test_consumer.py index 88daff4ac..5880f07ee 100644 --- a/celery/tests/worker/test_consumer.py +++ b/celery/tests/worker/test_consumer.py @@ -41,6 +41,9 @@ def get_consumer(self, no_hub=False, **kwargs): consumer.conninfo = consumer.connection return consumer + def test_repr(self): + self.assertTrue(repr(self.get_consumer())) + def test_taskbuckets_defaultdict(self): c = self.get_consumer() self.assertIsNone(c.task_buckets['fooxasdwx.wewe']) @@ -68,6 +71,44 @@ def test_gevent_bug_disables_connection_timeout(self): self.get_consumer() self.assertIsNone(self.app.conf.broker_connection_timeout) + def test_limit_moved_to_pool(self): + with patch('celery.worker.consumer.task_reserved') as reserved: + c = self.get_consumer() + c.on_task_request = Mock(name='on_task_request') + request = Mock(name='request') + c._limit_move_to_pool(request) + reserved.assert_called_with(request) + c.on_task_request.assert_called_with(request) + + def test_update_prefetch_count(self): + c = self.get_consumer() + c._update_qos_eventually = Mock(name='update_qos') + c.initial_prefetch_count = None + c.pool.num_processes = None + c.prefetch_multiplier = 10 + self.assertIsNone(c._update_prefetch_count(1)) + c.initial_prefetch_count = 10 + c.pool.num_processes = 10 + c._update_prefetch_count(8) + c._update_qos_eventually.assert_called_with(8) + self.assertEqual(c.initial_prefetch_count, 10 * 10) + + def test_flush_events(self): + c = self.get_consumer() + c.event_dispatcher = None + c._flush_events() + c.event_dispatcher = Mock(name='evd') + c._flush_events() + c.event_dispatcher.flush.assert_called_with() + + def test_on_send_event_buffered(self): + c = self.get_consumer() + c.hub = None + c.on_send_event_buffered() + c.hub = Mock(name='hub') + c.on_send_event_buffered() + c.hub._ready.add.assert_called_with(c._flush_events) + def test_limit_task(self): c = self.get_consumer() @@ -460,6 +501,14 @@ def test_periodic(self): with self.assertRaises(KeyError): state.workers['foo'] + def test_on_message__task(self): + c = self.Consumer() + g = Gossip(c) + self.assertTrue(g.enabled) + message = Mock(name='message') + message.delivery_info = {'routing_key': 'task.failed'} + g.on_message(Mock(name='prepare'), message) + def test_on_message(self): c = self.Consumer() g = Gossip(c) diff --git a/celery/tests/worker/test_control.py b/celery/tests/worker/test_control.py index d2cd234af..73896a55c 100644 --- a/celery/tests/worker/test_control.py +++ b/celery/tests/worker/test_control.py @@ -18,7 +18,6 @@ from celery.worker import state as worker_state from celery.worker.request import Request from celery.worker.state import revoked -from celery.worker.control import Panel from celery.worker.pidbox import Pidbox, gPidbox from celery.tests.case import AppCase, Mock, TaskMessage, call, patch @@ -132,7 +131,7 @@ def create_state(self, **kwargs): def create_panel(self, **kwargs): return self.app.control.mailbox.Node(hostname=hostname, state=self.create_state(**kwargs), - handlers=Panel.data) + handlers=control.Panel.data) def test_enable_events(self): consumer = Consumer(self.app) @@ -168,21 +167,36 @@ def test_hello(self): consumer = Consumer(self.app) panel = self.create_panel(consumer=consumer) panel.state.app.clock.value = 313 + panel.state.hostname = 'elaine@vandelay.com' worker_state.revoked.add('revoked1') try: - x = panel.handle('hello', {'from_node': 'george@vandelay.com'}) - self.assertIn('revoked1', x['revoked']) + self.assertIsNone(panel.handle('hello', { + 'from_node': 'elaine@vandelay.com', + })) + x = panel.handle('hello', { + 'from_node': 'george@vandelay.com', + }) self.assertEqual(x['clock'], 314) # incremented + x = panel.handle('hello', { + 'from_node': 'george@vandelay.com', + 'revoked': {'1234', '4567', '891'} + }) + self.assertIn('revoked1', x['revoked']) + self.assertIn('1234', x['revoked']) + self.assertIn('4567', x['revoked']) + self.assertIn('891', x['revoked']) + self.assertEqual(x['clock'], 315) # incremented finally: worker_state.revoked.discard('revoked1') def test_conf(self): - return consumer = Consumer(self.app) panel = self.create_panel(consumer=consumer) - self.app.conf.SOME_KEY6 = 'hello world' + panel.app = self.app + panel.app.finalize() + self.app.conf.some_key6 = 'hello world' x = panel.handle('dump_conf') - self.assertIn('SOME_KEY6', x) + self.assertIn('some_key6', x) def test_election(self): consumer = Consumer(self.app) @@ -193,6 +207,14 @@ def test_election(self): ) consumer.gossip.election.assert_called_with('id', 'topic', 'action') + def test_election__no_gossip(self): + consumer = Mock(name='consumer') + consumer.gossip = None + panel = self.create_panel(consumer=consumer) + panel.handle( + 'election', {'id': 'id', 'topic': 'topic', 'action': 'action'}, + ) + def test_heartbeat(self): consumer = Consumer(self.app) panel = self.create_panel(consumer=consumer) @@ -236,11 +258,27 @@ def test_active_queues(self): self.assertListEqual(list(sorted(q['name'] for q in r)), ['bar', 'foo']) + def test_active_queues__empty(self): + consumer = Mock(name='consumer') + panel = self.create_panel(consumer=consumer) + consumer.task_consumer = None + self.assertFalse(panel.handle('active_queues')) + def test_dump_tasks(self): info = '\n'.join(self.panel.handle('dump_tasks')) self.assertIn('mytask', info) self.assertIn('rate_limit=200', info) + def test_dump_tasks2(self): + prev, control.DEFAULT_TASK_INFO_ITEMS = ( + control.DEFAULT_TASK_INFO_ITEMS, []) + try: + info = '\n'.join(self.panel.handle('dump_tasks')) + self.assertIn('mytask', info) + self.assertNotIn('rate_limit=200', info) + finally: + control.DEFAULT_TASK_INFO_ITEMS = prev + def test_stats(self): prev_count, worker_state.total_count = worker_state.total_count, 100 try: @@ -493,7 +531,7 @@ def reply(self, data, exchange, routing_key, **kwargs): panel = _Node(hostname=hostname, state=self.create_state(consumer=Consumer(self.app)), - handlers=Panel.data, + handlers=control.Panel.data, mailbox=self.app.control.mailbox) r = panel.dispatch('ping', reply_to={'exchange': 'x', 'routing_key': 'x'}) @@ -584,3 +622,30 @@ def test_pool_restart_reload_modules(self): self.assertTrue(consumer.controller.pool.restart.called) self.assertTrue(_reload.called) self.assertFalse(_import.called) + + def test_query_task(self): + consumer = Consumer(self.app) + consumer.controller = _WC(app=self.app) + consumer.controller.consumer = consumer + panel = self.create_panel(consumer=consumer) + panel.app = self.app + req1 = Request( + TaskMessage(self.mytask.name, args=(2, 2)), + app=self.app, + ) + worker_state.reserved_requests.add(req1) + try: + self.assertFalse(panel.handle('query_task', {'ids': {'1daa'}})) + ret = panel.handle('query_task', {'ids': {req1.id}}) + self.assertIn(req1.id, ret) + self.assertEqual(ret[req1.id][0], 'reserved') + worker_state.active_requests.add(req1) + try: + ret = panel.handle('query_task', {'ids': {req1.id}}) + self.assertEqual(ret[req1.id][0], 'active') + finally: + worker_state.active_requests.clear() + ret = panel.handle('query_task', {'ids': {req1.id}}) + self.assertEqual(ret[req1.id][0], 'reserved') + finally: + worker_state.reserved_requests.clear() diff --git a/celery/tests/worker/test_loops.py b/celery/tests/worker/test_loops.py index 306a61c7e..f8dc07f7b 100644 --- a/celery/tests/worker/test_loops.py +++ b/celery/tests/worker/test_loops.py @@ -1,11 +1,14 @@ from __future__ import absolute_import +import errno import socket from kombu.async import Hub, READ, WRITE, ERR from celery.bootsteps import CLOSE, RUN -from celery.exceptions import InvalidTaskError, WorkerShutdown, WorkerTerminate +from celery.exceptions import ( + InvalidTaskError, WorkerLostError, WorkerShutdown, WorkerTerminate, +) from celery.five import Empty from celery.platforms import EX_FAILURE from celery.worker import state @@ -129,6 +132,13 @@ def test_drain_after_consume(self): _quick_drain, [p.fun for p in x.hub._ready], ) + def test_pool_did_not_start_at_startup(self): + x = X(self.app) + x.obj.restart_count = 0 + x.obj.pool.did_start_ok.return_value = False + with self.assertRaises(WorkerLostError): + asynloop(*x.args) + def test_setup_heartbeat(self): x = X(self.app, heartbeat=10) x.hub.call_repeatedly = Mock(name='x.hub.call_repeatedly()') @@ -423,3 +433,26 @@ def test_ignores_socket_errors_when_closed(self): x = X(self.app) x.close_then_error(x.connection.drain_events) self.assertIsNone(synloop(*x.args)) + + +class test_quick_drain(AppCase): + + def setup(self): + self.connection = Mock(name='connection') + + def test_drain(self): + _quick_drain(self.connection, timeout=33.3) + self.connection.drain_events.assert_called_with(timeout=33.3) + + def test_drain_error(self): + exc = KeyError() + exc.errno = 313 + self.connection.drain_events.side_effect = exc + with self.assertRaises(KeyError): + _quick_drain(self.connection, timeout=33.3) + + def test_drain_error_EAGAIN(self): + exc = KeyError() + exc.errno = errno.EAGAIN + self.connection.drain_events.side_effect = exc + _quick_drain(self.connection, timeout=33.3) diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index 1eca31def..3f73dfa20 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -12,10 +12,8 @@ from kombu.common import QoS, ignore_errors from kombu.transport.base import Message -from celery.app.defaults import DEFAULTS from celery.bootsteps import RUN, CLOSE, StartStopStep from celery.concurrency.base import BasePool -from celery.datastructures import AttributeDict from celery.exceptions import ( WorkerShutdown, WorkerTerminate, TaskRevokedError, InvalidTaskError, ) @@ -30,9 +28,7 @@ from celery.utils.serialization import pickle from celery.utils.timer2 import Timer -from celery.tests.case import ( - AppCase, Mock, SkipTest, TaskMessage, patch, restore_logging, -) +from celery.tests.case import AppCase, Mock, SkipTest, TaskMessage, patch def MockStep(step=None): @@ -875,47 +871,6 @@ def test_use_pidfile(self, create_pidlock): worker.stop() self.assertTrue(worker.pidlock.release.called) - @patch('celery.platforms.signals') - @patch('celery.platforms.set_mp_process_title') - def test_process_initializer(self, set_mp_process_title, _signals): - with restore_logging(): - from celery import signals - from celery._state import _tls - from celery.concurrency.prefork import ( - process_initializer, WORKER_SIGRESET, WORKER_SIGIGNORE, - ) - - def on_worker_process_init(**kwargs): - on_worker_process_init.called = True - on_worker_process_init.called = False - signals.worker_process_init.connect(on_worker_process_init) - - def Loader(*args, **kwargs): - loader = Mock(*args, **kwargs) - loader.conf = {} - loader.override_backends = {} - return loader - - with self.Celery(loader=Loader) as app: - app.conf = AttributeDict(DEFAULTS) - process_initializer(app, 'awesome.worker.com') - _signals.ignore.assert_any_call(*WORKER_SIGIGNORE) - _signals.reset.assert_any_call(*WORKER_SIGRESET) - self.assertTrue(app.loader.init_worker.call_count) - self.assertTrue(on_worker_process_init.called) - self.assertIs(_tls.current_app, app) - set_mp_process_title.assert_called_with( - 'celeryd', hostname='awesome.worker.com', - ) - - with patch('celery.app.trace.setup_worker_optimizations') as S: - os.environ['FORKED_BY_MULTIPROCESSING'] = "1" - try: - process_initializer(app, 'luke.worker.com') - S.assert_called_with(app, 'luke.worker.com') - finally: - os.environ.pop('FORKED_BY_MULTIPROCESSING', None) - def test_attrs(self): worker = self.worker self.assertIsNotNone(worker.timer) diff --git a/celery/utils/abstract.py b/celery/utils/abstract.py index 669f347fb..f2a7e1504 100644 --- a/celery/utils/abstract.py +++ b/celery/utils/abstract.py @@ -32,7 +32,7 @@ def _subclasshook_using(cls, parent, C): ) or NotImplemented -class CallableTask(_AbstractClass, Callable): +class CallableTask(_AbstractClass, Callable): # pragma: no cover __required_attributes__ = frozenset({ 'delay', 'apply_async', 'apply', }) @@ -54,7 +54,7 @@ def __subclasshook__(cls, C): return cls._subclasshook_using(CallableTask, C) -class CallableSignature(CallableTask): +class CallableSignature(CallableTask): # pragma: no cover __required_attributes__ = frozenset({ 'clone', 'freeze', 'set', 'link', 'link_error', '__or__', }) diff --git a/celery/utils/debug.py b/celery/utils/debug.py index 79ac4e1e3..50a2b8282 100644 --- a/celery/utils/debug.py +++ b/celery/utils/debug.py @@ -31,7 +31,7 @@ (2 ** 30.0, 'GB'), (2 ** 20.0, 'MB'), (2 ** 10.0, 'kB'), - (0.0, '{0!d}b'), + (0.0, 'b'), ) _process = None @@ -78,7 +78,7 @@ def sample_mem(): return current_rss -def _memdump(samples=10): +def _memdump(samples=10): # pragma: no cover S = _mem_sample prev = list(S) if len(S) <= samples else sample(S, samples) _mem_sample[:] = [] @@ -88,7 +88,7 @@ def _memdump(samples=10): return prev, after_collect -def memdump(samples=10, file=None): +def memdump(samples=10, file=None): # pragma: no cover """Dump memory statistics. Will print a sample of all RSS memory samples added by @@ -151,7 +151,7 @@ def mem_rss(): return humanbytes(p.get_memory_info().rss) -def ps(): +def ps(): # pragma: no cover """Return the global :class:`psutil.Process` instance, or :const:`None` if :mod:`psutil` is not installed.""" global _process diff --git a/celery/worker/control.py b/celery/worker/control.py index 36f066b03..69bd42d00 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -52,21 +52,14 @@ def _find_requests_by_id(ids, requests): @Panel.register def query_task(state, ids, **kwargs): ids = maybe_list(ids) - - def reqinfo(state, req): - return state, req.info() - - reqs = { + return dict({ req.id: ('reserved', req.info()) for req in _find_requests_by_id(ids, worker_state.reserved_requests) - } - reqs.update({ + }, **{ req.id: ('active', req.info()) for req in _find_requests_by_id(ids, worker_state.active_requests) }) - return reqs - @Panel.register def revoke(state, task_id, terminate=False, signal=None, **kwargs): @@ -368,7 +361,6 @@ def active_queues(state): def _wanted_config_key(key): return (isinstance(key, string_t) and - key.isupper() and not key.startswith('__')) From 2208158efb7f1d5410b646c518c930c3c4f72600 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 17 Nov 2015 19:32:55 -0800 Subject: [PATCH 0804/1103] flakes --- celery/backends/cassandra.py | 2 +- celery/tests/backends/test_cassandra.py | 10 +++------- celery/tests/worker/test_worker.py | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index 631c104b7..3caa7d255 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -129,7 +129,7 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, def process_cleanup(self): if self._connection is not None: - self._connection.shutdown() # also shuts down _session + self._connection.shutdown() # also shuts down _session self._connection = None self._session = None diff --git a/celery/tests/backends/test_cassandra.py b/celery/tests/backends/test_cassandra.py index 84bb05aa9..1875b2005 100644 --- a/celery/tests/backends/test_cassandra.py +++ b/celery/tests/backends/test_cassandra.py @@ -106,9 +106,8 @@ def test_process_cleanup(self): self.assertIsNone(x._session) def test_timeouting_cluster(self): - """ - Tests behaviour when Cluster.connect raises cassandra.OperationTimedOut - """ + """Tests behaviour when Cluster.connect raises + cassandra.OperationTimedOut.""" with mock_module(*CASSANDRA_MODULES): from celery.backends import cassandra as mod @@ -139,11 +138,8 @@ def shutdown(self): x.process_cleanup() # should not raise - def test_please_free_memory(self): - """ - Ensure that Cluster object IS shut down. - """ + """Ensure that Cluster object IS shut down.""" with mock_module(*CASSANDRA_MODULES): from celery.backends import cassandra as mod diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index 3f73dfa20..7ea6da27d 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -47,7 +47,7 @@ def mock_event_dispatcher(): class PlaceHolder(object): - pass + pass def find_step(obj, typ): From b27856df0e2b36fdb0c5da47361d7e0dc6f42b1b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 18 Nov 2015 16:35:05 -0800 Subject: [PATCH 0805/1103] Joining in task now raises RuntimeError instead of warning --- celery/result.py | 5 +---- celery/tests/tasks/test_result.py | 9 +++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/celery/result.py b/celery/result.py index ddda0051e..dc0cd6563 100644 --- a/celery/result.py +++ b/celery/result.py @@ -33,15 +33,12 @@ Never call result.get() within a task! See http://docs.celeryq.org/en/latest/userguide/tasks.html\ #task-synchronous-subtasks - -In Celery 4.0 this will result in an exception being -raised instead of just being a warning. """ def assert_will_not_block(): if task_join_will_block(): - warnings.warn(RuntimeWarning(E_WOULDBLOCK)) + raise RuntimeError(E_WOULDBLOCK) @contextmanager diff --git a/celery/tests/tasks/test_result.py b/celery/tests/tasks/test_result.py index 067998885..b9c9bd45b 100644 --- a/celery/tests/tasks/test_result.py +++ b/celery/tests/tasks/test_result.py @@ -9,6 +9,7 @@ AsyncResult, EagerResult, result_from_tuple, + assert_will_not_block, ) from celery.utils import uuid from celery.utils.serialization import pickle @@ -57,6 +58,14 @@ def mytask(): pass self.mytask = mytask + @patch('celery.result.task_join_will_block') + def test_assert_will_not_block(self, task_join_will_block): + task_join_will_block.return_value = True + with self.assertRaises(RuntimeError): + assert_will_not_block() + task_join_will_block.return_value = False + assert_will_not_block() + def test_compat_properties(self): x = self.app.AsyncResult('1') self.assertEqual(x.task_id, x.id) From 8e45b264404957f57bcdb341af86554052df0c35 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 19 Nov 2015 12:38:50 -0800 Subject: [PATCH 0806/1103] events.state: Deprecates methods/properties scheduled for removal in 4.0 - Worker.update_heartbeat Use Worker.event(None, timestamp, received) - Worker.on_online Use Worker.event('online', timestamp, received, fields) - Worker.on_offline Use Worker.event('offline', timestamp, received, fields) - Worker.on_heartbeat Use Worker.event('heartbeat', timestamp, received, fields) - Worker._defaults {k: getattr(worker, k) for k in worker._fields} - Task.on_sent Use Task.event('sent', timestamp, received, fields) - Task.on_received Use Task.event('received', timestamp, received, fields) - Task.on_started Use Task.event('started', timestamp, received, fields) - Task.on_failed Use Task.event('failed', timestamp, received, fields) - Task.on_retried Use Task.event('retried', timestamp, received, fields) - Task.on_succeeded Use Task.event('succeeded', timestamp, received, fields) - Task.on_revoked Use Task.event('revoked', timestamp, received, fields) - Task.on_unknown_event Use Task.event(short_type, timestamp, received, fields) - Task.update Use Task.event(short_type, timestamp, received, fields) - Task.merge Utility function, not really public --- celery/events/state.py | 72 ++---------------------------------------- 1 file changed, 3 insertions(+), 69 deletions(-) diff --git a/celery/events/state.py b/celery/events/state.py index bc03f0c78..c0fcef094 100644 --- a/celery/events/state.py +++ b/celery/events/state.py @@ -98,7 +98,8 @@ def __eq__(this, other): cls.__eq__ = __eq__ def __ne__(this, other): - return not this.__eq__(other) + res = this.__eq__(other) + return True if res is NotImplemented else not res cls.__ne__ = __ne__ def __hash__(this): @@ -118,7 +119,7 @@ class Worker(object): _fields = ('hostname', 'pid', 'freq', 'heartbeats', 'clock', 'active', 'processed', 'loadavg', 'sw_ident', 'sw_ver', 'sw_sys') - if not PYPY: + if not PYPY: # pragma: no cover __slots__ = _fields + ('event', '__dict__', '__weakref__') def __init__(self, hostname=None, pid=None, freq=60, @@ -200,28 +201,6 @@ def alive(self, nowfun=time): def id(self): return '{0.hostname}.{0.pid}'.format(self) - @deprecated(4.0, 5.0) - def update_heartbeat(self, received, timestamp): - self.event(None, timestamp, received) - - @deprecated(4.0, 5.0) - def on_online(self, timestamp=None, local_received=None, **fields): - self.event('online', timestamp, local_received, fields) - - @deprecated(4.0, 5.0) - def on_offline(self, timestamp=None, local_received=None, **fields): - self.event('offline', timestamp, local_received, fields) - - @deprecated(4.0, 5.0) - def on_heartbeat(self, timestamp=None, local_received=None, **fields): - self.event('heartbeat', timestamp, local_received, fields) - - @class_property - def _defaults(cls): - """Deprecated, to be removed in 5.0""" - source = cls() - return {k: getattr(source, k) for k in cls._fields} - @with_unique_field('uuid') class Task(object): @@ -345,51 +324,6 @@ def origin(self): def ready(self): return self.state in states.READY_STATES - @deprecated(4.0, 5.0) - def on_sent(self, timestamp=None, **fields): - self.event('sent', timestamp, fields) - - @deprecated(4.0, 5.0) - def on_received(self, timestamp=None, **fields): - self.event('received', timestamp, fields) - - @deprecated(4.0, 5.0) - def on_started(self, timestamp=None, **fields): - self.event('started', timestamp, fields) - - @deprecated(4.0, 5.0) - def on_failed(self, timestamp=None, **fields): - self.event('failed', timestamp, fields) - - @deprecated(4.0, 5.0) - def on_retried(self, timestamp=None, **fields): - self.event('retried', timestamp, fields) - - @deprecated(4.0, 5.0) - def on_succeeded(self, timestamp=None, **fields): - self.event('succeeded', timestamp, fields) - - @deprecated(4.0, 5.0) - def on_revoked(self, timestamp=None, **fields): - self.event('revoked', timestamp, fields) - - @deprecated(4.0, 5.0) - def on_unknown_event(self, shortype, timestamp=None, **fields): - self.event(shortype, timestamp, fields) - - @deprecated(4.0, 5.0) - def update(self, state, timestamp, fields, - _state=states.state, RETRY=states.RETRY): - return self.event(state, timestamp, None, fields) - - @deprecated(4.0, 5.0) - def merge(self, state, timestamp, fields): - keep = self.merge_rules.get(state) - if keep is not None: - fields = {k: v for k, v in items(fields) if k in keep} - for key, value in items(fields): - setattr(self, key, value) - @class_property def _defaults(cls): """Deprecated, to be removed in 5.0.""" From 89fa04c0e5125d6253d6ca0db11c1af076b20c10 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 19 Nov 2015 13:12:51 -0800 Subject: [PATCH 0807/1103] Use redis.StrictRedis --- celery/backends/redis.py | 4 ++-- celery/tests/backends/test_redis.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 3af35cd96..cf11eacc8 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -149,7 +149,7 @@ def set(self, key, value, **retry_policy): def _set(self, key, value): with self.client.pipeline() as pipe: if self.expires: - pipe.setex(key, value, self.expires) + pipe.setex(key, self.expires, value) else: pipe.set(key, value) pipe.publish(key, value) @@ -237,7 +237,7 @@ def on_chord_part_return(self, request, state, result, propagate=None): def _create_client(self, socket_timeout=None, socket_connect_timeout=None, **params): - return self.redis.Redis( + return self.redis.StrictRedis( connection_pool=self.ConnectionPool( socket_timeout=socket_timeout and float(socket_timeout), socket_connect_timeout=socket_connect_timeout and float( diff --git a/celery/tests/backends/test_redis.py b/celery/tests/backends/test_redis.py index cbb534f5e..8f2c2a76e 100644 --- a/celery/tests/backends/test_redis.py +++ b/celery/tests/backends/test_redis.py @@ -63,7 +63,7 @@ def __init__(self, host=None, port=None, db=None, password=None, **kw): def get(self, key): return self.keyspace.get(key) - def setex(self, key, value, expires): + def setex(self, key, expires, value): self.set(key, value) self.expire(key, expires) @@ -98,7 +98,7 @@ def llen(self, key): class redis(object): - Redis = Redis + StrictRedis = Redis class ConnectionPool(object): From 9982773022d3de2d41ca59509220763da527b20e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 13:34:16 -0800 Subject: [PATCH 0808/1103] Group is now lazy until .apply_async, but having regen support lazy __getitem__ for iterators --- celery/canvas.py | 48 ++++++++++++++++++++------------------ celery/utils/functional.py | 32 +++++++++++++++++++++---- 2 files changed, 52 insertions(+), 28 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 1367a633c..09db879f5 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -27,7 +27,7 @@ from celery.result import GroupResult from celery.utils import abstract from celery.utils.functional import ( - maybe_list, is_list, regen, chunks as _chunks, + maybe_list, is_list, _regen, regen, chunks as _chunks, ) from celery.utils.text import truncate @@ -661,7 +661,7 @@ def _maybe_group(tasks, app): elif isinstance(tasks, abstract.CallableSignature): tasks = [tasks] else: - tasks = [signature(t, app=app) for t in regen(tasks)] + tasks = [signature(t, app=app) for t in tasks] return tasks @@ -670,9 +670,12 @@ class group(Signature): tasks = _getitem_property('kwargs.tasks') def __init__(self, *tasks, **options): - app = options.get('app') if len(tasks) == 1: - tasks = _maybe_group(tasks[0], app) + tasks = tasks[0] + if isinstance(tasks, group): + tasks = tasks.tasks + if not isinstance(tasks, _regen): + tasks = regen(tasks) Signature.__init__( self, 'celery.group', (), {'tasks': tasks}, **options ) @@ -691,25 +694,24 @@ def _prepared(self, tasks, partial_args, group_id, root_id, app, dict=dict, CallableSignature=abstract.CallableSignature, from_dict=Signature.from_dict): for task in tasks: - if isinstance(task, dict): - if isinstance(task, CallableSignature): - # local sigs are always of type Signature, and we - # clone them to make sure we do not modify the originals. - task = task.clone() - else: - # serialized sigs must be converted to Signature. - task = from_dict(task, app=app) - if isinstance(task, group): - # needs yield_from :( - unroll = task._prepared( - task.tasks, partial_args, group_id, root_id, app, - ) - for taskN, resN in unroll: - yield taskN, resN - else: - if partial_args and not task.immutable: - task.args = tuple(partial_args) + tuple(task.args) - yield task, task.freeze(group_id=group_id, root_id=root_id) + if isinstance(task, CallableSignature): + # local sigs are always of type Signature, and we + # clone them to make sure we do not modify the originals. + task = task.clone() + else: + # serialized sigs must be converted to Signature. + task = from_dict(task, app=app) + if isinstance(task, group): + # needs yield_from :( + unroll = task._prepared( + task.tasks, partial_args, group_id, root_id, app, + ) + for taskN, resN in unroll: + yield taskN, resN + else: + if partial_args and not task.immutable: + task.args = tuple(partial_args) + tuple(task.args) + yield task, task.freeze(group_id=group_id, root_id=root_id) def _apply_tasks(self, tasks, producer=None, app=None, add_to_parent=None, **options): diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 31ebbfed1..1966b9003 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -14,10 +14,9 @@ from collections import OrderedDict from functools import partial, wraps from inspect import getargspec, isfunction -from itertools import islice +from itertools import chain, islice from amqp import promise -from kombu.utils import cached_property from kombu.utils.functional import lazy, maybe_evaluate, is_list, maybe_list from celery.five import UserDict, UserList, items, keys, range @@ -320,6 +319,8 @@ class _regen(UserList, list): # must be subclass of list so that json can encode. def __init__(self, it): self.__it = it + self.__index = 0 + self.__consumed = [] def __reduce__(self): return list, (self.data,) @@ -327,9 +328,30 @@ def __reduce__(self): def __length_hint__(self): return self.__it.__length_hint__() - @cached_property + def __iter__(self): + return chain(self.__consumed, self.__it) + + def __getitem__(self, index): + if index < 0: + return self.data[index] + try: + return self.__consumed[index] + except IndexError: + try: + for i in range(self.__index, index + 1): + self.__consumed.append(next(self.__it)) + except StopIteration: + raise IndexError(index) + else: + return self.__consumed[index] + + @property def data(self): - return list(self.__it) + try: + self.__consumed.extend(list(self.__it)) + except StopIteration: + pass + return self.__consumed def dictfilter(d=None, **kw): @@ -365,7 +387,7 @@ def head_from_fun(fun, bound=False, debug=False): fun_args=_argsfromspec(getargspec(fun)), fun_value=1, ) - if debug: + if debug: # pragma: no cover print(definition, file=sys.stderr) namespace = {'__name__': 'headof_{0}'.format(name)} exec(definition, namespace) From 85edb51823472b7b6dad3954400f5c8f2dbccdf7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 13:34:32 -0800 Subject: [PATCH 0809/1103] celery.signature did not pass app properly for custom Signatures. --- celery/canvas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index 09db879f5..1cefab080 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -999,10 +999,11 @@ def __repr__(self): def signature(varies, *args, **kwargs): + app = kwargs.get('app') if isinstance(varies, dict): if isinstance(varies, abstract.CallableSignature): return varies.clone() - return Signature.from_dict(varies) + return Signature.from_dict(varies, app=app) return Signature(varies, *args, **kwargs) subtask = signature # XXX compat From 91f1d4c87bfe4385c9d2d1a4682a9f4ccb88a936 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 13:36:24 -0800 Subject: [PATCH 0810/1103] [>3.1] Result.__ne__ did not take NotImplemented into account --- celery/result.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/celery/result.py b/celery/result.py index dc0cd6563..3754e92e8 100644 --- a/celery/result.py +++ b/celery/result.py @@ -299,7 +299,8 @@ def __eq__(self, other): return NotImplemented def __ne__(self, other): - return not self.__eq__(other) + res = self.__eq__(other) + return True if res is NotImplemented else not res def __copy__(self): return self.__class__( @@ -717,7 +718,8 @@ def __eq__(self, other): return NotImplemented def __ne__(self, other): - return not self.__eq__(other) + res = self.__eq__(other) + return True if res is NotImplemented else not res def __repr__(self): return '<{0}: [{1}]>'.format(type(self).__name__, @@ -802,7 +804,8 @@ def __eq__(self, other): return NotImplemented def __ne__(self, other): - return not self.__eq__(other) + res = self.__eq__(other) + return True if res is NotImplemented else not res def __repr__(self): return '<{0}: {1} [{2}]>'.format(type(self).__name__, self.id, From bc40a250f839a79e09567bbecee86dee0ba0fc26 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 15:19:48 -0800 Subject: [PATCH 0811/1103] [>3.1?] Chord did not set app correctly. --- celery/canvas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index 1cefab080..f3c99ca11 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -857,7 +857,7 @@ def __init__(self, header, body=None, task='celery.chord', Signature.__init__( self, task, args, dict(kwargs, header=_maybe_group(header, app), - body=maybe_signature(body, app=self._app)), **options + body=maybe_signature(body, app=app)), app=app, **options ) self.subtask_type = 'chord' From 914453d3defba424b3d8b4065b22dc4da8adb2d5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 18:39:20 -0800 Subject: [PATCH 0812/1103] Fixes bug with detect_settings --- celery/app/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/celery/app/utils.py b/celery/app/utils.py index 9078294a8..396d06538 100644 --- a/celery/app/utils.py +++ b/celery/app/utils.py @@ -181,15 +181,17 @@ def detect_settings(conf, preconf={}, ignore_keys=set(), prefix=None, is_in_new = have.intersection(all_keys) is_in_old = have.intersection(old_keys) + info = None if is_in_new: # have new setting names info, left = _settings_info, is_in_old if is_in_old and len(is_in_old) > len(is_in_new): # Majority of the settings are old. info, left = _old_settings_info, is_in_new - elif is_in_old: + if is_in_old: # have old setting names, or a majority of the names are old. - info, left = _old_settings_info, is_in_new + if not info: + info, left = _old_settings_info, is_in_new if is_in_new and len(is_in_new) > len(is_in_old): # Majority of the settings are new info, left = _settings_info, is_in_old From c694a0ae40ae36c48e753990458e1a1d7ae35119 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 18:39:58 -0800 Subject: [PATCH 0813/1103] MongoDB: This code was not reached --- celery/backends/mongodb.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index bd1075ba7..fe863ea56 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -31,7 +31,9 @@ from pymongo.errors import InvalidDocument # noqa else: # pragma: no cover Binary = None # noqa - InvalidDocument = None # noqa + + class InvalidDocument(Exception): # noqa + pass __all__ = ['MongoBackend'] @@ -83,6 +85,9 @@ def __init__(self, app=None, url=None, **kwargs): # update conf with mongo uri data, only if uri was given if self.url: + if self.url == 'mongodb://': + self.url += 'localhost' + uri_data = pymongo.uri_parser.parse_uri(self.url) # build the hosts list to create a mongo connection hostslist = [ @@ -149,10 +154,6 @@ def _get_connection(self): if isinstance(host, string_t) \ and not host.startswith('mongodb://'): host = 'mongodb://{0}:{1}'.format(host, self.port) - - if host == 'mongodb://': - host += 'localhost' - # don't change self.options conf = dict(self.options) conf['host'] = host From bca90b239dd305d6b8355a5dd7b5857df04be9f3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 18:40:21 -0800 Subject: [PATCH 0814/1103] Redis: No longer implements_incr with new_join --- celery/backends/redis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index cf11eacc8..06554b83a 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -54,7 +54,6 @@ class RedisBackend(KeyValueStoreBackend): supports_autoexpire = True supports_native_join = True - implements_incr = True def __init__(self, host=None, port=None, db=None, password=None, max_connections=None, url=None, From 5b4d6ffcbec689fc3b8b0cc06154eed2bd4d896f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 18:41:10 -0800 Subject: [PATCH 0815/1103] Removed Py2.6-ism --- celery/bin/celeryd_detach.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/celery/bin/celeryd_detach.py b/celery/bin/celeryd_detach.py index d9d6141d7..c845a72ff 100644 --- a/celery/bin/celeryd_detach.py +++ b/celery/bin/celeryd_detach.py @@ -121,10 +121,7 @@ class detached_celeryd(object): 'for the list of supported worker arguments.') command = sys.executable execv_path = sys.executable - if sys.version_info < (2, 7): # does not support pkg/__main__.py - execv_argv = ['-m', 'celery.__main__', 'worker'] - else: - execv_argv = ['-m', 'celery', 'worker'] + execv_argv = ['-m', 'celery', 'worker'] def __init__(self, app=None): self.app = app @@ -146,8 +143,7 @@ def parse_options(self, prog_name, argv): return options, values, parser.leftovers def execute_from_commandline(self, argv=None): - if argv is None: - argv = sys.argv + argv = sys.argv if argv is None else argv config = [] seen_cargs = 0 for arg in argv: From abc2e4b938f1f1405ce2ce124937148d5aa1b9ef Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 18:41:28 -0800 Subject: [PATCH 0816/1103] multi: Removed Py2.6-ism --- celery/bin/multi.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/celery/bin/multi.py b/celery/bin/multi.py index 03f9e79b3..4938e3ddd 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -143,6 +143,7 @@ * --verbose: Show more output. * --no-color: Don't display colors. """ +CELERY_EXE = 'celery' multi_args_t = namedtuple( 'multi_args_t', ('name', 'argv', 'expander', 'namespace'), @@ -153,12 +154,6 @@ def main(): sys.exit(MultiTool().execute_from_commandline(sys.argv)) -CELERY_EXE = 'celery' -if sys.version_info < (2, 7): - # pkg.__main__ first supported in Py2.7 - CELERY_EXE = 'celery.__main__' - - def celery_exe(*args): return ' '.join((CELERY_EXE,) + args) From bbf1373464f517f5c3c7d3e15e8933b33ad36be1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 18:41:47 -0800 Subject: [PATCH 0817/1103] multi: Refactored long function --- celery/bin/multi.py | 60 +++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/celery/bin/multi.py b/celery/bin/multi.py index 4938e3ddd..1191ffd94 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -455,34 +455,50 @@ def DOWN(self): return str(self.colored.magenta('DOWN')) +def _args_for_node(p, name, prefix, suffix, cmd, append, options): + name, nodename, expand = _get_nodename( + name, prefix, suffix, options) + + argv = ([expand(cmd)] + + [format_opt(opt, expand(value)) + for opt, value in items(p.optmerge(name, options))] + + [p.passthrough]) + if append: + argv.append(expand(append)) + return multi_args_t(nodename, argv, expand, name) + + def multi_args(p, cmd='celery worker', append='', prefix='', suffix=''): names = p.values options = dict(p.options) - passthrough = p.passthrough ranges = len(names) == 1 if ranges: try: - noderange = int(names[0]) + names, prefix = _get_ranges(names) except ValueError: pass - else: - names = [str(n) for n in range(1, noderange + 1)] - prefix = 'celery' cmd = options.pop('--cmd', cmd) append = options.pop('--append', append) hostname = options.pop('--hostname', options.pop('-n', socket.gethostname())) prefix = options.pop('--prefix', prefix) or '' suffix = options.pop('--suffix', suffix) or hostname - if suffix in ('""', "''"): - suffix = '' + suffix = '' if suffix in ('""', "''") else suffix - for ns_name, ns_opts in list(items(p.namespaces)): - if ',' in ns_name or (ranges and '-' in ns_name): - for subns in parse_ns_range(ns_name, ranges): - p.namespaces[subns].update(ns_opts) - p.namespaces.pop(ns_name) + _update_ns_opts(p, names) + _update_ns_ranges(p, ranges) + return (_args_for_node(p, name, prefix, suffix, cmd, append, options) + for name in names) + + +def _get_ranges(names): + noderange = int(names[0]) + names = [str(n) for n in range(1, noderange + 1)] + prefix = 'celery' + return names, prefix + +def _update_ns_opts(p, names): # Numbers in args always refers to the index in the list of names. # (e.g. `start foo bar baz -c:1` where 1 is foo, 2 is bar, and so on). for ns_name, ns_opts in list(items(p.namespaces)): @@ -495,7 +511,16 @@ def multi_args(p, cmd='celery worker', append='', prefix='', suffix=''): except IndexError: raise KeyError('No node at index %r' % (ns_name,)) - for name in names: + +def _update_ns_ranges(p, ranges): + for ns_name, ns_opts in list(items(p.namespaces)): + if ',' in ns_name or (ranges and '-' in ns_name): + for subns in parse_ns_range(ns_name, ranges): + p.namespaces[subns].update(ns_opts) + p.namespaces.pop(ns_name) + + +def _get_nodename(name, prefix, suffix, options): hostname = suffix if '@' in name: nodename = options['-n'] = host_format(name) @@ -506,18 +531,11 @@ def multi_args(p, cmd='celery worker', append='', prefix='', suffix=''): nodename = options['-n'] = host_format( '{0}@{1}'.format(shortname, hostname), ) - expand = partial( node_format, nodename=nodename, N=shortname, d=hostname, h=nodename, i='%i', I='%I', ) - argv = ([expand(cmd)] + - [format_opt(opt, expand(value)) - for opt, value in items(p.optmerge(name, options))] + - [passthrough]) - if append: - argv.append(expand(append)) - yield multi_args_t(nodename, argv, expand, name) + return name, nodename, expand class NamespacedOptionParser(object): From 670a093c3cd7dcbfcfd420cb136769741866e886 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 18:42:38 -0800 Subject: [PATCH 0818/1103] events.state: Added default attribute values for parent_id, root_id --- celery/events/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/events/state.py b/celery/events/state.py index c0fcef094..19bdfc907 100644 --- a/celery/events/state.py +++ b/celery/events/state.py @@ -208,7 +208,7 @@ class Task(object): name = received = sent = started = succeeded = failed = retried = \ revoked = args = kwargs = eta = expires = retries = worker = result = \ exception = timestamp = runtime = traceback = exchange = \ - routing_key = client = None + routing_key = root_id = parent_id = client = None state = states.PENDING clock = 0 From 50bdc6d9a261ba5f13dc248d17e4093c05d34ad8 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 18:43:10 -0800 Subject: [PATCH 0819/1103] events.state: Removed Task._defaults as scheduled for removal in 4.0 --- celery/events/state.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/celery/events/state.py b/celery/events/state.py index 19bdfc907..91e1f5d97 100644 --- a/celery/events/state.py +++ b/celery/events/state.py @@ -33,8 +33,7 @@ from kombu.utils import cached_property from celery import states -from celery.five import class_property, items, values -from celery.utils import deprecated +from celery.five import items, values from celery.utils.functional import LRUCache, memoize from celery.utils.log import get_logger @@ -324,12 +323,6 @@ def origin(self): def ready(self): return self.state in states.READY_STATES - @class_property - def _defaults(cls): - """Deprecated, to be removed in 5.0.""" - source = cls() - return {k: getattr(source, k) for k in source._fields} - class State(object): """Records clusters state.""" From 3074472c707d7c5fdf6966612b89e704b2f64fd0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 18:44:01 -0800 Subject: [PATCH 0820/1103] cmdline config: namespace can be None --- celery/loaders/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/loaders/base.py b/celery/loaders/base.py index 39699689b..02ec1624a 100644 --- a/celery/loaders/base.py +++ b/celery/loaders/base.py @@ -185,7 +185,7 @@ def cmdline_config_parser( 'list': 'json', 'dict': 'json'}): from celery.app.defaults import Option, NAMESPACES - namespace = namespace.lower() + namespace = namespace and namespace.lower() typemap = dict(Option.typemap, **extra_types) def getarg(arg): From 5f6d921ab1827ed787c47d3415acb8d600a545f2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 18:44:57 -0800 Subject: [PATCH 0821/1103] Result: Raised Exception should be ImproperlyConfigured --- celery/result.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/celery/result.py b/celery/result.py index 3754e92e8..5d312949a 100644 --- a/celery/result.py +++ b/celery/result.py @@ -22,7 +22,7 @@ from ._state import _set_task_join_will_block, task_join_will_block from .app import app_or_default from .datastructures import DependencyGraph, GraphFormatter -from .exceptions import IncompleteStream, TimeoutError +from .exceptions import ImproperlyConfigured, IncompleteStream, TimeoutError from .five import items, range, string_t, monotonic from .utils import deprecated @@ -633,7 +633,8 @@ def join(self, timeout=None, propagate=True, interval=0.5, remaining = None if on_message is not None: - raise Exception('Your backend not supported on_message callback') + raise ImproperlyConfigured( + 'Backend does not support on_message callback') results = [] for result in self.results: From 38122907f22ec3df7e275a3d33e134a271d8082c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 18:45:24 -0800 Subject: [PATCH 0822/1103] Removes deprecated ResultSet.subtasks (use ResultSet.results) --- celery/result.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/celery/result.py b/celery/result.py index 5d312949a..be62dee98 100644 --- a/celery/result.py +++ b/celery/result.py @@ -726,11 +726,6 @@ def __repr__(self): return '<{0}: [{1}]>'.format(type(self).__name__, ', '.join(r.id for r in self.results)) - @property - def subtasks(self): - """Deprecated alias to :attr:`results`.""" - return self.results - @property def supports_native_join(self): try: From a28d300463ecd14d7981ecc12e437f9f5ffd7834 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 20 Nov 2015 18:48:29 -0800 Subject: [PATCH 0823/1103] 99% coverage (excluding celery.concurrency.asynpool and experimental backends) --- .coveragerc | 3 + celery/apps/worker.py | 11 +- celery/backends/base.py | 4 +- celery/backends/redis.py | 7 +- celery/bin/base.py | 2 +- celery/bin/celery.py | 4 +- celery/canvas.py | 5 +- celery/events/state.py | 4 +- celery/fixups/django.py | 40 +++-- celery/local.py | 9 +- celery/platforms.py | 12 +- celery/result.py | 1 - celery/tests/app/test_app.py | 37 ++++ celery/tests/app/test_beat.py | 12 ++ celery/tests/app/test_utils.py | 28 ++- celery/tests/backends/test_amqp.py | 19 +- celery/tests/backends/test_base.py | 133 +++++++++++++- celery/tests/backends/test_mongodb.py | 37 +++- celery/tests/backends/test_redis.py | 219 ++++++++++++++++++------ celery/tests/bin/celery.py | 2 + celery/tests/bin/test_base.py | 58 +++++++ celery/tests/bin/test_celery.py | 47 +++++ celery/tests/bin/test_celeryd_detach.py | 25 ++- celery/tests/bin/test_multi.py | 5 + celery/tests/bin/test_worker.py | 12 +- celery/tests/case.py | 63 +++++-- celery/tests/contrib/test_rdb.py | 5 + celery/tests/events/test_events.py | 70 +++++++- celery/tests/events/test_state.py | 89 +++++++++- celery/tests/fixups/test_django.py | 88 ++++++++++ celery/tests/tasks/test_canvas.py | 146 +++++++++++++++- celery/tests/tasks/test_result.py | 138 +++++++++++++-- celery/tests/tasks/test_trace.py | 34 ++++ celery/tests/utils/test_functional.py | 107 +++++++++++- celery/tests/utils/test_imports.py | 1 + celery/tests/utils/test_local.py | 6 + celery/tests/utils/test_platforms.py | 118 ++++++++++++- celery/tests/utils/test_saferepr.py | 9 + celery/tests/utils/test_timer2.py | 18 +- celery/tests/utils/test_timeutils.py | 4 +- celery/tests/worker/test_autoreload.py | 61 +++++-- celery/tests/worker/test_bootsteps.py | 25 +++ celery/tests/worker/test_components.py | 49 +++++- celery/tests/worker/test_control.py | 4 + celery/tests/worker/test_worker.py | 55 +++++- celery/utils/log.py | 16 +- celery/utils/saferepr.py | 6 +- celery/utils/timeutils.py | 4 +- celery/worker/components.py | 7 +- 49 files changed, 1668 insertions(+), 191 deletions(-) create mode 100644 celery/tests/bin/celery.py diff --git a/.coveragerc b/.coveragerc index 39ff403db..39b043f9c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -16,3 +16,6 @@ omit = *celery/backends/couchdb.py *celery/backends/couchbase.py *celery/backends/cassandra.py + *celery/backends/riak.py + *celery/concurrency/asynpool.py + *celery/utils/debug.py diff --git a/celery/apps/worker.py b/celery/apps/worker.py index 0cdf0fdb8..af1ec025d 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -147,18 +147,19 @@ def on_init_blueprint(self): trace.setup_worker_optimizations(self.app, self.hostname) def on_start(self): + app = self.app if not self._custom_logging and self.redirect_stdouts: - self.app.log.redirect_stdouts(self.redirect_stdouts_level) + app.log.redirect_stdouts(self.redirect_stdouts_level) WorkController.on_start(self) # this signal can be used to e.g. change queues after # the -Q option has been applied. signals.celeryd_after_setup.send( - sender=self.hostname, instance=self, conf=self.app.conf, + sender=self.hostname, instance=self, conf=app.conf, ) - if not self.app.conf.value_set_for('accept_content'): + if not app.conf.value_set_for('accept_content'): # pragma: no cover warnings.warn(CDeprecationWarning(W_PICKLE_DEPRECATED)) if self.purge: @@ -187,7 +188,7 @@ def setup_logging(self, colorize=None): def purge_messages(self): count = self.app.control.purge() - if count: + if count: # pragma: no cover print('purge: Erased {0} {1} from the queue.\n'.format( count, pluralize(count, 'message'))) @@ -209,7 +210,7 @@ def startup_info(self): appr = '{0}:{1:#x}'.format(app.main or '__main__', id(app)) if not isinstance(app.loader, AppLoader): loader = qualname(app.loader) - if loader.startswith('celery.loaders'): + if loader.startswith('celery.loaders'): # pragma: no cover loader = loader[14:] appr += ' ({0})'.format(loader) if self.autoscale: diff --git a/celery/backends/base.py b/celery/backends/base.py index ba7f014c5..2a2cb613c 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -394,7 +394,7 @@ class KeyValueStoreBackend(BaseBackend): implements_incr = False def __init__(self, *args, **kwargs): - if hasattr(self.key_t, '__func__'): + if hasattr(self.key_t, '__func__'): # pragma: no cover self.key_t = self.key_t.__func__ # remove binding self._encode_prefixes() super(KeyValueStoreBackend, self).__init__(*args, **kwargs) @@ -583,7 +583,7 @@ def on_chord_part_return(self, request, state, result, **kwargs): ) val = self.incr(key) size = len(deps) - if val > size: + if val > size: # pragma: no cover logger.warning('Chord counter incremented too many times for %r', gid) elif val == size: diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 06554b83a..ae8f7fd82 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -39,6 +39,10 @@ You need to install the redis library in order to use \ the Redis result store backend.""" +E_LOST = """\ +Connection to Redis lost: Retry (%s/%s) %s.\ +""" + logger = get_logger(__name__) error = logger.error @@ -137,8 +141,7 @@ def ensure(self, fun, args, **policy): def on_connection_error(self, max_retries, exc, intervals, retries): tts = next(intervals) - error('Connection to Redis lost: Retry (%s/%s) %s.', - retries, max_retries or 'Inf', + error(E_LOST, retries, max_retries or 'Inf', humanize_seconds(tts, 'in ')) return tts diff --git a/celery/bin/base.py b/celery/bin/base.py index d39dee309..9ce89286a 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -95,7 +95,7 @@ try: input = raw_input -except NameError: +except NameError: # pragma: no cover pass # always enable DeprecationWarnings, so our users can see them. diff --git a/celery/bin/celery.py b/celery/bin/celery.py index 3df1966c6..4e08bbfde 100644 --- a/celery/bin/celery.py +++ b/celery/bin/celery.py @@ -740,13 +740,13 @@ def _relocate_args_from_start(self, argv, index=0): # is (maybe) a value for this option rest.extend([value, nxt]) index += 1 - except IndexError: + except IndexError: # pragma: no cover rest.append(value) break else: break index += 1 - if argv[index:]: + if argv[index:]: # pragma: no cover # if there are more arguments left then divide and swap # we assume the first argument in argv[i:] is the command # name. diff --git a/celery/canvas.py b/celery/canvas.py index f3c99ca11..bc45c65b2 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -261,7 +261,8 @@ def set_parent_id(self, parent_id): def apply_async(self, args=(), kwargs={}, route_name=None, **options): try: _apply = self._apply_async - except IndexError: # no tasks for chain, etc to find type + except IndexError: # pragma: no cover + # no tasks for chain, etc to find type return # For callbacks: extra args are prepended to the stored args. if args or kwargs or options: @@ -337,7 +338,7 @@ def election(self): def __repr__(self): return self.reprcall() - if JSON_NEEDS_UNICODE_KEYS: + if JSON_NEEDS_UNICODE_KEYS: # pragma: no cover def items(self): for k, v in dict.items(self): yield k.decode() if isinstance(k, bytes) else k, v diff --git a/celery/events/state.py b/celery/events/state.py index 91e1f5d97..cfb12ecb9 100644 --- a/celery/events/state.py +++ b/celery/events/state.py @@ -166,7 +166,7 @@ def event(type_, timestamp=None, if drift > max_drift: _warn_drift(self.hostname, drift, local_received, timestamp) - if local_received: + if local_received: # pragma: no cover hearts = len(heartbeats) if hearts > hbmax - 1: hb_pop(0) @@ -218,7 +218,7 @@ class Task(object): 'timestamp', 'runtime', 'traceback', 'exchange', 'routing_key', 'clock', 'client', 'root_id', 'parent_id', ) - if not PYPY: + if not PYPY: # pragma: no cover __slots__ = ('__dict__', '__weakref__') #: How to merge out of order events. diff --git a/celery/fixups/django.py b/celery/fixups/django.py index 379ce34b9..e7578004a 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -15,7 +15,7 @@ if sys.version_info[0] < 3 and not hasattr(sys, 'pypy_version_info'): from StringIO import StringIO -else: +else: # pragma: no cover from io import StringIO @@ -66,12 +66,16 @@ def install(self): signals.worker_init.connect(self.on_worker_init) return self - @cached_property + @property def worker_fixup(self): if self._worker_fixup is None: self._worker_fixup = DjangoWorkerFixup(self.app) return self._worker_fixup + @worker_fixup.setter + def worker_fixup(self, value): + self._worker_fixup = value + def on_import_modules(self, **kwargs): # call django.setup() before task modules are imported self.worker_fixup.validate_models() @@ -160,36 +164,40 @@ def __init__(self, app): _oracle_database_errors ) - def validate_models(self): + def django_setup(self): import django try: django_setup = django.setup - except AttributeError: + except AttributeError: # pragma: no cover pass else: django_setup() - s = StringIO() + + def validate_models(self): + self.django_setup() try: from django.core.management.validation import get_validation_errors except ImportError: - from django.core.management.base import BaseCommand - cmd = BaseCommand() - try: - # since django 1.5 - from django.core.management.base import OutputWrapper - cmd.stdout = OutputWrapper(sys.stdout) - cmd.stderr = OutputWrapper(sys.stderr) - except ImportError: - cmd.stdout, cmd.stderr = sys.stdout, sys.stderr - - cmd.check() + self._validate_models_django17() else: + s = StringIO() num_errors = get_validation_errors(s, None) if num_errors: raise RuntimeError( 'One or more Django models did not validate:\n{0}'.format( s.getvalue())) + def _validate_models_django17(self): + from django.core.management import base + print(base) + cmd = base.BaseCommand() + try: + cmd.stdout = base.OutputWrapper(sys.stdout) + cmd.stderr = base.OutputWrapper(sys.stderr) + except ImportError: # before django 1.5 + cmd.stdout, cmd.stderr = sys.stdout, sys.stderr + cmd.check() + def install(self): signals.beat_embedded_init.connect(self.close_database) signals.worker_ready.connect(self.on_worker_ready) diff --git a/celery/local.py b/celery/local.py index 2e4b12bd6..032e81b30 100644 --- a/celery/local.py +++ b/celery/local.py @@ -99,9 +99,10 @@ def _get_current_object(self): loc = object.__getattribute__(self, '_Proxy__local') if not hasattr(loc, '__release_local__'): return loc(*self.__args, **self.__kwargs) - try: + try: # pragma: no cover + # not sure what this is about return getattr(loc, self.__name__) - except AttributeError: + except AttributeError: # pragma: no cover raise RuntimeError('no object bound to {0.__name__}'.format(self)) @property @@ -286,7 +287,7 @@ def __exit__(self, *a, **kw): def __reduce__(self): return self._get_current_object().__reduce__() - if not PY3: + if not PY3: # pragma: no cover def __cmp__(self, other): return cmp(self._get_current_object(), other) # noqa @@ -361,7 +362,7 @@ def __evaluate__(self, finally: try: object.__delattr__(self, '__pending__') - except AttributeError: + except AttributeError: # pragma: no cover pass return thing diff --git a/celery/platforms.py b/celery/platforms.py index 75d71db85..fd4410df3 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -21,10 +21,6 @@ from collections import namedtuple -try: - from billiard.process import current_process -except ImportError: - current_process = None from billiard.compat import get_fdmax, close_open_fds # fileno used to be in this module from kombu.utils import maybe_fileno @@ -34,6 +30,11 @@ from .local import try_import from .five import items, reraise, string_t +try: + from billiard.process import current_process +except ImportError: # pragma: no cover + current_process = None + _setproctitle = try_import('setproctitle') resource = try_import('resource') pwd = try_import('pwd') @@ -340,7 +341,8 @@ def close(self, *args): def _detach(self): if os.fork() == 0: # first child os.setsid() # create new session - if os.fork() > 0: # second child + if os.fork() > 0: # pragma: no cover + # second child os._exit(0) else: os._exit(0) diff --git a/celery/result.py b/celery/result.py index be62dee98..42ff01f64 100644 --- a/celery/result.py +++ b/celery/result.py @@ -9,7 +9,6 @@ from __future__ import absolute_import import time -import warnings from collections import OrderedDict, deque from contextlib import contextmanager diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index ad5c5fbcd..de7324ded 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -303,6 +303,43 @@ def test_pending_configuration__compat_settings(self): self.assertEqual(app.conf.broker_url, 'foo://bar') self.assertEqual(app.conf.result_backend, 'foo') + def test_pending_configuration__compat_settings_mixing(self): + with self.Celery(broker='foo://bar', backend='foo') as app: + app.conf.update( + CELERY_ALWAYS_EAGER=4, + CELERY_DEFAULT_DELIVERY_MODE=63, + CELERYD_AGENT='foo:Barz', + worker_consumer='foo:Fooz', + ) + with self.assertRaises(ImproperlyConfigured): + self.assertEqual(app.conf.task_always_eager, 4) + + def test_pending_configuration__compat_settings_mixing_new(self): + with self.Celery(broker='foo://bar', backend='foo') as app: + app.conf.update( + task_always_eager=4, + task_default_delivery_mode=63, + worker_agent='foo:Barz', + CELERYD_CONSUMER='foo:Fooz', + CELERYD_AUTOSCALER='foo:Xuzzy', + ) + with self.assertRaises(ImproperlyConfigured): + self.assertEqual(app.conf.worker_consumer, 'foo:Fooz') + + def test_pending_configuration__compat_settings_mixing_alt(self): + with self.Celery(broker='foo://bar', backend='foo') as app: + app.conf.update( + task_always_eager=4, + task_default_delivery_mode=63, + worker_agent='foo:Barz', + CELERYD_CONSUMER='foo:Fooz', + worker_consumer='foo:Fooz', + CELERYD_AUTOSCALER='foo:Xuzzy', + worker_autoscaler='foo:Xuzzy' + ) + self.assertEqual(app.conf.task_always_eager, 4) + self.assertEqual(app.conf.worker_autoscaler, 'foo:Xuzzy') + def test_pending_configuration__setdefault(self): with self.Celery(broker='foo://bar') as app: app.conf.setdefault('worker_agent', 'foo:Bar') diff --git a/celery/tests/app/test_beat.py b/celery/tests/app/test_beat.py index da4638c8a..1eab5bcb7 100644 --- a/celery/tests/app/test_beat.py +++ b/celery/tests/app/test_beat.py @@ -83,6 +83,18 @@ def test_repr(self): entry = self.create_entry() self.assertIn('= n: + mock.side_effect = side_effect + return mock.return_value + mock.side_effect = on_call + return mock + + def on_nth_call_return(self, mock, retval, n=1): + + def on_call(*args, **kwargs): + if mock.call_count >= n: + mock.return_value = retval + return mock.return_value + mock.side_effect = on_call + return mock + + def mask_modules(self, *modules): + self.wrap_context(mask_modules(*modules)) + + def wrap_context(self, context): + ret = context.__enter__() + self.addCleanup(partial(context.__exit__, None, None, None)) + return ret + + def mock_environ(self, env_name, env_value): + return self.wrap_context(mock_environ(env_name, env_value)) def assertWarns(self, expected_warning): return _AssertWarnsContext(expected_warning, self, None) @@ -543,19 +577,28 @@ def wrap_logger(logger, loglevel=logging.ERROR): logger.handlers = old_handlers +@contextmanager +def mock_environ(env_name, env_value): + sentinel = object() + prev_val = os.environ.get(env_name, sentinel) + os.environ[env_name] = env_value + try: + yield env_value + finally: + if prev_val is sentinel: + os.environ.pop(env_name, None) + else: + os.environ[env_name] = prev_val + + def with_environ(env_name, env_value): def _envpatched(fun): @wraps(fun) def _patch_environ(*args, **kwargs): - prev_val = os.environ.get(env_name) - os.environ[env_name] = env_value - try: + with mock_environ(env_name, env_value): return fun(*args, **kwargs) - finally: - os.environ[env_name] = prev_val or '' - return _patch_environ return _envpatched diff --git a/celery/tests/contrib/test_rdb.py b/celery/tests/contrib/test_rdb.py index 26b3a5498..23e5699dd 100644 --- a/celery/tests/contrib/test_rdb.py +++ b/celery/tests/contrib/test_rdb.py @@ -58,6 +58,11 @@ def test_rdb(self, get_avail_port): # _close_session rdb._close_session() + rdb.active = True + rdb._handle = None + rdb._client = None + rdb._sock = None + rdb._close_session() # do_continue rdb.set_continue = Mock() diff --git a/celery/tests/events/test_events.py b/celery/tests/events/test_events.py index 1e16f93ef..41899c1ce 100644 --- a/celery/tests/events/test_events.py +++ b/celery/tests/events/test_events.py @@ -2,11 +2,13 @@ import socket -from celery.events import Event -from celery.tests.case import AppCase, Mock +from celery.events import CLIENT_CLOCK_SKEW, Event + +from celery.tests.case import AppCase, Mock, call class MockProducer(object): + raise_on_publish = False def __init__(self, *args, **kwargs): @@ -93,6 +95,44 @@ def test_send(self): eventer.flush() + def test_send_buffer_group(self): + buf_received = [None] + producer = MockProducer() + producer.connection = self.app.connection() + connection = Mock() + connection.transport.driver_type = 'amqp' + eventer = self.app.events.Dispatcher( + connection, enabled=False, + buffer_group={'task'}, buffer_limit=2, + ) + eventer.producer = producer + eventer.enabled = True + eventer._publish = Mock(name='_publish') + + def on_eventer_publish(events, *args, **kwargs): + buf_received[0] = list(events) + eventer._publish.side_effect = on_eventer_publish + self.assertFalse(eventer._group_buffer['task']) + eventer.on_send_buffered = Mock(name='on_send_buffered') + eventer.send('task-received', uuid=1) + prev_buffer = eventer._group_buffer['task'] + self.assertTrue(eventer._group_buffer['task']) + eventer.on_send_buffered.assert_called_with() + eventer.send('task-received', uuid=1) + self.assertFalse(eventer._group_buffer['task']) + eventer._publish.assert_has_calls( + call([], eventer.producer, 'task.multi'), + ) + # clear in place + self.assertIs(eventer._group_buffer['task'], prev_buffer) + self.assertEqual(len(buf_received[0]), 2) + eventer.on_send_buffered = None + eventer.send('task-received', uuid=1) + + def test_flush_no_groups_no_errors(self): + eventer = self.app.events.Dispatcher(Mock()) + eventer.flush(errors=False, groups=False) + def test_enter_exit(self): with self.app.connection() as conn: d = self.app.events.Dispatcher(conn) @@ -174,6 +214,10 @@ def my_handler(event): r._receive(message, object()) self.assertTrue(got_event[0]) + def test_accept_argument(self): + r = self.app.events.Receiver(Mock(), accept={'app/foo'}) + self.assertEqual(r.accept, {'app/foo'}) + def test_catch_all_event(self): message = {'type': 'world-war'} @@ -217,6 +261,28 @@ def test_event_from_message_localize_disabled(self): self.assertFalse(ts_adjust.called) r.adjust_clock.assert_called_with(313) + def test_event_from_message_clock_from_client(self): + r = self.app.events.Receiver(Mock(), node_id='celery.tests') + r.clock.value = 302 + r.adjust_clock = Mock() + + body = {'type': 'task-sent'} + r.event_from_message( + body, localize=False, adjust_timestamp=Mock(), + ) + self.assertEqual(body['clock'], r.clock.value + CLIENT_CLOCK_SKEW) + + def test_receive_multi(self): + r = self.app.events.Receiver(Mock(name='connection')) + r.process = Mock(name='process') + efm = r.event_from_message = Mock(name='event_from_message') + + def on_efm(*args): + return args + efm.side_effect = on_efm + r._receive([1, 2, 3], Mock()) + r.process.assert_has_calls([call(1), call(2), call(3)]) + def test_itercapture_limit(self): connection = self.app.connection() channel = connection.channel() diff --git a/celery/tests/events/test_state.py b/celery/tests/events/test_state.py index f51dfe74e..841a8a989 100644 --- a/celery/tests/events/test_state.py +++ b/celery/tests/events/test_state.py @@ -10,11 +10,12 @@ from celery import states from celery.events import Event from celery.events.state import ( + HEARTBEAT_EXPIRE_WINDOW, + HEARTBEAT_DRIFT_MAX, State, Worker, Task, - HEARTBEAT_EXPIRE_WINDOW, - HEARTBEAT_DRIFT_MAX, + heartbeat_expires, ) from celery.five import range from celery.utils import uuid @@ -104,6 +105,7 @@ def setup(self): traceback='line 1 at main', hostname='utest1'), Event('task-succeeded', uuid=tid, result='4', runtime=0.1234, hostname='utest1'), + Event('foo-bar'), ] @@ -181,6 +183,12 @@ def test_equality(self): hash(Worker(hostname='foo')), hash(Worker(hostname='bar')), ) + def test_heartbeat_expires__Decimal(self): + self.assertEqual( + heartbeat_expires(Decimal(344313.37), freq=60, expire_window=200), + 344433.37, + ) + def test_compatible_with_Decimal(self): w = Worker('george@vandelay.com') timestamp, local_received = Decimal(_float_to_decimal(time())), time() @@ -192,6 +200,39 @@ def test_compatible_with_Decimal(self): }) self.assertTrue(w.alive) + def test_eq_ne_other(self): + self.assertEqual(Worker('a@b.com'), Worker('a@b.com')) + self.assertNotEqual(Worker('a@b.com'), Worker('b@b.com')) + self.assertNotEqual(Worker('a@b.com'), object()) + + def test_reduce_direct(self): + w = Worker('george@vandelay.com') + w.event('worker-online', 10.0, 13.0, fields={ + 'hostname': 'george@vandelay.com', + 'timestamp': 10.0, + 'local_received': 13.0, + 'freq': 60, + }) + fun, args = w.__reduce__() + w2 = fun(*args) + self.assertEqual(w2.hostname, w.hostname) + self.assertEqual(w2.pid, w.pid) + self.assertEqual(w2.freq, w.freq) + self.assertEqual(w2.heartbeats, w.heartbeats) + self.assertEqual(w2.clock, w.clock) + self.assertEqual(w2.active, w.active) + self.assertEqual(w2.processed, w.processed) + self.assertEqual(w2.loadavg, w.loadavg) + self.assertEqual(w2.sw_ident, w.sw_ident) + + def test_update(self): + w = Worker('george@vandelay.com') + w.update({'idx': '301'}, foo=1, clock=30, bah='foo') + self.assertEqual(w.idx, '301') + self.assertEqual(w.foo, 1) + self.assertEqual(w.clock, 30) + self.assertEqual(w.bah, 'foo') + def test_survives_missing_timestamp(self): worker = Worker(hostname='foo') worker.event('heartbeat') @@ -263,6 +304,12 @@ def test_info(self): sorted(task.info(['args', 'kwargs']).keys())) self.assertFalse(list(task.info('foo'))) + def test_reduce_direct(self): + task = Task(uuid='uuid', name='tasks.add', args='(2, 2)') + fun, args = task.__reduce__() + task2 = fun(*args) + self.assertEqual(task, task2) + def test_ready(self): task = Task(uuid='abcdefg', name='tasks.add') @@ -341,6 +388,39 @@ def test_task_descending_clock_ordering(self): self.assertEqual(now[1][0], tC) self.assertEqual(now[2][0], tA) + def test_get_or_create_task(self): + state = State() + task, created = state.get_or_create_task('id1') + self.assertEqual(task.uuid, 'id1') + self.assertTrue(created) + task2, created2 = state.get_or_create_task('id1') + self.assertIs(task2, task) + self.assertFalse(created2) + + def test_get_or_create_worker(self): + state = State() + worker, created = state.get_or_create_worker('george@vandelay.com') + self.assertEqual(worker.hostname, 'george@vandelay.com') + self.assertTrue(created) + worker2, created2 = state.get_or_create_worker('george@vandelay.com') + self.assertIs(worker2, worker) + self.assertFalse(created2) + + def test_get_or_create_worker__with_defaults(self): + state = State() + worker, created = state.get_or_create_worker( + 'george@vandelay.com', pid=30, + ) + self.assertEqual(worker.hostname, 'george@vandelay.com') + self.assertEqual(worker.pid, 30) + self.assertTrue(created) + worker2, created2 = state.get_or_create_worker( + 'george@vandelay.com', pid=40, + ) + self.assertIs(worker2, worker) + self.assertEqual(worker2.pid, 40) + self.assertFalse(created2) + def test_worker_online_offline(self): r = ev_worker_online_offline(State()) next(r) @@ -478,10 +558,11 @@ def test_task_types(self): r.play() self.assertEqual(sorted(r.state.task_types()), ['task1', 'task2']) - def test_tasks_by_timestamp(self): + def test_tasks_by_time(self): r = ev_snapshot(State()) r.play() - self.assertEqual(len(list(r.state.tasks_by_timestamp())), 20) + self.assertEqual(len(list(r.state.tasks_by_time())), 20) + self.assertEqual(len(list(r.state.tasks_by_time(reverse=False))), 20) def test_tasks_by_type(self): r = ev_snapshot(State()) diff --git a/celery/tests/fixups/test_django.py b/celery/tests/fixups/test_django.py index 0249a5c95..423292f7f 100644 --- a/celery/tests/fixups/test_django.py +++ b/celery/tests/fixups/test_django.py @@ -31,6 +31,45 @@ def fixup_context(self, app): class test_DjangoFixup(FixupCase): Fixup = DjangoFixup + def test_setting_default_app(self): + from celery.fixups import django + prev, django.default_app = django.default_app, None + try: + app = Mock(name='app') + DjangoFixup(app) + app.set_default.assert_called_with() + finally: + django.default_app = prev + + @patch('celery.fixups.django.DjangoWorkerFixup') + def test_worker_fixup_property(self, DjangoWorkerFixup): + f = DjangoFixup(self.app) + f._worker_fixup = None + self.assertIs(f.worker_fixup, DjangoWorkerFixup()) + self.assertIs(f.worker_fixup, DjangoWorkerFixup()) + + def test_on_import_modules(self): + f = DjangoFixup(self.app) + f.worker_fixup = Mock(name='worker_fixup') + f.on_import_modules() + f.worker_fixup.validate_models.assert_called_with() + + def test_autodiscover_tasks_pre17(self): + self.mask_modules('django.apps') + f = DjangoFixup(self.app) + f._settings = Mock(name='_settings') + self.assertIs(f.autodiscover_tasks(), f._settings.INSTALLED_APPS) + + @patch('django.apps.apps', create=True) + def test_autodiscover_tasks(self, apps): + f = DjangoFixup(self.app) + configs = [Mock(name='c1'), Mock(name='c2')] + apps.get_app_configs.return_value = configs + self.assertEqual( + f.autodiscover_tasks(), + [c.name for c in configs], + ) + def test_fixup(self): with patch('celery.fixups.django.DjangoFixup') as Fixup: with patch.dict(os.environ, DJANGO_SETTINGS_MODULE=''): @@ -149,6 +188,11 @@ def test_on_worker_process_init(self): f._db.connection = None f.on_worker_process_init() + f.validate_models = Mock(name='validate_models') + self.mock_environ('FORKED_BY_MULTIPROCESSING', '1') + f.on_worker_process_init() + f.validate_models.assert_called_with() + def test_on_task_prerun(self): task = Mock() with self.fixup_context(self.app) as (f, _, _): @@ -204,6 +248,13 @@ def test_close_database(self): _close.assert_called_with() self.assertEqual(f._db_recycles, 1) + def test_close_database__django16(self): + with self.fixup_context(self.app) as (f, _, _): + f._db.connections = Mock(name='db.connections') + f._db.connections.all.side_effect = AttributeError() + f._close_database() + f._db.close_old_connections.assert_called_with() + def test__close_database(self): with self.fixup_context(self.app) as (f, _, _): conns = [Mock(), Mock(), Mock()] @@ -245,6 +296,43 @@ def test_on_worker_ready(self): f._settings.DEBUG = True f.on_worker_ready() + def test_validate_models(self): + self.patch('celery.fixups.django.symbol_by_name') + self.patch('celery.fixups.django.import_module') + f = self.Fixup(self.app) + self.mock_modules('django.core.management.validation') + f.django_setup = Mock(name='django.setup') + from django.core.management.validation import get_validation_errors + get_validation_errors.return_value = 0 + f.validate_models() + f.django_setup.assert_called_with() + get_validation_errors.return_value = 3 + with self.assertRaises(RuntimeError): + f.validate_models() + + self.mask_modules('django.core.management.validation') + f._validate_models_django17 = Mock('validate17') + f.validate_models() + f._validate_models_django17.assert_called_with() + + def test_validate_models_django17(self): + self.patch('celery.fixups.django.symbol_by_name') + self.patch('celery.fixups.django.import_module') + self.mock_modules('django.core.management.base') + from django.core.management import base + f = self.Fixup(self.app) + f._validate_models_django17() + base.BaseCommand.assert_called_with() + base.BaseCommand().check.assert_called_with() + + def test_django_setup(self): + self.patch('celery.fixups.django.symbol_by_name') + self.patch('celery.fixups.django.import_module') + django, = self.mock_modules('django') + f = self.Fixup(self.app) + f.django_setup() + django.setup.assert_called_with() + def test_mysql_errors(self): with patch_modules('MySQLdb'): import MySQLdb as mod diff --git a/celery/tests/tasks/test_canvas.py b/celery/tests/tasks/test_canvas.py index 6855aad82..e8ba66e22 100644 --- a/celery/tests/tasks/test_canvas.py +++ b/celery/tests/tasks/test_canvas.py @@ -12,10 +12,13 @@ chunks, _maybe_group, maybe_signature, + maybe_unroll_group, ) from celery.result import EagerResult -from celery.tests.case import AppCase, ContextMock, Mock +from celery.tests.case import ( + AppCase, ContextMock, MagicMock, Mock, depends_on_current_app, +) SIG = Signature({'task': 'TASK', 'args': ('A1',), @@ -24,6 +27,18 @@ 'subtask_type': ''}) +class test_maybe_unroll_group(AppCase): + + def test_when_no_len_and_no_length_hint(self): + g = MagicMock(name='group') + g.tasks.__len__.side_effect = TypeError() + g.tasks.__length_hint__ = Mock() + g.tasks.__length_hint__.return_value = 0 + self.assertIs(maybe_unroll_group(g), g) + g.tasks.__length_hint__.side_effect = AttributeError() + self.assertIs(maybe_unroll_group(g), g) + + class CanvasCase(AppCase): def setup(self): @@ -60,6 +75,12 @@ def test_getitem_property(self): self.assertEqual(SIG.options, {'task_id': 'TASK_ID'}) self.assertEqual(SIG.subtask_type, '') + def test_call(self): + x = Signature('foo', (1, 2), {'arg1': 33}, app=self.app) + x.type = Mock(name='type') + x(3, 4, arg2=66) + x.type.assert_called_with(3, 4, 1, 2, arg1=33, arg2=66) + def test_link_on_scalar(self): x = Signature('TASK', link=Signature('B')) self.assertTrue(x.options['link']) @@ -68,6 +89,16 @@ def test_link_on_scalar(self): self.assertIn(Signature('B'), x.options['link']) self.assertIn(Signature('C'), x.options['link']) + def test_json(self): + x = Signature('TASK', link=Signature('B', app=self.app), app=self.app) + self.assertDictEqual(x.__json__(), dict(x)) + + @depends_on_current_app + def test_reduce(self): + x = Signature('TASK', (2, 4), app=self.app) + fun, args = x.__reduce__() + self.assertEqual(fun(*args), x) + def test_replace(self): x = Signature('TASK', ('A'), {}) self.assertTupleEqual(x.replace(args=('B',)).args, ('B',)) @@ -255,6 +286,35 @@ def assert_group_to_chord_parent_ids(self, freezefun): self.assertEqual(tasks[-4].parent_id, tasks[-3].id) self.assertEqual(tasks[-4].root_id, 'root') + def test_splices_chains(self): + c = chain( + self.add.s(5, 5), + chain(self.add.s(6), self.add.s(7), self.add.s(8), app=self.app), + app=self.app, + ) + c.freeze() + tasks, _ = c._frozen + self.assertEqual(len(tasks), 4) + + def test_from_dict_no_tasks(self): + self.assertTrue(chain.from_dict( + dict(chain(app=self.app)), app=self.app)) + + @depends_on_current_app + def test_app_falls_back_to_default(self): + from celery._state import current_app + self.assertIs(chain().app, current_app) + + def test_handles_dicts(self): + c = chain( + self.add.s(5, 5), dict(self.add.s(8)), app=self.app, + ) + c.freeze() + tasks, _ = c._frozen + for task in tasks: + self.assertIsInstance(task, Signature) + self.assertIs(task.app, self.app) + def test_group_to_chord(self): c = ( self.add.s(5) | @@ -316,7 +376,7 @@ def clone(self, *args, **kwargs): def s(*args, **kwargs): return static(self.add, args, kwargs, type=self.add, app=self.app) - c = s(2, 2) | s(4, 4) | s(8, 8) + c = s(2, 2) | s(4) | s(8) r1 = c.apply_async(task_id='some_id') self.assertEqual(r1.id, 'some_id') @@ -423,6 +483,11 @@ def test_reverse(self): self.assertIsInstance(signature(x), group) self.assertIsInstance(signature(dict(x)), group) + def test_group_with_group_argument(self): + g1 = group(self.add.s(2, 2), self.add.s(4, 4), app=self.app) + g2 = group(g1, app=self.app) + self.assertIs(g2.tasks, g1.tasks) + def test_maybe_group_sig(self): self.assertListEqual( _maybe_group(self.add.s(2, 2), self.app), [self.add.s(2, 2)], @@ -437,6 +502,35 @@ def test_apply_async(self): x = group([self.add.s(4, 4), self.add.s(8, 8)]) x.apply_async() + def test_prepare_with_dict(self): + x = group([self.add.s(4, 4), dict(self.add.s(8, 8))], app=self.app) + x.apply_async() + + def test_group_in_group(self): + g1 = group(self.add.s(2, 2), self.add.s(4, 4), app=self.app) + g2 = group(self.add.s(8, 8), g1, self.add.s(16, 16), app=self.app) + g2.apply_async() + + def test_set_immutable(self): + g1 = group(Mock(name='t1'), Mock(name='t2'), app=self.app) + g1.set_immutable(True) + for task in g1.tasks: + task.set_immutable.assert_called_with(True) + + def test_link(self): + g1 = group(Mock(name='t1'), Mock(name='t2'), app=self.app) + sig = Mock(name='sig') + g1.link(sig) + g1.tasks[0].link.assert_called_with(sig.clone().set(immutable=True)) + + def test_link_error(self): + g1 = group(Mock(name='t1'), Mock(name='t2'), app=self.app) + sig = Mock(name='sig') + g1.link_error(sig) + g1.tasks[0].link_error.assert_called_with( + sig.clone().set(immutable=True), + ) + def test_apply_empty(self): x = group(app=self.app) x.apply() @@ -500,6 +594,41 @@ def test_clone_clones_body(self): z = y.clone() self.assertIsNone(z.kwargs.get('body')) + def test_argument_is_group(self): + x = chord(group(self.add.s(2, 2), self.add.s(4, 4), app=self.app)) + self.assertTrue(x.tasks) + + def test_set_parent_id(self): + x = chord(group(self.add.s(2, 2))) + x.tasks = [self.add.s(2, 2)] + x.set_parent_id('pid') + + def test_app_when_app(self): + app = Mock(name='app') + x = chord([self.add.s(4, 4)], app=app) + self.assertIs(x.app, app) + + def test_app_when_app_in_task(self): + t1 = Mock(name='t1') + t2 = Mock(name='t2') + x = chord([t1, self.add.s(4, 4)]) + self.assertIs(x.app, x.tasks[0].app) + t1.app = None + x = chord([t1], body=t2) + self.assertIs(x.app, t2._app) + + @depends_on_current_app + def test_app_fallback_to_current(self): + from celery._state import current_app + t1 = Mock(name='t1') + t1.app = t1._app = None + x = chord([t1], body=t1) + self.assertIs(x.app, current_app) + + def test_set_immutable(self): + x = chord([Mock(name='t1'), Mock(name='t2')], app=self.app) + x.set_immutable(True) + def test_links_to_body(self): x = chord([self.add.s(2, 2), self.add.s(4, 4)], body=self.mul.s(4)) x.link(self.div.s(2)) @@ -519,6 +648,12 @@ def test_repr(self): x.kwargs['body'] = None self.assertIn('without body', repr(x)) + def test_freeze_tasks_is_not_group(self): + x = chord([self.add.s(2, 2)], body=self.add.s(), app=self.app) + x.freeze() + x.tasks = [self.add.s(2, 2)] + x.freeze() + class test_maybe_signature(CanvasCase): @@ -530,6 +665,13 @@ def test_is_dict(self): maybe_signature(dict(self.add.s()), app=self.app), Signature, ) + def test_is_list(self): + sigs = [dict(self.add.s(2, 2)), dict(self.add.s(4, 4))] + sigs = maybe_signature(sigs, app=self.app) + for sig in sigs: + self.assertIsInstance(sig, Signature) + self.assertIs(sig.app, self.app) + def test_when_sig(self): s = self.add.s() self.assertIs(maybe_signature(s, app=self.app), s) diff --git a/celery/tests/tasks/test_result.py b/celery/tests/tasks/test_result.py index b9c9bd45b..433e081b4 100644 --- a/celery/tests/tasks/test_result.py +++ b/celery/tests/tasks/test_result.py @@ -3,18 +3,23 @@ from contextlib import contextmanager from celery import states -from celery.exceptions import IncompleteStream, TimeoutError +from celery.exceptions import ( + ImproperlyConfigured, IncompleteStream, TimeoutError, +) from celery.five import range from celery.result import ( AsyncResult, EagerResult, + ResultSet, result_from_tuple, assert_will_not_block, ) from celery.utils import uuid from celery.utils.serialization import pickle -from celery.tests.case import AppCase, Mock, depends_on_current_app, patch +from celery.tests.case import ( + AppCase, Mock, call, depends_on_current_app, patch, +) def mock_task(name, state, result): @@ -66,12 +71,22 @@ def test_assert_will_not_block(self, task_join_will_block): task_join_will_block.return_value = False assert_will_not_block() + def test_without_id(self): + with self.assertRaises(ValueError): + AsyncResult(None, app=self.app) + def test_compat_properties(self): x = self.app.AsyncResult('1') self.assertEqual(x.task_id, x.id) x.task_id = '2' self.assertEqual(x.id, '2') + @depends_on_current_app + def test_reduce_direct(self): + x = AsyncResult('1', app=self.app) + fun, args = x.__reduce__() + self.assertEqual(fun(*args), x) + def test_children(self): x = self.app.AsyncResult('1') children = [EagerResult(str(i), i, states.SUCCESS) for i in range(3)] @@ -167,6 +182,15 @@ def test_reduce(self): a2 = self.app.AsyncResult('uuid') self.assertEqual(pickle.loads(pickle.dumps(a2)).id, 'uuid') + def test_maybe_set_cache_empty(self): + self.app.AsyncResult('uuid')._maybe_set_cache(None) + + def test_set_cache__children(self): + r1 = self.app.AsyncResult('id1') + r2 = self.app.AsyncResult('id2') + r1._set_cache({'children': [r2.as_tuple()]}) + self.assertIn(r2, r1.children) + def test_successful(self): ok_res = self.app.AsyncResult(self.task1['id']) nok_res = self.app.AsyncResult(self.task3['id']) @@ -224,13 +248,22 @@ def test_get_traceback(self): pending_res = self.app.AsyncResult(uuid()) self.assertFalse(pending_res.traceback) + def test_get__backend_gives_None(self): + res = self.app.AsyncResult(self.task1['id']) + res.backend.wait_for = Mock(name='wait_for') + res.backend.wait_for.return_value = None + self.assertIsNone(res.get()) + def test_get(self): ok_res = self.app.AsyncResult(self.task1['id']) ok2_res = self.app.AsyncResult(self.task2['id']) nok_res = self.app.AsyncResult(self.task3['id']) nok2_res = self.app.AsyncResult(self.task4['id']) - self.assertEqual(ok_res.get(), 'the') + callback = Mock(name='callback') + + self.assertEqual(ok_res.get(callback=callback), 'the') + callback.assert_called_with(ok_res.id, 'the') self.assertEqual(ok2_res.get(), 'quick') with self.assertRaises(KeyError): nok_res.get() @@ -238,6 +271,21 @@ def test_get(self): self.assertIsInstance(nok2_res.result, KeyError) self.assertEqual(ok_res.info, 'the') + def test_eq_ne(self): + r1 = self.app.AsyncResult(self.task1['id']) + r2 = self.app.AsyncResult(self.task1['id']) + r3 = self.app.AsyncResult(self.task2['id']) + self.assertEqual(r1, r2) + self.assertNotEqual(r1, r3) + self.assertEqual(r1, r2.id) + self.assertNotEqual(r1, r3.id) + + @depends_on_current_app + def test_reduce_restore(self): + r1 = self.app.AsyncResult(self.task1['id']) + fun, args = r1.__reduce__() + self.assertEqual(fun(*args), r1) + def test_get_timeout(self): res = self.app.AsyncResult(self.task4['id']) # has RETRY state with self.assertRaises(TimeoutError): @@ -288,6 +336,29 @@ def test_get(self): x.get() self.assertTrue(x.join_native.called) + def test_eq_ne(self): + g1 = self.app.ResultSet( + self.app.AsyncResult('id1'), + self.app.AsyncResult('id2'), + ) + g2 = self.app.ResultSet( + self.app.AsyncResult('id1'), + self.app.AsyncResult('id2'), + ) + g3 = self.app.ResultSet( + self.app.AsyncResult('id3'), + self.app.AsyncResult('id1'), + ) + self.assertEqual(g1, g2) + self.assertNotEqual(g1, g3) + self.assertNotEqual(g1, object()) + + def test_takes_app_from_first_task(self): + x = ResultSet([self.app.AsyncResult('id1')]) + self.assertIs(x.app, x.results[0].app) + x.app = self.app + self.assertIs(x.app, self.app) + def test_get_empty(self): x = self.app.ResultSet([]) self.assertIsNone(x.supports_native_join) @@ -432,6 +503,24 @@ def test_is_pickleable(self): ts2 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())]) self.assertEqual(pickle.loads(pickle.dumps(ts2)), ts2) + @depends_on_current_app + def test_reduce(self): + ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())]) + fun, args = ts.__reduce__() + ts2 = fun(*args) + self.assertEqual(ts2.id, ts.id) + self.assertEqual(ts, ts2) + + def test_eq_ne(self): + ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())]) + ts2 = self.app.GroupResult(ts.id, ts.results) + ts3 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())]) + ts4 = self.app.GroupResult(ts.id, [self.app.AsyncResult(uuid())]) + self.assertEqual(ts, ts2) + self.assertNotEqual(ts, ts3) + self.assertNotEqual(ts, ts4) + self.assertNotEqual(ts, object()) + def test_len(self): self.assertEqual(len(self.ts), self.size) @@ -439,7 +528,7 @@ def test_eq_other(self): self.assertFalse(self.ts == 1) @depends_on_current_app - def test_reduce(self): + def test_pickleable(self): self.assertTrue(pickle.loads(pickle.dumps(self.ts))) def test_iterate_raises(self): @@ -471,8 +560,8 @@ def test_save_restore(self): ts.save() with self.assertRaises(AttributeError): ts.save(backend=object()) - self.assertEqual(self.app.GroupResult.restore(ts.id).subtasks, - ts.subtasks) + self.assertEqual(self.app.GroupResult.restore(ts.id).results, + ts.results) ts.delete() self.assertIsNone(self.app.GroupResult.restore(ts.id)) with self.assertRaises(AttributeError): @@ -480,13 +569,18 @@ def test_save_restore(self): def test_join_native(self): backend = SimpleBackend() - subtasks = [self.app.AsyncResult(uuid(), backend=backend) - for i in range(10)] - ts = self.app.GroupResult(uuid(), subtasks) + results = [self.app.AsyncResult(uuid(), backend=backend) + for i in range(10)] + ts = self.app.GroupResult(uuid(), results) ts.app.backend = backend - backend.ids = [subtask.id for subtask in subtasks] + backend.ids = [result.id for result in results] res = ts.join_native() self.assertEqual(res, list(range(10))) + callback = Mock(name='callback') + self.assertFalse(ts.join_native(callback=callback)) + callback.assert_has_calls([ + call(r.id, i) for i, r in enumerate(ts.results) + ]) def test_join_native_raises(self): ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())]) @@ -518,11 +612,11 @@ def test_children_is_results(self): def test_iter_native(self): backend = SimpleBackend() - subtasks = [self.app.AsyncResult(uuid(), backend=backend) - for i in range(10)] - ts = self.app.GroupResult(uuid(), subtasks) + results = [self.app.AsyncResult(uuid(), backend=backend) + for i in range(10)] + ts = self.app.GroupResult(uuid(), results) ts.app.backend = backend - backend.ids = [subtask.id for subtask in subtasks] + backend.ids = [result.id for result in results] self.assertEqual(len(list(ts.iter_native())), 10) def test_iterate_yields(self): @@ -555,6 +649,9 @@ def test_join_timeout(self): ar4.get = Mock() ts2 = self.app.GroupResult(uuid(), [ar4]) self.assertTrue(ts2.join(timeout=0.1)) + callback = Mock(name='callback') + self.assertFalse(ts2.join(timeout=0.1, callback=callback)) + callback.assert_called_with(ar4.id, ar4.get()) def test_iter_native_when_empty_group(self): ts = self.app.GroupResult(uuid(), []) @@ -579,6 +676,15 @@ def test_successful(self): def test_failed(self): self.assertFalse(self.ts.failed()) + def test_maybe_reraise(self): + self.ts.results = [Mock(name='r1')] + self.ts.maybe_reraise() + self.ts.results[0].maybe_reraise.assert_called_with() + + def test_join__on_message(self): + with self.assertRaises(ImproperlyConfigured): + self.ts.join(on_message=Mock()) + def test_waiting(self): self.assertFalse(self.ts.waiting()) @@ -603,11 +709,11 @@ class test_failed_AsyncResult(test_GroupResult): def setup(self): self.app.conf.result_serializer = 'pickle' self.size = 11 - subtasks = make_mock_group(self.app, 10) + results = make_mock_group(self.app, 10) failed = mock_task('ts11', states.FAILURE, KeyError('Baz')) save_result(self.app, failed) failed_res = self.app.AsyncResult(failed['id']) - self.ts = self.app.GroupResult(uuid(), subtasks + [failed_res]) + self.ts = self.app.GroupResult(uuid(), results + [failed_res]) def test_completed_count(self): self.assertEqual(self.ts.completed_count(), len(self.ts) - 1) diff --git a/celery/tests/tasks/test_trace.py b/celery/tests/tasks/test_trace.py index aaaa6986c..a1b9e1ace 100644 --- a/celery/tests/tasks/test_trace.py +++ b/celery/tests/tasks/test_trace.py @@ -16,6 +16,8 @@ log_policy_expected, log_policy_unexpected, trace_task, + _trace_task_ret, + _fast_trace_task, setup_worker_optimizations, reset_worker_optimizations, ) @@ -178,6 +180,11 @@ def rejecting(): retval, info = self.trace(rejecting, (), {}) self.assertEqual(info.state, states.REJECTED) + def test_backend_cleanup_raises(self): + self.add.backend.process_cleanup = Mock() + self.add.backend.process_cleanup.side_effect = RuntimeError() + self.trace(self.add, (2, 2), {}) + @patch('celery.canvas.maybe_signature') def test_callbacks__scalar(self, maybe_signature): sig = Mock(name='sig') @@ -188,6 +195,18 @@ def test_callbacks__scalar(self, maybe_signature): (4,), parent_id='id-1', root_id='root', ) + @patch('celery.canvas.maybe_signature') + def test_chain_proto2(self, maybe_signature): + sig = Mock(name='sig') + sig2 = Mock(name='sig2') + request = {'chain': [sig2, sig], 'root_id': 'root'} + maybe_signature.return_value = sig + retval, _ = self.trace(self.add, (2, 2), {}, request=request) + sig.apply_async.assert_called_with( + (4, ), parent_id='id-1', root_id='root', + chain=[sig2], + ) + @patch('celery.canvas.maybe_signature') def test_callbacks__EncodeError(self, maybe_signature): sig = Mock(name='sig') @@ -253,6 +272,21 @@ def test_trace_exception(self): self.assertEqual(info.state, states.FAILURE) self.assertIs(info.retval, exc) + def test_trace_task_ret__no_content_type(self): + _trace_task_ret( + self.add.name, 'id1', {}, ((2, 2), {}), None, None, + app=self.app, + ) + + def test_fast_trace_task__no_content_type(self): + self.app.tasks[self.add.name].__trace__ = build_tracer( + self.add.name, self.add, app=self.app, + ) + _fast_trace_task( + self.add.name, 'id1', {}, ((2, 2), {}), None, None, + app=self.app, _loc=[self.app.tasks, {}, 'hostname'] + ) + def test_trace_exception_propagate(self): with self.assertRaises(KeyError): self.trace(self.raises, (KeyError('foo'),), {}, propagate=True) diff --git a/celery/tests/utils/test_functional.py b/celery/tests/utils/test_functional.py index 043646fe0..e2ef575c3 100644 --- a/celery/tests/utils/test_functional.py +++ b/celery/tests/utils/test_functional.py @@ -3,21 +3,37 @@ import pickle import sys +from itertools import count + from kombu.utils.functional import lazy from celery.five import THREAD_TIMEOUT_MAX, items, range, nextfun from celery.utils.functional import ( + DummyContext, LRUCache, + head_from_fun, firstmethod, first, + maybe_list, + memoize, mlazy, padlist, - maybe_list, + regen, ) from celery.tests.case import Case, SkipTest +class test_DummyContext(Case): + + def test_context(self): + with DummyContext(): + pass + with self.assertRaises(KeyError): + with DummyContext(): + raise KeyError() + + class test_LRUCache(Case): def test_expires(self): @@ -176,6 +192,24 @@ def test_maybe_list(self): self.assertIsNone(maybe_list(None)) +class test_memoize(Case): + + def test_memoize(self): + counter = count(1) + + @memoize(maxsize=2) + def x(i): + return next(counter) + + self.assertEqual(x(1), 1) + self.assertEqual(x(1), 1) + self.assertEqual(x(2), 2) + self.assertEqual(x(3), 3) + self.assertEqual(x(1), 4) + x.clear() + self.assertEqual(x(3), 5) + + class test_mlazy(Case): def test_is_memoized(self): @@ -186,3 +220,74 @@ def test_is_memoized(self): self.assertTrue(p.evaluated) self.assertEqual(p(), 20) self.assertEqual(repr(p), '20') + + +class test_regen(Case): + + def test_regen_list(self): + l = [1, 2] + r = regen(iter(l)) + self.assertIs(regen(l), l) + self.assertEqual(r, l) + self.assertEqual(r, l) + self.assertEqual(r.__length_hint__(), 0) + + fun, args = r.__reduce__() + self.assertEqual(fun(*args), l) + + def test_regen_gen(self): + g = regen(iter(list(range(10)))) + self.assertEqual(g[7], 7) + self.assertEqual(g[6], 6) + self.assertEqual(g[5], 5) + self.assertEqual(g[4], 4) + self.assertEqual(g[3], 3) + self.assertEqual(g[2], 2) + self.assertEqual(g[1], 1) + self.assertEqual(g[0], 0) + self.assertEqual(g.data, list(range(10))) + self.assertEqual(g[8], 8) + self.assertEqual(g[0], 0) + g = regen(iter(list(range(10)))) + self.assertEqual(g[0], 0) + self.assertEqual(g[1], 1) + self.assertEqual(g.data, list(range(10))) + g = regen(iter([1])) + self.assertEqual(g[0], 1) + with self.assertRaises(IndexError): + g[1] + self.assertEqual(g.data, [1]) + + g = regen(iter(list(range(10)))) + self.assertEqual(g[-1], 9) + self.assertEqual(g[-2], 8) + self.assertEqual(g[-3], 7) + self.assertEqual(g[-4], 6) + self.assertEqual(g[-5], 5) + self.assertEqual(g[5], 5) + self.assertEqual(g.data, list(range(10))) + + self.assertListEqual(list(iter(g)), list(range(10))) + + +class test_head_from_fun(Case): + + def test_from_cls(self): + class X(object): + def __call__(x, y, kwarg=1): + pass + + g = head_from_fun(X()) + with self.assertRaises(TypeError): + g(1) + g(1, 2) + g(1, 2, kwarg=3) + + def test_from_fun(self): + def f(x, y, kwarg=1): + pass + g = head_from_fun(f) + with self.assertRaises(TypeError): + g(1) + g(1, 2) + g(1, 2, kwarg=3) diff --git a/celery/tests/utils/test_imports.py b/celery/tests/utils/test_imports.py index d714451f9..f477d8f62 100644 --- a/celery/tests/utils/test_imports.py +++ b/celery/tests/utils/test_imports.py @@ -19,6 +19,7 @@ def test_find_module(self): imp.return_value = None with self.assertRaises(NotAPackage): find_module('foo.bar.baz', imp=imp) + self.assertTrue(find_module('celery.worker.request')) def test_qualname(self): Class = type('Fox', (object,), {'__module__': 'quick.brown'}) diff --git a/celery/tests/utils/test_local.py b/celery/tests/utils/test_local.py index 67b44b221..febcb8a97 100644 --- a/celery/tests/utils/test_local.py +++ b/celery/tests/utils/test_local.py @@ -31,6 +31,12 @@ def test_std_class_attributes(self): self.assertEqual(Proxy.__module__, 'celery.local') self.assertIsInstance(Proxy.__doc__, str) + def test_doc(self): + def real(): + pass + x = Proxy(real, __doc__='foo') + self.assertEqual(x.__doc__, 'foo') + def test_name(self): def real(): diff --git a/celery/tests/utils/test_platforms.py b/celery/tests/utils/test_platforms.py index 5c4e568d5..10b345a2a 100644 --- a/celery/tests/utils/test_platforms.py +++ b/celery/tests/utils/test_platforms.py @@ -12,7 +12,9 @@ from celery.platforms import ( get_fdmax, ignore_errno, + check_privileges, set_process_title, + set_mp_process_title, signals, maybe_drop_privileges, setuid, @@ -61,9 +63,14 @@ class test_fd_by_path(Case): def test_finds(self): test_file = tempfile.NamedTemporaryFile() - keep = fd_by_path([test_file.name]) - self.assertEqual(keep, [test_file.file.fileno()]) - test_file.close() + try: + keep = fd_by_path([test_file.name]) + self.assertEqual(keep, [test_file.file.fileno()]) + with patch('os.open') as _open: + _open.side_effect = OSError() + self.assertFalse(fd_by_path([test_file.name])) + finally: + test_file.close() class test_close_open_fds(Case): @@ -99,13 +106,27 @@ def test_otherwise(self): class test_set_process_title(Case): - def when_no_setps(self): - prev = platforms._setproctitle = platforms._setproctitle, None + def test_no_setps(self): + prev, platforms._setproctitle = platforms._setproctitle, None try: set_process_title('foo') finally: platforms._setproctitle = prev + @patch('celery.platforms.set_process_title') + @patch('celery.platforms.current_process') + def test_mp_no_hostname(self, current_process, set_process_title): + current_process().name = 'Foo' + set_mp_process_title('foo', info='hello') + set_process_title.assert_called_with('foo:Foo', info='hello') + + @patch('celery.platforms.set_process_title') + @patch('celery.platforms.current_process') + def test_mp_hostname(self, current_process, set_process_title): + current_process().name = 'Foo' + set_mp_process_title('foo', hostname='a@q.com', info='hello') + set_process_title.assert_called_with('foo: a@q.com:Foo', info='hello') + class test_Signals(Case): @@ -146,6 +167,11 @@ def test_ignore(self, set): signals.ignore('SIGTERM') set.assert_called_with(signals.signum('TERM'), signals.ignored) + @patch('signal.signal') + def test_reset(self, set): + signals.reset('SIGINT') + set.assert_called_with(signals.signum('INT'), signals.default) + @patch('signal.signal') def test_setitem(self, set): def handle(*args): @@ -180,13 +206,27 @@ def test_when_actual(self, getrlimit): class test_maybe_drop_privileges(Case): + def test_on_windows(self): + prev, sys.platform = sys.platform, 'win32' + try: + maybe_drop_privileges() + finally: + sys.platform = prev + + @patch('os.getegid') + @patch('os.getgid') + @patch('os.geteuid') + @patch('os.getuid') @patch('celery.platforms.parse_uid') @patch('pwd.getpwuid') @patch('celery.platforms.setgid') @patch('celery.platforms.setuid') @patch('celery.platforms.initgroups') def test_with_uid(self, initgroups, setuid, setgid, - getpwuid, parse_uid): + getpwuid, parse_uid, getuid, geteuid, + getgid, getegid): + geteuid.return_value = 10 + getuid.return_value = 10 class pw_struct(object): pw_gid = 50001 @@ -204,6 +244,40 @@ def raise_on_second_call(*args, **kwargs): initgroups.assert_called_with(5001, 50001) setuid.assert_has_calls([call(5001), call(0)]) + setuid.side_effect = raise_on_second_call + + def to_root_on_second_call(mock, first): + return_value = [first] + + def on_first_call(*args, **kwargs): + ret, return_value[0] = return_value[0], 0 + return ret + mock.side_effect = on_first_call + to_root_on_second_call(geteuid, 10) + to_root_on_second_call(getuid, 10) + with self.assertRaises(AssertionError): + maybe_drop_privileges(uid='user') + + getuid.return_value = getuid.side_effect = None + geteuid.return_value = geteuid.side_effect = None + getegid.return_value = 0 + getgid.return_value = 0 + setuid.side_effect = raise_on_second_call + with self.assertRaises(AssertionError): + maybe_drop_privileges(gid='group') + + getuid.reset_mock() + geteuid.reset_mock() + setuid.reset_mock() + getuid.side_effect = geteuid.side_effect = None + + def raise_on_second_call(*args, **kwargs): + setuid.side_effect = OSError() + setuid.side_effect.errno = errno.ENOENT + setuid.side_effect = raise_on_second_call + with self.assertRaises(OSError): + maybe_drop_privileges(uid='user') + @patch('celery.platforms.parse_uid') @patch('celery.platforms.parse_gid') @patch('celery.platforms.setgid') @@ -421,6 +495,20 @@ def test_open(self, dup2, open, close, closer, umask, chdir, pass x.after_chdir.assert_called_with() + x = DaemonContext(workdir='/opt/workdir', umask="0755") + self.assertEqual(x.umask, 493) + x = DaemonContext(workdir='/opt/workdir', umask="493") + self.assertEqual(x.umask, 493) + + x.redirect_to_null(None) + + with patch('celery.platforms.mputil') as mputil: + x = DaemonContext(after_forkers=True) + x.open() + mputil._run_after_forkers.assert_called_with() + x = DaemonContext(after_forkers=False) + x.open() + class test_Pidfile(Case): @patch('celery.platforms.Pidfile') @@ -711,3 +799,21 @@ def test_setgroups_raises_EPERM(self, hack, sysconf, getgroups): with self.assertRaises(OSError): setgroups(list(range(400))) getgroups.assert_called_with() + + +class test_check_privileges(Case): + + def test_suspicious(self): + class Obj(object): + fchown = 13 + prev, platforms.os = platforms.os, Obj() + try: + with self.assertRaises(AssertionError): + check_privileges({'pickle'}) + finally: + platforms.os = prev + prev, platforms.os = platforms.os, object() + try: + check_privileges({'pickle'}) + finally: + platforms.os = prev diff --git a/celery/tests/utils/test_saferepr.py b/celery/tests/utils/test_saferepr.py index a7e8348ef..ce2b81df5 100644 --- a/celery/tests/utils/test_saferepr.py +++ b/celery/tests/utils/test_saferepr.py @@ -148,6 +148,15 @@ def test_text_maxlen(self): saferepr(D_D_TEXT, 100).endswith("...', ...}}") ) + def test_maxlevels(self): + saferepr(D_ALL, maxlevels=1) + + def test_recursion(self): + d = {1: 2, 3: {4: 5}} + d[3][6] = d + res = saferepr(d) + self.assertIn('Recursion on', res) + def test_same_as_repr(self): # Simple objects, small containers and classes that overwrite __repr__ # For those the result should be the same as repr(). diff --git a/celery/tests/utils/test_timer2.py b/celery/tests/utils/test_timer2.py index 582e54366..a549c78c6 100644 --- a/celery/tests/utils/test_timer2.py +++ b/celery/tests/utils/test_timer2.py @@ -5,7 +5,7 @@ import celery.utils.timer2 as timer2 -from celery.tests.case import Case, Mock, patch +from celery.tests.case import Case, Mock, patch, call from kombu.tests.case import redirect_stdouts @@ -98,6 +98,11 @@ def test_ensure_started_not_started(self): t.start = Mock() t.ensure_started() self.assertFalse(t.start.called) + t.running = False + t.on_start = Mock() + t.ensure_started() + t.on_start.assert_called_with(t) + t.start.assert_called_with() def test_call_repeatedly(self): t = timer2.Timer() @@ -136,6 +141,17 @@ def test_apply_entry_error_handled(self, logger): t.schedule.apply_entry(fun) self.assertTrue(logger.error.called) + @patch('celery.utils.timer2.sleep') + def test_on_tick(self, sleep): + on_tick = Mock(name='on_tick') + t = timer2.Timer(on_tick=on_tick) + ne = t._next_entry = Mock(name='_next_entry') + ne.return_value = 3.33 + self.on_nth_call_do(ne, t._is_shutdown.set, 3) + t.run() + sleep.assert_called_with(3.33) + on_tick.assert_has_class(call(3.33), call(3.33), call(3.33)) + @redirect_stdouts def test_apply_entry_error_not_handled(self, stdout, stderr): t = timer2.Timer() diff --git a/celery/tests/utils/test_timeutils.py b/celery/tests/utils/test_timeutils.py index f72794017..f97548d75 100644 --- a/celery/tests/utils/test_timeutils.py +++ b/celery/tests/utils/test_timeutils.py @@ -248,6 +248,6 @@ class test_utcoffset(Case): def test_utcoffset(self): with patch('celery.utils.timeutils._time') as _time: _time.daylight = True - self.assertIsNotNone(utcoffset()) + self.assertIsNotNone(utcoffset(time=_time)) _time.daylight = False - self.assertIsNotNone(utcoffset()) + self.assertIsNotNone(utcoffset(time=_time)) diff --git a/celery/tests/worker/test_autoreload.py b/celery/tests/worker/test_autoreload.py index e61b330ca..19de84176 100644 --- a/celery/tests/worker/test_autoreload.py +++ b/celery/tests/worker/test_autoreload.py @@ -18,7 +18,7 @@ Autoreloader, ) -from celery.tests.case import AppCase, Case, Mock, SkipTest, patch, mock_open +from celery.tests.case import AppCase, Case, Mock, patch, mock_open class test_WorkerComponent(AppCase): @@ -75,6 +75,7 @@ def test_start_stop_on_change(self): x._on_change = Mock() x.on_change('foo') x._on_change.assert_called_with('foo') + x.on_event_loop_close(Mock()) class test_StatMonitor(Case): @@ -99,6 +100,12 @@ def on_is_set(): stat.side_effect = OSError() x.start() + def test_register_with_event_loop(self): + hub = Mock(name='hub') + x = StatMonitor(['a']) + x.register_with_event_loop(hub) + hub.call_repeatedly.assert_called_with(2.0, x.find_changes) + @patch('os.stat') def test_mtime_stat_raises(self, stat): stat.side_effect = ValueError() @@ -122,10 +129,8 @@ def test_stop(self, close, kqueue): close.side_effect.errno = errno.EBADF x.stop() - def test_register_with_event_loop(self): - from kombu.utils import eventio - if eventio.kqueue is None: - raise SkipTest('version of kombu does not work with pypy') + @patch('kombu.utils.eventio.kqueue', create=True) + def test_register_with_event_loop(self, kqueue): x = KQueueMonitor(['a', 'b']) hub = Mock(name='hub') x.add_events = Mock(name='add_events()') @@ -136,6 +141,15 @@ def test_register_with_event_loop(self): x.handle_event, ) + def test_register_with_event_loop_no_kqueue(self): + from kombu.utils import eventio + prev, eventio.kqueue = eventio.kqueue, None + try: + x = KQueueMonitor(['a']) + x.register_with_event_loop(Mock()) + finally: + eventio.kqueue = prev + def test_on_event_loop_close(self): x = KQueueMonitor(['a', 'b']) x.close = Mock() @@ -201,21 +215,34 @@ class test_InotifyMonitor(Case): @patch('celery.worker.autoreload.pyinotify') def test_start(self, inotify): - x = InotifyMonitor(['a']) - inotify.IN_MODIFY = 1 - inotify.IN_ATTRIB = 2 + x = InotifyMonitor(['a']) + inotify.IN_MODIFY = 1 + inotify.IN_ATTRIB = 2 + x.start() + + inotify.WatchManager.side_effect = ValueError() + with self.assertRaises(ValueError): x.start() + x.stop() - inotify.WatchManager.side_effect = ValueError() - with self.assertRaises(ValueError): - x.start() - x.stop() + x._on_change = None + x.process_(Mock()) + x._on_change = Mock() + x.process_(Mock()) + self.assertTrue(x._on_change.called) - x._on_change = None - x.process_(Mock()) - x._on_change = Mock() - x.process_(Mock()) - self.assertTrue(x._on_change.called) + x.create_notifier = Mock() + x._wm = Mock() + hub = Mock() + x.register_with_event_loop(hub) + x.create_notifier.assert_called_with() + hub.add_reader.assert_called_with(x._wm.get_fd(), x.on_readable) + + x.on_event_loop_close(hub) + x._notifier = Mock() + x.on_readable() + x._notifier.read_events.assert_called_with() + x._notifier.process_events.assert_called_with() class test_default_implementation(Case): diff --git a/celery/tests/worker/test_bootsteps.py b/celery/tests/worker/test_bootsteps.py index f35f66919..8482fd825 100644 --- a/celery/tests/worker/test_bootsteps.py +++ b/celery/tests/worker/test_bootsteps.py @@ -148,6 +148,12 @@ def get_consumers(self, c): step = Step(self) step.start(self) + def test_close_no_consumer_channel(self): + step = bootsteps.ConsumerStep(Mock()) + step.consumers = [Mock()] + step.consumers[0].channel = None + step._close(Mock()) + class test_StartStopStep(AppCase): @@ -177,6 +183,11 @@ def test_start__stop(self): x.obj = None self.assertIsNone(x.start(self)) + def test_terminate__no_obj(self): + x = self.Def(self) + x.obj = None + x.terminate(Mock()) + def test_include_when_disabled(self): x = self.Def(self) x.enabled = False @@ -237,6 +248,20 @@ def test_send_all_with_None_steps(self): parent.steps = [None, None, None] blueprint.send_all(parent, 'close', 'Closing', reverse=False) + def test_send_all_raises(self): + parent = Mock() + blueprint = self.Blueprint(app=self.app) + parent.steps = [Mock()] + parent.steps[0].foo.side_effect = KeyError() + blueprint.send_all(parent, 'foo', propagate=False) + with self.assertRaises(KeyError): + blueprint.send_all(parent, 'foo', propagate=True) + + def test_stop_state_in_TERMINATE(self): + blueprint = self.Blueprint(app=self.app) + blueprint.state = bootsteps.TERMINATE + blueprint.stop(Mock()) + def test_join_raises_IGNORE_ERRORS(self): prev, bootsteps.IGNORE_ERRORS = bootsteps.IGNORE_ERRORS, (KeyError,) try: diff --git a/celery/tests/worker/test_components.py b/celery/tests/worker/test_components.py index 4a5f898bf..7a65bc4a7 100644 --- a/celery/tests/worker/test_components.py +++ b/celery/tests/worker/test_components.py @@ -4,10 +4,46 @@ # here to complete coverage. Should move everyting to this module at some # point [-ask] +from celery.exceptions import ImproperlyConfigured from celery.platforms import IS_WINDOWS -from celery.worker.components import Pool +from celery.worker.components import Beat, Hub, Pool, Timer -from celery.tests.case import AppCase, Mock, SkipTest +from celery.tests.case import AppCase, Mock, SkipTest, patch + + +class test_Timer(AppCase): + + def test_create__eventloop(self): + w = Mock(name='w') + w.use_eventloop = True + Timer(w).create(w) + self.assertFalse(w.timer.queue) + + +class test_Hub(AppCase): + + def setup(self): + self.w = Mock(name='w') + self.hub = Hub(self.w) + self.w.hub = Mock(name='w.hub') + + @patch('celery.worker.components.set_event_loop') + @patch('celery.worker.components.get_event_loop') + def test_create(self, get_event_loop, set_event_loop): + self.hub._patch_thread_primitives = Mock(name='ptp') + self.assertIs(self.hub.create(self.w), self.hub) + self.hub._patch_thread_primitives.assert_called_with(self.w) + + def test_start(self): + self.hub.start(self.w) + + def test_stop(self): + self.hub.stop(self.w) + self.w.hub.close.assert_called_with() + + def test_terminate(self): + self.hub.terminate(self.w) + self.w.hub.close.assert_called_with() class test_Pool(AppCase): @@ -46,3 +82,12 @@ def test_create_calls_instantiate_with_max_memory(self): self.assertEqual( comp.instantiate.call_args[1]['max_memory_per_child'], 32) + + +class test_Beat(AppCase): + + def test_create__green(self): + w = Mock(name='w') + w.pool_cls.__module__ = 'foo_gevent' + with self.assertRaises(ImproperlyConfigured): + Beat(w).create(w) diff --git a/celery/tests/worker/test_control.py b/celery/tests/worker/test_control.py index 73896a55c..691e6e51d 100644 --- a/celery/tests/worker/test_control.py +++ b/celery/tests/worker/test_control.py @@ -562,6 +562,10 @@ def test_pool_restart(self): consumer.update_strategies.assert_called_with() self.assertFalse(_reload.called) self.assertFalse(_import.called) + consumer.controller.pool.restart.side_effect = NotImplementedError() + panel.handle('pool_restart', {'reloader': _reload}) + consumer.controller.consumer = None + panel.handle('pool_restart', {'reloader': _reload}) def test_pool_restart_import_modules(self): consumer = Consumer(self.app) diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index 7ea6da27d..874d5def6 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -12,10 +12,11 @@ from kombu.common import QoS, ignore_errors from kombu.transport.base import Message -from celery.bootsteps import RUN, CLOSE, StartStopStep +from celery.bootsteps import RUN, CLOSE, TERMINATE, StartStopStep from celery.concurrency.base import BasePool from celery.exceptions import ( - WorkerShutdown, WorkerTerminate, TaskRevokedError, InvalidTaskError, + WorkerShutdown, WorkerTerminate, TaskRevokedError, + InvalidTaskError, ImproperlyConfigured, ) from celery.five import Empty, range, Queue as FastQueue from celery.platforms import EX_FAILURE @@ -828,6 +829,17 @@ def test_setup_queues_worker_direct(self): worker_direct(self.worker.hostname), ) + def test_setup_queues__missing_queue(self): + self.app.amqp.queues.select = Mock(name='select') + self.app.amqp.queues.deselect = Mock(name='deselect') + self.app.amqp.queues.select.side_effect = KeyError() + self.app.amqp.queues.deselect.side_effect = KeyError() + with self.assertRaises(ImproperlyConfigured): + self.worker.setup_queues("x,y", exclude="foo,bar") + self.app.amqp.queues.select = Mock(name='select') + with self.assertRaises(ImproperlyConfigured): + self.worker.setup_queues("x,y", exclude="foo,bar") + def test_send_worker_shutdown(self): with patch('celery.signals.worker_shutdown') as ws: self.worker._send_worker_shutdown() @@ -1031,6 +1043,23 @@ def test_signal_consumer_close(self): worker.consumer.close.side_effect = AttributeError() worker.signal_consumer_close() + def test_rusage__no_resource(self): + from celery import worker + prev, worker.resource = worker.resource, None + try: + self.worker.pool = Mock(name='pool') + with self.assertRaises(NotImplementedError): + self.worker.rusage() + self.worker.stats() + finally: + worker.resource = prev + + def test_repr(self): + self.assertTrue(repr(self.worker)) + + def test_str(self): + self.assertEqual(str(self.worker), self.worker.hostname) + def test_start__stop(self): worker = self.worker worker.blueprint.shutdown_complete.set() @@ -1046,7 +1075,7 @@ def test_start__stop(self): for w in worker.steps: self.assertTrue(w.start.call_count) worker.consumer = Mock() - worker.stop() + worker.stop(exitcode=3) for stopstep in worker.steps: self.assertTrue(stopstep.close.call_count) self.assertTrue(stopstep.stop.call_count) @@ -1061,6 +1090,24 @@ def test_start__stop(self): worker.start() worker.stop() + def test_start__KeyboardInterrupt(self): + worker = self.worker + worker.blueprint = Mock(name='blueprint') + worker.blueprint.start.side_effect = KeyboardInterrupt() + worker.stop = Mock(name='stop') + worker.start() + worker.stop.assert_called_with(exitcode=EX_FAILURE) + + def test_register_with_event_loop(self): + worker = self.worker + hub = Mock(name='hub') + worker.blueprint = Mock(name='blueprint') + worker.register_with_event_loop(hub) + worker.blueprint.send_all.assert_called_with( + worker, 'register_with_event_loop', args=(hub,), + description='hub.register', + ) + def test_step_raises(self): worker = self.worker step = Mock() @@ -1087,6 +1134,8 @@ def test_start__terminate(self): worker.terminate() for step in worker.steps: self.assertTrue(step.terminate.call_count) + worker.blueprint.state = TERMINATE + worker.terminate() def test_Hub_crate(self): w = Mock() diff --git a/celery/utils/log.py b/celery/utils/log.py index 778519001..5907ca7c3 100644 --- a/celery/utils/log.py +++ b/celery/utils/log.py @@ -59,7 +59,7 @@ def iter_open_logger_fds(): try: for handler in logger.handlers: try: - if handler not in seen: + if handler not in seen: # pragma: no cover yield handler.stream seen.add(handler) except AttributeError: @@ -91,7 +91,7 @@ def logger_isa(l, p, max=1000): this = this.parent if not this: break - else: + else: # pragma: no cover raise RuntimeError('Logger hierarchy exceeds {0}'.format(max)) return False @@ -99,7 +99,7 @@ def logger_isa(l, p, max=1000): def get_logger(name): l = _get_logger(name) if logging.root not in (l, l.parent) and l is not base_logger: - if not logger_isa(l, base_logger): + if not logger_isa(l, base_logger): # pragma: no cover l.parent = base_logger return l task_logger = get_logger('celery.task') @@ -154,7 +154,7 @@ def format(self, record): if isinstance(msg, string_t): return text_t(color(safe_str(msg))) return safe_str(color(msg)) - except UnicodeDecodeError: + except UnicodeDecodeError: # pragma: no cover return safe_str(msg) # skip colors except Exception as exc: prev_msg, record.exc_info, record.msg = ( @@ -258,7 +258,7 @@ def isatty(self): def get_multiprocessing_logger(): try: from billiard import util - except ImportError: + except ImportError: # pragma: no cover pass else: return util.get_logger() @@ -267,17 +267,17 @@ def get_multiprocessing_logger(): def reset_multiprocessing_logger(): try: from billiard import util - except ImportError: + except ImportError: # pragma: no cover pass else: - if hasattr(util, '_logger'): + if hasattr(util, '_logger'): # pragma: no cover util._logger = None def current_process(): try: from billiard import process - except ImportError: + except ImportError: # pragma: no cover pass else: return process.current_process() diff --git a/celery/utils/saferepr.py b/celery/utils/saferepr.py index 090369b9d..93acba08d 100644 --- a/celery/utils/saferepr.py +++ b/celery/utils/saferepr.py @@ -36,7 +36,7 @@ IS_PY3 = sys.version_info[0] == 3 -if IS_PY3: +if IS_PY3: # pragma: no cover range_t = (range, ) else: class range_t(object): # noqa @@ -110,7 +110,7 @@ def _saferepr(o, maxlen=None, maxlevels=3, seen=None): val = saferepr(token.value, maxlen, maxlevels) elif isinstance(token, _quoted): val = token.value - if IS_PY3 and isinstance(val, bytes): + if IS_PY3 and isinstance(val, bytes): # pragma: no cover val = "b'%s'" % (bytes_to_str(truncate_bytes(val, maxlen)),) else: val = "'%s'" % (truncate(val, maxlen),) @@ -163,7 +163,7 @@ def reprstream(stack, seen=None, maxlevels=3, level=0, isinstance=isinstance): yield text_t(val), it elif isinstance(val, chars_t): yield _quoted(val), it - elif isinstance(val, range_t): + elif isinstance(val, range_t): # pragma: no cover yield repr(val), it else: if isinstance(val, set_t): diff --git a/celery/utils/timeutils.py b/celery/utils/timeutils.py index 708f57a9d..570c34490 100644 --- a/celery/utils/timeutils.py +++ b/celery/utils/timeutils.py @@ -86,7 +86,7 @@ def dst(self, dt): def tzname(self, dt): return _time.tzname[self._isdst(dt)] - if PY3: + if PY3: # pragma: no cover def fromutc(self, dt): # The base tzinfo class no longer implements a DST @@ -122,7 +122,7 @@ def to_local(self, dt, local=None, orig=None): dt = make_aware(dt, orig or self.utc) return localize(dt, self.tz_or_local(local)) - if PY33: + if PY33: # pragma: no cover def to_system(self, dt): # tz=None is a special case since Python 3.3, and will diff --git a/celery/worker/components.py b/celery/worker/components.py index 200173d74..7d31acc69 100644 --- a/celery/worker/components.py +++ b/celery/worker/components.py @@ -92,7 +92,7 @@ def _patch_thread_primitives(self, w): # multiprocessing's ApplyResult uses this lock. try: from billiard import pool - except ImportError: + except ImportError: # pragma: no cover pass else: pool.Lock = DummyLock @@ -137,8 +137,9 @@ def terminate(self, w): if w.pool: w.pool.terminate() - def create(self, w, semaphore=None, max_restarts=None): - if w.app.conf.worker_pool in ('eventlet', 'gevent'): + def create(self, w, semaphore=None, max_restarts=None, + green_pools={'eventlet', 'gevent'}): + if w.app.conf.worker_pool in green_pools: # pragma: no cover warnings.warn(UserWarning(W_POOL_SETTING)) threaded = not w.use_eventloop or IS_WINDOWS procs = w.min_concurrency From c673fe201bb03005a9a7d3f4ed00671c840e1917 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 23 Nov 2015 12:27:37 -0800 Subject: [PATCH 0824/1103] Fixes failing test --- celery/beat.py | 6 ++++++ celery/tests/app/test_beat.py | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/celery/beat.py b/celery/beat.py index 2c63f12e0..16871fd10 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -145,6 +145,12 @@ def __repr__(self): def __lt__(self, other): if isinstance(other, ScheduleEntry): + # How the object is ordered doesn't really matter, as + # in the scheduler heap, the order is decided by the + # preceding members of the tuple ``(time, priority, entry)``. + # + # If all that is left to order on is the entry then it can + # just as well be random. return id(self) < id(other) return NotImplemented diff --git a/celery/tests/app/test_beat.py b/celery/tests/app/test_beat.py index 1eab5bcb7..e842267f1 100644 --- a/celery/tests/app/test_beat.py +++ b/celery/tests/app/test_beat.py @@ -92,8 +92,9 @@ def test_reduce(self): def test_lt(self): e1 = self.create_entry(schedule=timedelta(seconds=10)) e2 = self.create_entry(schedule=timedelta(seconds=2)) - self.assertLess(e2, e1) - self.assertTrue(e1 < object()) + # order doesn't matter, see comment in __lt__ + res1 = e1 < e2 # noqa + res2 = e1 < object() # noqa def test_update(self): entry = self.create_entry() From 782817000b10a60ec934959664d33cf7410794ce Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 23 Nov 2015 12:33:53 -0800 Subject: [PATCH 0825/1103] Fixes second test failing --- celery/tests/events/test_events.py | 4 ++-- celery/tests/fixups/test_django.py | 5 +++-- celery/tests/utils/test_timer2.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/celery/tests/events/test_events.py b/celery/tests/events/test_events.py index 41899c1ce..44ef3c58f 100644 --- a/celery/tests/events/test_events.py +++ b/celery/tests/events/test_events.py @@ -120,9 +120,9 @@ def on_eventer_publish(events, *args, **kwargs): eventer.on_send_buffered.assert_called_with() eventer.send('task-received', uuid=1) self.assertFalse(eventer._group_buffer['task']) - eventer._publish.assert_has_calls( + eventer._publish.assert_has_calls([ call([], eventer.producer, 'task.multi'), - ) + ]) # clear in place self.assertIs(eventer._group_buffer['task'], prev_buffer) self.assertEqual(len(buf_received[0]), 2) diff --git a/celery/tests/fixups/test_django.py b/celery/tests/fixups/test_django.py index 423292f7f..8da192e03 100644 --- a/celery/tests/fixups/test_django.py +++ b/celery/tests/fixups/test_django.py @@ -60,8 +60,9 @@ def test_autodiscover_tasks_pre17(self): f._settings = Mock(name='_settings') self.assertIs(f.autodiscover_tasks(), f._settings.INSTALLED_APPS) - @patch('django.apps.apps', create=True) - def test_autodiscover_tasks(self, apps): + def test_autodiscover_tasks(self): + self.mock_modules('django.apps') + from django.apps import apps f = DjangoFixup(self.app) configs = [Mock(name='c1'), Mock(name='c2')] apps.get_app_configs.return_value = configs diff --git a/celery/tests/utils/test_timer2.py b/celery/tests/utils/test_timer2.py index a549c78c6..5bcd1ba37 100644 --- a/celery/tests/utils/test_timer2.py +++ b/celery/tests/utils/test_timer2.py @@ -150,7 +150,7 @@ def test_on_tick(self, sleep): self.on_nth_call_do(ne, t._is_shutdown.set, 3) t.run() sleep.assert_called_with(3.33) - on_tick.assert_has_class(call(3.33), call(3.33), call(3.33)) + on_tick.assert_has_calls([call(3.33), call(3.33), call(3.33)]) @redirect_stdouts def test_apply_entry_error_not_handled(self, stdout, stderr): From c60d990719426c88d1424a9f564313c53e942ee6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 23 Nov 2015 13:29:19 -0800 Subject: [PATCH 0826/1103] Fixes py3 test problems --- celery/tests/app/test_beat.py | 5 ++++- celery/tests/bin/test_base.py | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/celery/tests/app/test_beat.py b/celery/tests/app/test_beat.py index e842267f1..6ce5a8d2e 100644 --- a/celery/tests/app/test_beat.py +++ b/celery/tests/app/test_beat.py @@ -94,7 +94,10 @@ def test_lt(self): e2 = self.create_entry(schedule=timedelta(seconds=2)) # order doesn't matter, see comment in __lt__ res1 = e1 < e2 # noqa - res2 = e1 < object() # noqa + try: + res2 = e1 < object() # noqa + except TypeError: + pass def test_update(self): entry = self.create_entry() diff --git a/celery/tests/bin/test_base.py b/celery/tests/bin/test_base.py index 8ff572161..fd6657f40 100644 --- a/celery/tests/bin/test_base.py +++ b/celery/tests/bin/test_base.py @@ -245,8 +245,11 @@ def test_find_app_suspects(self): with self.assertRaises(AttributeError): cmd.find_app(__name__) - @patch('celery.bin.base.input') - def test_ask(self, input): + def test_ask(self): + try: + input = self.patch('celery.bin.base.input') + except AttributeError: + input = self.patch('builtins.input') cmd = MockCommand(app=self.app) input.return_value = 'yes' self.assertEqual(cmd.ask('q', ('yes', 'no'), 'no'), 'yes') From 8c62dbe76c617d6b03b9623d5350edca580b08af Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 23 Nov 2015 13:29:39 -0800 Subject: [PATCH 0827/1103] Fixes pypy3 tests --- celery/tests/app/test_app.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index de7324ded..304037265 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -845,18 +845,21 @@ def test_timezone__none_set(self): self.assertEqual(tz, timezone.get_timezone('UTC')) def test_compat_on_configure(self): - on_configure = Mock(name='on_configure') + _on_configure = Mock(name='on_configure') class CompatApp(Celery): def on_configure(self, *args, **kwargs): - on_configure(*args, **kwargs) + # on pypy3 if named on_configure the class function + # will be called, instead of the mock defined above, + # so we add the underscore. + _on_configure(*args, **kwargs) with CompatApp(set_as_current=False) as app: app.loader = Mock() app.loader.conf = {} app._load_config() - on_configure.assert_called_with() + _on_configure.assert_called_with() def test_add_periodic_task(self): From 4cb46eaf8a4d37b15c79fc64c147ed75008b5d82 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 23 Nov 2015 13:29:49 -0800 Subject: [PATCH 0828/1103] Tox min cover percentage: 96 --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9a087101b..8230c4208 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,9 @@ sitepackages = False recreate = False commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -q -U -r{toxinidir}/requirements/dev.txt - nosetests -xsv --with-coverage --cover-inclusive --cover-erase [] + nosetests -xsv --with-coverage \ + --cover-inclusive --cover-min-percentage=96 --cover-erase [] + basepython = 2.7: python2.7 3.4: python3.4 From b4d122adea1c82427f33dac95ee0a70aee3b5108 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 23 Nov 2015 15:20:42 -0800 Subject: [PATCH 0829/1103] Attempt to fix CI --- celery/tests/utils/test_platforms.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/celery/tests/utils/test_platforms.py b/celery/tests/utils/test_platforms.py index 10b345a2a..2864dccf4 100644 --- a/celery/tests/utils/test_platforms.py +++ b/celery/tests/utils/test_platforms.py @@ -218,12 +218,13 @@ def test_on_windows(self): @patch('os.geteuid') @patch('os.getuid') @patch('celery.platforms.parse_uid') + @patch('celery.platforms.parse_gid') @patch('pwd.getpwuid') @patch('celery.platforms.setgid') @patch('celery.platforms.setuid') @patch('celery.platforms.initgroups') def test_with_uid(self, initgroups, setuid, setgid, - getpwuid, parse_uid, getuid, geteuid, + getpwuid, parse_gid, parse_uid, getuid, geteuid, getgid, getegid): geteuid.return_value = 10 getuid.return_value = 10 @@ -237,6 +238,7 @@ def raise_on_second_call(*args, **kwargs): setuid.side_effect = raise_on_second_call getpwuid.return_value = pw_struct() parse_uid.return_value = 5001 + parse_gid.return_value = 5001 maybe_drop_privileges(uid='user') parse_uid.assert_called_with('user') getpwuid.assert_called_with(5001) From 227fdab4acd6d79c6b97f3e4712f7d91e6e56c86 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 23 Nov 2015 15:31:47 -0800 Subject: [PATCH 0830/1103] pypy3 lowering our coverage minimum :( --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8230c4208..6e006f4aa 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@ recreate = False commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -q -U -r{toxinidir}/requirements/dev.txt nosetests -xsv --with-coverage \ - --cover-inclusive --cover-min-percentage=96 --cover-erase [] + --cover-inclusive --cover-min-percentage=95 --cover-erase [] basepython = 2.7: python2.7 From f63bea5feda09df75612089745e8ce72bb871687 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 24 Nov 2015 12:40:53 -0800 Subject: [PATCH 0831/1103] Moved dictfilter to Kombu (Issue celery/kombu#542) --- celery/utils/functional.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 1966b9003..cf5b9df1e 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -17,7 +17,9 @@ from itertools import chain, islice from amqp import promise -from kombu.utils.functional import lazy, maybe_evaluate, is_list, maybe_list +from kombu.utils.functional import ( + dictfilter, lazy, maybe_evaluate, is_list, maybe_list, +) from celery.five import UserDict, UserList, items, keys, range @@ -354,12 +356,6 @@ def data(self): return self.__consumed -def dictfilter(d=None, **kw): - """Remove all keys from dict ``d`` whose value is :const:`None`""" - d = kw if d is None else (dict(d, **kw) if kw else d) - return {k: v for k, v in items(d) if v is not None} - - def _argsfromspec(spec, replace_defaults=True): if spec.defaults: split = len(spec.defaults) From 5d144c94b597a28c070fd197f69b1bed9ca2a4d2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 24 Nov 2015 12:41:07 -0800 Subject: [PATCH 0832/1103] autoexchange was being ignored --- celery/app/amqp.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index c87f454e8..574213221 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -91,8 +91,7 @@ def __getitem__(self, name): return dict.__getitem__(self, name) def __setitem__(self, name, queue): - if self.default_exchange and (not queue.exchange or - not queue.exchange.name): + if self.default_exchange and not queue.exchange: queue.exchange = self.default_exchange dict.__setitem__(self, name, queue) if queue.alias: From a33ddfa1bc988e923c751c282e72375ed23b348c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 24 Nov 2015 14:12:00 -0800 Subject: [PATCH 0833/1103] Fixes tests --- celery/app/amqp.py | 2 +- celery/tests/app/test_amqp.py | 2 +- celery/worker/components.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 574213221..8ea5455a1 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -525,7 +525,7 @@ def send_task_message(producer, name, message, eta=body['eta'], taskset=body['taskset']) if sent_event: evd = event_dispatcher or default_evd - exname = exchange or self.exchange + exname = exchange if isinstance(exname, Exchange): exname = exname.name sent_event.update({ diff --git a/celery/tests/app/test_amqp.py b/celery/tests/app/test_amqp.py index 200182ba2..06104e26b 100644 --- a/celery/tests/app/test_amqp.py +++ b/celery/tests/app/test_amqp.py @@ -133,7 +133,7 @@ def test_add_default_exchange(self): ex = Exchange('fff', 'fanout') q = Queues(default_exchange=ex) q.add(Queue('foo')) - self.assertEqual(q['foo'].exchange, ex) + self.assertEqual(q['foo'].exchange.name, '') def test_alias(self): q = Queues() diff --git a/celery/worker/components.py b/celery/worker/components.py index 7d31acc69..3a3c56927 100644 --- a/celery/worker/components.py +++ b/celery/worker/components.py @@ -73,7 +73,9 @@ def include_if(self, w): def create(self, w): w.hub = get_event_loop() if w.hub is None: - w.hub = set_event_loop(_Hub(w.timer)) + w.hub = set_event_loop(( + w._conninfo.requires_hub + if w._conninfo.requires_hub else _Hub)(w.timer)) self._patch_thread_primitives(w) return self From 625d00b80f3c637894524379c7a00494f18c02b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=B4she=20van=20der=20Sterre?= Date: Fri, 31 Jan 2014 07:43:21 +0100 Subject: [PATCH 0834/1103] Initial support for a filesystem based result backend. --- celery/backends/__init__.py | 1 + celery/backends/filesystem.py | 91 +++++++++++++++++++ celery/tests/backends/test_filesystem.py | 79 ++++++++++++++++ docs/configuration.rst | 24 +++++ .../reference/celery.backends.filesystem.rst | 11 +++ docs/internals/reference/index.rst | 1 + 6 files changed, 207 insertions(+) create mode 100644 celery/backends/filesystem.py create mode 100644 celery/tests/backends/test_filesystem.py create mode 100644 docs/internals/reference/celery.backends.filesystem.rst diff --git a/celery/backends/__init__.py b/celery/backends/__init__.py index eec585227..91ad500c4 100644 --- a/celery/backends/__init__.py +++ b/celery/backends/__init__.py @@ -33,6 +33,7 @@ 'couchbase': 'celery.backends.couchbase:CouchBaseBackend', 'couchdb': 'celery.backends.couchdb:CouchDBBackend', 'riak': 'celery.backends.riak:RiakBackend', + 'file': 'celery.backends.filesystem:FilesystemBackend', 'disabled': 'celery.backends.base:DisabledBackend', } diff --git a/celery/backends/filesystem.py b/celery/backends/filesystem.py new file mode 100644 index 000000000..d124f5711 --- /dev/null +++ b/celery/backends/filesystem.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +""" + celery.backends.filesystem + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Filesystem result store backend. +""" +from __future__ import absolute_import + +from celery.exceptions import ImproperlyConfigured +from celery.backends.base import KeyValueStoreBackend +from celery.utils import uuid + +import os +import locale +default_encoding = locale.getpreferredencoding(False) + +# Python 2 does not have FileNotFoundError and IsADirectoryError +try: + FileNotFoundError +except NameError: + FileNotFoundError = IOError + IsADirectoryError = IOError + + +class FilesystemBackend(KeyValueStoreBackend): + def __init__(self, url=None, open=open, unlink=os.unlink, sep=os.sep, + encoding=default_encoding, *args, **kwargs): + """Initialize the filesystem backend. + + Keyword arguments (in addition to those of KeyValueStoreBackend): + url -- URL to the directory we should use + open -- open function to use when opening files + unlink -- unlink function to use when deleting files + sep -- directory seperator (to join the directory with the key) + encoding -- encoding used on the filesystem + + """ + + super(FilesystemBackend, self).__init__(*args, **kwargs) + path = self._find_path(url) + + # We need the path and seperator as bytes objects + self.path = path.encode(encoding) + self.sep = sep.encode(encoding) + + self.open = open + self.unlink = unlink + + # Lets verify that we have everything setup right + self._do_directory_test(b'.fs-backend-' + uuid().encode(encoding)) + + def _find_path(self, url): + if url is not None and url.startswith('file:///'): + return url[7:] + if hasattr(self.app.conf, 'CELERY_RESULT_FSPATH'): + return self.app.conf.CELERY_RESULT_FSPATH + raise ImproperlyConfigured( + 'You need to configure a path for the Filesystem backend') + + def _do_directory_test(self, key): + try: + self.set(key, b'test value') + assert self.get(key) == b'test value' + self.delete(key) + except IOError: + raise ImproperlyConfigured( + 'The configured path for the Filesystem backend does not ' + 'work correctly, please make sure that it exists and has ' + 'the correct permissions.') + + def _filename(self, key): + return self.sep.join((self.path, key)) + + def get(self, key): + try: + with self.open(self._filename(key), 'rb') as infile: + return infile.read() + except FileNotFoundError: + return None + + def set(self, key, value): + with self.open(self._filename(key), 'wb') as outfile: + outfile.write(value) + + def mget(self, keys): + for key in keys: + yield self.get(key) + + def delete(self, key): + self.unlink(self._filename(key)) diff --git a/celery/tests/backends/test_filesystem.py b/celery/tests/backends/test_filesystem.py new file mode 100644 index 000000000..a1a5e0231 --- /dev/null +++ b/celery/tests/backends/test_filesystem.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import + +from celery import states +from celery.tests.case import AppCase +from celery.backends.filesystem import FilesystemBackend +from celery.exceptions import ImproperlyConfigured +from celery.utils import uuid + +import os +import shutil +import tempfile + + +class test_FilesystemBackend(AppCase): + def setup(self): + self.directory = tempfile.mkdtemp() + self.url = 'file://' + self.directory + self.path = self.directory.encode('ascii') + + def teardown(self): + shutil.rmtree(self.directory) + + def test_a_path_is_required(self): + with self.assertRaises(ImproperlyConfigured): + FilesystemBackend(app=self.app) + + def test_a_path_in_app_conf(self): + self.app.conf.CELERY_RESULT_FSPATH = self.url[7:] + tb = FilesystemBackend(app=self.app) + self.assertEqual(tb.path, self.path) + + def test_a_path_in_url(self): + tb = FilesystemBackend(app=self.app, url=self.url) + self.assertEqual(tb.path, self.path) + + def test_path_is_incorrect(self): + with self.assertRaises(ImproperlyConfigured): + FilesystemBackend(app=self.app, url=self.url + '-incorrect') + + def test_missing_task_is_PENDING(self): + tb = FilesystemBackend(app=self.app, url=self.url) + self.assertEqual(tb.get_status('xxx-does-not-exist'), states.PENDING) + + def test_mark_as_done_writes_file(self): + tb = FilesystemBackend(app=self.app, url=self.url) + tb.mark_as_done(uuid(), 42) + self.assertEqual(len(os.listdir(self.directory)), 1) + + def test_done_task_is_SUCCESS(self): + tb = FilesystemBackend(app=self.app, url=self.url) + tid = uuid() + tb.mark_as_done(tid, 42) + self.assertEqual(tb.get_status(tid), states.SUCCESS) + + def test_correct_result(self): + data = {'foo': 'bar'} + + tb = FilesystemBackend(app=self.app, url=self.url) + tid = uuid() + tb.mark_as_done(tid, data) + self.assertEqual(tb.get_result(tid), data) + + def test_get_many(self): + data = {uuid(): 'foo', uuid(): 'bar', uuid(): 'baz'} + + tb = FilesystemBackend(app=self.app, url=self.url) + for key, value in data.items(): + tb.mark_as_done(key, value) + + for key, result in tb.get_many(data.keys()): + self.assertEqual(result['result'], data[key]) + + def test_forget_deletes_file(self): + tb = FilesystemBackend(app=self.app, url=self.url) + tid = uuid() + tb.mark_as_done(tid, 42) + tb.forget(tid) + self.assertEqual(len(os.listdir(self.directory)), 0) diff --git a/docs/configuration.rst b/docs/configuration.rst index 31c80beae..c3a7f9b1c 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -527,6 +527,10 @@ Can be one of the following: Older AMQP backend (badly) emulating a database-based backend. See :ref:`conf-amqp-result-backend`. +* filesystem + Use a shared directory to store the results. + See :ref:`conf-filesystem-result-backend`. + .. warning: While the AMQP result backend is very efficient, you must make sure @@ -1199,6 +1203,26 @@ Example configuration result_backend = 'amqp' result_expires = 18000 # 5 hours. +.. _conf-filesystem-result-backend: + +Filesystem backend settings +--------------------------- + +This backend can be configured using a file URL, for example:: + + CELERY_RESULT_BACKEND = 'file:///var/celery/results' + +The configured directory needs to be shared and writeable by all servers using +the backend. + +If you are trying Celery on a single system you can simply use the backend +without any further configuration. For larger clusters you could use NFS, +`GlusterFS`_, CIFS, `HDFS`_ (using FUSE) or any other filesystem. + +.. _`GlusterFS`: http://www.gluster.org/ +.. _`HDFS`: http://hadoop.apache.org/ + + .. _conf-messaging: Message Routing diff --git a/docs/internals/reference/celery.backends.filesystem.rst b/docs/internals/reference/celery.backends.filesystem.rst new file mode 100644 index 000000000..c5560d6b8 --- /dev/null +++ b/docs/internals/reference/celery.backends.filesystem.rst @@ -0,0 +1,11 @@ +========================================== + celery.backends.filesystem +========================================== + +.. contents:: + :local: +.. currentmodule:: celery.backends.filesystem + +.. automodule:: celery.backends.filesystem + :members: + :undoc-members: diff --git a/docs/internals/reference/index.rst b/docs/internals/reference/index.rst index 16897b9d0..34b513902 100644 --- a/docs/internals/reference/index.rst +++ b/docs/internals/reference/index.rst @@ -34,6 +34,7 @@ celery.backends.riak celery.backends.cassandra celery.backends.couchbase + celery.backends.filesystem celery.app.trace celery.app.annotations celery.app.routes From a2b6d183257dcbd75369ecd03d815668868173f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=B4she=20van=20der=20Sterre?= Date: Tue, 11 Feb 2014 11:58:14 +0100 Subject: [PATCH 0835/1103] Added myself to CONTRIBUTORS.txt and AUTHORS.txt --- CONTRIBUTORS.txt | 1 + docs/AUTHORS.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 1d4f33e49..b6af4d4b7 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -156,6 +156,7 @@ Antoine Legrand, 2014/01/09 Pepijn de Vos, 2014/01/15 Dan McGee, 2014/01/27 Paul Kilgo, 2014/01/28 +Môshe van der Sterre, 2014/01/31 Martin Davidsson, 2014/02/08 Chris Clark, 2014/02/20 Matthew Duggan, 2014/04/10 diff --git a/docs/AUTHORS.txt b/docs/AUTHORS.txt index 8ff42cbbb..2f88710de 100644 --- a/docs/AUTHORS.txt +++ b/docs/AUTHORS.txt @@ -106,6 +106,7 @@ Miguel Hernandez Martos Mikhail Gusarov Mikhail Korobov Mitar +Môshe van der Sterre Neil Chintomby Noah Kantrowitz Norman Richards From 667a68f704f54f1723bc2315e56f8fe185c35e2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=B4she=20van=20der=20Sterre?= Date: Sun, 21 Jun 2015 22:10:15 +0200 Subject: [PATCH 0836/1103] set() should also work with encoded strings --- celery/backends/filesystem.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/celery/backends/filesystem.py b/celery/backends/filesystem.py index d124f5711..1b70933e1 100644 --- a/celery/backends/filesystem.py +++ b/celery/backends/filesystem.py @@ -7,6 +7,8 @@ """ from __future__ import absolute_import +from kombu.utils.encoding import ensure_bytes + from celery.exceptions import ImproperlyConfigured from celery.backends.base import KeyValueStoreBackend from celery.utils import uuid @@ -81,7 +83,7 @@ def get(self, key): def set(self, key, value): with self.open(self._filename(key), 'wb') as outfile: - outfile.write(value) + outfile.write(ensure_bytes(value)) def mget(self, keys): for key in keys: From 70cfaecae4aaf36485deafdb9392f7cffa56880a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 25 Nov 2015 12:46:46 -0800 Subject: [PATCH 0837/1103] Transport can now decide which event loop to use --- celery/worker/components.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/worker/components.py b/celery/worker/components.py index 3a3c56927..1856710a4 100644 --- a/celery/worker/components.py +++ b/celery/worker/components.py @@ -73,9 +73,9 @@ def include_if(self, w): def create(self, w): w.hub = get_event_loop() if w.hub is None: + required_hub = getattr(w._conninfo, 'requires_hub', None) w.hub = set_event_loop(( - w._conninfo.requires_hub - if w._conninfo.requires_hub else _Hub)(w.timer)) + required_hub if required_hub else _Hub)(w.timer)) self._patch_thread_primitives(w) return self From 1532594c11d888576c3f42720bd43e48c4630304 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 25 Nov 2015 12:47:31 -0800 Subject: [PATCH 0838/1103] flakes --- celery/utils/functional.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index cf5b9df1e..80d0ac9de 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -21,7 +21,7 @@ dictfilter, lazy, maybe_evaluate, is_list, maybe_list, ) -from celery.five import UserDict, UserList, items, keys, range +from celery.five import UserDict, UserList, keys, range __all__ = ['LRUCache', 'is_list', 'maybe_list', 'memoize', 'mlazy', 'noop', 'first', 'firstmethod', 'chunks', 'padlist', 'mattrgetter', 'uniq', From 55d4a4c3eb15640fbfcda016ffac79fa0252286a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 25 Nov 2015 12:47:35 -0800 Subject: [PATCH 0839/1103] Use OptionGroup to separate worker arguments --- celery/bin/base.py | 42 ++++++----- celery/bin/beat.py | 22 +++--- celery/bin/celery.py | 2 +- celery/bin/celeryd_detach.py | 21 +++--- celery/bin/events.py | 24 +++--- celery/bin/worker.py | 138 +++++++++++++++++++++++++---------- 6 files changed, 154 insertions(+), 95 deletions(-) diff --git a/celery/bin/base.py b/celery/bin/base.py index 9ce89286a..a67e9aa6d 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -80,7 +80,9 @@ from collections import defaultdict from heapq import heappush from inspect import getargspec -from optparse import OptionParser, IndentedHelpFormatter, make_option as Option +from optparse import ( + OptionParser, OptionGroup, IndentedHelpFormatter, make_option as Option, +) from pprint import pformat from celery import VERSION_BANNER, Celery, maybe_patch_concurrency @@ -328,6 +330,9 @@ def get_options(self): """Get supported command-line options.""" return self.option_list + def prepare_arguments(self, parser): + pass + def expanduser(self, value): if isinstance(value, string_t): return os.path.expanduser(value) @@ -413,20 +418,21 @@ def parse_options(self, prog_name, arguments, command=None): return self.parser.parse_args(arguments) def create_parser(self, prog_name, command=None): - option_list = ( - self.preload_options + - self.get_options() + - tuple(self.app.user_options['preload']) - ) - return self.prepare_parser(self.Parser( + parser = self.Parser( prog=prog_name, usage=self.usage(command), version=self.version, epilog=self.epilog, formatter=HelpFormatter(), description=self.description, - option_list=option_list, - )) + ) + parser.option_list.extend(self.preload_options) + self.prepare_arguments(parser) + option_list = self.get_options() + if option_list: + parser.option_lisat.extend(option_list) + parser.option_list.extend(self.app.user_options['preload']) + return self.prepare_parser(parser) def prepare_parser(self, parser): docs = [self.parse_doc(doc) for doc in (self.doc, __doc__) if doc] @@ -662,12 +668,12 @@ def no_color(self, value): self._colored.enabled = not self._no_color -def daemon_options(default_pidfile=None, default_logfile=None): - return ( - Option('-f', '--logfile', default=default_logfile), - Option('--pidfile', default=default_pidfile), - Option('--uid', default=None), - Option('--gid', default=None), - Option('--umask', default=None), - Option('--executable', default=None), - ) +def daemon_options(parser, default_pidfile=None, default_logfile=None): + group = OptionGroup(parser, "Daemonization Options") + group.add_option('-f', '--logfile', default=default_logfile), + group.add_option('--pidfile', default=default_pidfile), + group.add_option('--uid', default=None), + group.add_option('--gid', default=None), + group.add_option('--umask', default=None), + group.add_option('--executable', default=None), + parser.add_option_group(group) diff --git a/celery/bin/beat.py b/celery/bin/beat.py index f203b3b47..911b5f0f9 100644 --- a/celery/bin/beat.py +++ b/celery/bin/beat.py @@ -44,7 +44,7 @@ from celery.platforms import detached, maybe_drop_privileges -from celery.bin.base import Command, Option, daemon_options +from celery.bin.base import Command, daemon_options __all__ = ['beat'] @@ -78,19 +78,15 @@ def run(self, detach=False, logfile=None, pidfile=None, uid=None, else: return beat().run() - def get_options(self): + def prepare_arguments(self, parser): c = self.app.conf - - return ( - (Option('--detach', action='store_true'), - Option('-s', '--schedule', - default=c.beat_schedule_filename), - Option('--max-interval', type='float'), - Option('-S', '--scheduler', dest='scheduler_cls'), - Option('-l', '--loglevel', default='WARN')) + - daemon_options(default_pidfile='celerybeat.pid') + - tuple(self.app.user_options['beat']) - ) + parser.add_option('--detach', action='store_true') + parser.add_option('-s', '--schedule', default=c.beat_schedule_filename) + parser.add_option('--max-interval', type='float') + parser.add_option('-S', '--scheduler', dest='scheduler_cls') + parser.add_option('-l', '--loglevel', default='WARN') + daemon_options(parser, default_pidfile='celerybeat.pid') + parser.option_list.extend(self.app.user_options['beat']) def main(app=None): diff --git a/celery/bin/celery.py b/celery/bin/celery.py index 4e08bbfde..91b788480 100644 --- a/celery/bin/celery.py +++ b/celery/bin/celery.py @@ -90,7 +90,7 @@ class multi(Command): respects_app_option = False def get_options(self): - return () + pass def run_from_argv(self, prog_name, argv, command=None): from celery.bin.multi import MultiTool diff --git a/celery/bin/celeryd_detach.py b/celery/bin/celeryd_detach.py index c845a72ff..8b3cc87ce 100644 --- a/celery/bin/celeryd_detach.py +++ b/celery/bin/celeryd_detach.py @@ -21,7 +21,7 @@ from celery.platforms import EX_FAILURE, detached from celery.utils.log import get_logger -from celery.bin.base import daemon_options, Option +from celery.bin.base import daemon_options __all__ = ['detached_celeryd', 'detach'] @@ -29,13 +29,6 @@ C_FAKEFORK = os.environ.get('C_FAKEFORK') -OPTION_LIST = daemon_options(default_pidfile='celeryd.pid') + ( - Option('--workdir', default=None, dest='working_directory'), - Option('--fake', - default=False, action='store_true', dest='fake', - help="Don't fork (for debugging purposes)"), -) - def detach(path, argv, logfile=None, pidfile=None, uid=None, gid=None, umask=None, working_directory=None, fake=False, app=None, @@ -114,7 +107,6 @@ def _process_short_opts(self, rargs, values): class detached_celeryd(object): - option_list = OPTION_LIST usage = '%prog [options] [celeryd options]' version = celery.VERSION_BANNER description = ('Detaches Celery worker nodes. See `celery worker --help` ' @@ -128,13 +120,13 @@ def __init__(self, app=None): def Parser(self, prog_name): return PartialOptionParser(prog=prog_name, - option_list=self.option_list, usage=self.usage, description=self.description, version=self.version) def parse_options(self, prog_name, argv): parser = self.Parser(prog_name) + self.prepare_arguments(parser) options, values = parser.parse_args(argv) if options.logfile: parser.leftovers.append('--logfile={0}'.format(options.logfile)) @@ -161,6 +153,15 @@ def execute_from_commandline(self, argv=None): **vars(options) )) + def prepare_arguments(self, parser): + daemon_options(parser, default_pidfile='celeryd.pid') + parser.add_option('--workdir', default=None, dest='working_directory') + parser.add_option( + '--fake', + default=False, action='store_true', dest='fake', + help="Don't fork (for debugging purposes)", + ) + def main(app=None): detached_celeryd(app).execute_from_commandline() diff --git a/celery/bin/events.py b/celery/bin/events.py index 8cc61b6d6..dc92dff0a 100644 --- a/celery/bin/events.py +++ b/celery/bin/events.py @@ -42,7 +42,7 @@ from functools import partial from celery.platforms import detached, set_process_title, strargv -from celery.bin.base import Command, Option, daemon_options +from celery.bin.base import Command, daemon_options __all__ = ['events'] @@ -117,18 +117,16 @@ def set_process_status(self, prog, info=''): info = '{0} {1}'.format(info, strargv(sys.argv)) return set_process_title(prog, info=info) - def get_options(self): - return ( - (Option('-d', '--dump', action='store_true'), - Option('-c', '--camera'), - Option('--detach', action='store_true'), - Option('-F', '--frequency', '--freq', - type='float', default=1.0), - Option('-r', '--maxrate'), - Option('-l', '--loglevel', default='INFO')) + - daemon_options(default_pidfile='celeryev.pid') + - tuple(self.app.user_options['events']) - ) + def prepare_arguments(self, parser): + parser.add_option('-d', '--dump', action='store_true') + parser.add_option('-c', '--camera') + parser.add_option('--detach', action='store_true') + parser.add_option('-F', '--frequency', '--freq', + type='float', default=1.0) + parser.add_option('-r', '--maxrate') + parser.add_option('-l', '--loglevel', default='INFO') + daemon_options(parser, default_pidfile='celeryev.pid') + parser.option_list.extend(self.app.user_options['events']) def main(): diff --git a/celery/bin/worker.py b/celery/bin/worker.py index 914957dcd..2d91f4a47 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -146,8 +146,10 @@ import sys +from optparse import OptionGroup + from celery import concurrency -from celery.bin.base import Command, Option, daemon_options +from celery.bin.base import Command, daemon_options from celery.bin.celeryd_detach import detached_celeryd from celery.five import string_t from celery.platforms import maybe_drop_privileges @@ -227,46 +229,102 @@ def with_pool_option(self, argv): # that may have to be loaded as early as possible. return (['-P'], ['--pool']) - def get_options(self): + def prepare_arguments(self, parser): conf = self.app.conf - return ( - Option('-c', '--concurrency', - default=conf.worker_concurrency, type='int'), - Option('-P', '--pool', default=conf.worker_pool, dest='pool_cls'), - Option('--purge', '--discard', default=False, action='store_true'), - Option('-l', '--loglevel', default='WARN'), - Option('-n', '--hostname'), - Option('-B', '--beat', action='store_true'), - Option('-s', '--schedule', dest='schedule_filename', - default=conf.beat_schedule_filename), - Option('--scheduler', dest='scheduler_cls'), - Option('-S', '--statedb', - default=conf.worker_state_db, dest='state_db'), - Option('-E', '--events', default=conf.worker_send_task_events, - action='store_true', dest='send_events'), - Option('--time-limit', type='float', dest='task_time_limit', - default=conf.task_time_limit), - Option('--soft-time-limit', dest='task_soft_time_limit', - default=conf.task_soft_time_limit, type='float'), - Option('--maxtasksperchild', dest='max_tasks_per_child', - default=conf.worker_max_tasks_per_child, type='int'), - Option('--prefetch-multiplier', dest='prefetch_multiplier', - default=conf.worker_prefetch_multiplier, type='int'), - Option('--maxmemperchild', dest='max_memory_per_child', - default=conf.worker_max_memory_per_child, type='int'), - Option('--queues', '-Q', default=[]), - Option('--exclude-queues', '-X', default=[]), - Option('--include', '-I', default=[]), - Option('--autoscale'), - Option('--autoreload', action='store_true'), - Option('--no-execv', action='store_true', default=False), - Option('--without-gossip', action='store_true', default=False), - Option('--without-mingle', action='store_true', default=False), - Option('--without-heartbeat', action='store_true', default=False), - Option('--heartbeat-interval', type='int'), - Option('-O', dest='optimization'), - Option('-D', '--detach', action='store_true'), - ) + daemon_options() + tuple(self.app.user_options['worker']) + + wopts = OptionGroup(parser, 'Worker Options') + wopts.add_option('-n', '--hostname') + wopts.add_option('-D', '--detach', action='store_true') + wopts.add_option( + '-S', '--statedb', + default=conf.worker_state_db, dest='state_db', + ) + wopts.add_option('-l', '--loglevel', default='WARN') + wopts.add_option('-O', dest='optimization') + wopts.add_option( + '--prefetch-multiplier', + dest='prefetch_multiplier', type='int', + default=conf.worker_prefetch_multiplier, + ) + parser.add_option_group(wopts) + + topts = OptionGroup(parser, 'Pool Options') + topts.add_option( + '-c', '--concurrency', + default=conf.worker_concurrency, type='int', + ) + topts.add_option( + '-P', '--pool', + default=conf.worker_pool, dest='pool_cls', + ) + topts.add_option( + '-E', '--events', + default=conf.worker_send_task_events, + action='store_true', dest='send_events', + ) + topts.add_option( + '--time-limit', + type='float', dest='task_time_limit', + default=conf.task_time_limit, + ) + topts.add_option( + '--soft-time-limit', + dest='task_soft_time_limit', type='float', + default=conf.task_soft_time_limit, + ) + topts.add_option( + '--maxtasksperchild', + dest='max_tasks_per_child', type='int', + default=conf.worker_max_tasks_per_child, + ) + topts.add_option( + '--maxmemperchild', + dest='max_memory_per_child', type='int', + default=conf.worker_max_memory_per_child, + ) + parser.add_option_group(topts) + + qopts = OptionGroup(parser, 'Queue Options') + qopts.add_option( + '--purge', '--discard', + default=False, action='store_true', + ) + qopts.add_option('--queues', '-Q', default=[]) + qopts.add_option('--exclude-queues', '-X', default=[]) + qopts.add_option('--include', '-I', default=[]) + parser.add_option_group(qopts) + + fopts = OptionGroup(parser, 'Features') + fopts.add_option('--autoscale') + fopts.add_option('--autoreload', action='store_true') + fopts.add_option( + '--without-gossip', action='store_true', default=False, + ) + fopts.add_option( + '--without-mingle', action='store_true', default=False, + ) + fopts.add_option( + '--without-heartbeat', action='store_true', default=False, + ) + fopts.add_option('--heartbeat-interval', type='int') + parser.add_option_group(fopts) + + daemon_options(parser) + + bopts = OptionGroup(parser, 'Embedded Beat Options') + bopts.add_option('-B', '--beat', action='store_true') + bopts.add_option( + '-s', '--schedule', dest='schedule_filename', + default=conf.beat_schedule_filename, + ) + bopts.add_option('--scheduler', dest='scheduler_cls') + parser.add_option_group(bopts) + + user_options = self.app.user_options['worker'] + if user_options: + uopts = OptionGroup(parser, 'User Options') + uopts.options_list.extend(user_options) + parser.add_option_group(uopts) def main(app=None): From 37c081ee07094ea31c0848f4c47ddb123bfe6b76 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 25 Nov 2015 13:11:30 -0800 Subject: [PATCH 0840/1103] Tests passing --- celery/app/defaults.py | 2 ++ celery/backends/filesystem.py | 34 ++++++++++++++---------- celery/bin/celeryd_detach.py | 20 +++++++++----- celery/tests/backends/test_filesystem.py | 12 +++++---- celery/tests/bin/test_celery.py | 2 +- celery/tests/bin/test_celeryd_detach.py | 4 ++- celery/tests/bin/test_events.py | 2 +- 7 files changed, 47 insertions(+), 29 deletions(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index a4d158d20..ae40b2ae5 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -185,6 +185,8 @@ def __repr__(self): ), persistent=Option(None, type='bool'), serializer=Option('json'), + + fspath=Option(None), ), riak=Namespace( __old__=old_ns('celery_riak'), diff --git a/celery/backends/filesystem.py b/celery/backends/filesystem.py index 1b70933e1..5368de4db 100644 --- a/celery/backends/filesystem.py +++ b/celery/backends/filesystem.py @@ -24,18 +24,26 @@ FileNotFoundError = IOError IsADirectoryError = IOError +E_PATH_INVALID = """\ +The configured path for the Filesystem backend does not +work correctly, please make sure that it exists and has +the correct permissions.\ +""" + class FilesystemBackend(KeyValueStoreBackend): + def __init__(self, url=None, open=open, unlink=os.unlink, sep=os.sep, encoding=default_encoding, *args, **kwargs): """Initialize the filesystem backend. Keyword arguments (in addition to those of KeyValueStoreBackend): - url -- URL to the directory we should use - open -- open function to use when opening files - unlink -- unlink function to use when deleting files - sep -- directory seperator (to join the directory with the key) - encoding -- encoding used on the filesystem + + :param url: URL to the directory we should use + :param open: open function to use when opening files + :param unlink: unlink function to use when deleting files + :param sep: directory seperator (to join the directory with the key) + :param encoding: encoding used on the filesystem """ @@ -55,10 +63,11 @@ def __init__(self, url=None, open=open, unlink=os.unlink, sep=os.sep, def _find_path(self, url): if url is not None and url.startswith('file:///'): return url[7:] - if hasattr(self.app.conf, 'CELERY_RESULT_FSPATH'): - return self.app.conf.CELERY_RESULT_FSPATH - raise ImproperlyConfigured( - 'You need to configure a path for the Filesystem backend') + path = self.app.conf.result_fspath + if not path: + raise ImproperlyConfigured( + 'You need to configure a path for the Filesystem backend') + return path def _do_directory_test(self, key): try: @@ -66,10 +75,7 @@ def _do_directory_test(self, key): assert self.get(key) == b'test value' self.delete(key) except IOError: - raise ImproperlyConfigured( - 'The configured path for the Filesystem backend does not ' - 'work correctly, please make sure that it exists and has ' - 'the correct permissions.') + raise ImproperlyConfigured(E_PATH_INVALID) def _filename(self, key): return self.sep.join((self.path, key)) @@ -79,7 +85,7 @@ def get(self, key): with self.open(self._filename(key), 'rb') as infile: return infile.read() except FileNotFoundError: - return None + pass def set(self, key, value): with self.open(self._filename(key), 'wb') as outfile: diff --git a/celery/bin/celeryd_detach.py b/celery/bin/celeryd_detach.py index 8b3cc87ce..66ff8a345 100644 --- a/celery/bin/celeryd_detach.py +++ b/celery/bin/celeryd_detach.py @@ -56,6 +56,9 @@ def __init__(self, *args, **kwargs): self.leftovers = [] OptionParser.__init__(self, *args, **kwargs) + def add_option_group(self, group): + self.option_list.extend(group.option_list) + def _process_long_opt(self, rargs, values): arg = rargs.pop(0) @@ -118,15 +121,18 @@ class detached_celeryd(object): def __init__(self, app=None): self.app = app - def Parser(self, prog_name): - return PartialOptionParser(prog=prog_name, - usage=self.usage, - description=self.description, - version=self.version) + def create_parser(self, prog_name): + p = PartialOptionParser( + prog=prog_name, + usage=self.usage, + description=self.description, + version=self.version, + ) + self.prepare_arguments(p) + return p def parse_options(self, prog_name, argv): - parser = self.Parser(prog_name) - self.prepare_arguments(parser) + parser = self.create_parser(prog_name) options, values = parser.parse_args(argv) if options.logfile: parser.leftovers.append('--logfile={0}'.format(options.logfile)) diff --git a/celery/tests/backends/test_filesystem.py b/celery/tests/backends/test_filesystem.py index a1a5e0231..87639da07 100644 --- a/celery/tests/backends/test_filesystem.py +++ b/celery/tests/backends/test_filesystem.py @@ -1,18 +1,20 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import os +import shutil +import tempfile + from celery import states -from celery.tests.case import AppCase from celery.backends.filesystem import FilesystemBackend from celery.exceptions import ImproperlyConfigured from celery.utils import uuid -import os -import shutil -import tempfile +from celery.tests.case import AppCase class test_FilesystemBackend(AppCase): + def setup(self): self.directory = tempfile.mkdtemp() self.url = 'file://' + self.directory @@ -26,7 +28,7 @@ def test_a_path_is_required(self): FilesystemBackend(app=self.app) def test_a_path_in_app_conf(self): - self.app.conf.CELERY_RESULT_FSPATH = self.url[7:] + self.app.conf.result_fspath = self.url[7:] tb = FilesystemBackend(app=self.app) self.assertEqual(tb.path, self.path) diff --git a/celery/tests/bin/test_celery.py b/celery/tests/bin/test_celery.py index ec6de724b..750f3f51a 100644 --- a/celery/tests/bin/test_celery.py +++ b/celery/tests/bin/test_celery.py @@ -575,7 +575,7 @@ def test_cancel_consumer(self): class test_multi(AppCase): def test_get_options(self): - self.assertTupleEqual(multi(app=self.app).get_options(), ()) + self.assertIsNone(multi(app=self.app).get_options()) def test_run_from_argv(self): with patch('celery.bin.multi.MultiTool') as MultiTool: diff --git a/celery/tests/bin/test_celeryd_detach.py b/celery/tests/bin/test_celeryd_detach.py index f818777f0..f12e445b2 100644 --- a/celery/tests/bin/test_celeryd_detach.py +++ b/celery/tests/bin/test_celeryd_detach.py @@ -56,11 +56,13 @@ class test_PartialOptionParser(AppCase): def test_parser(self): x = detached_celeryd(self.app) - p = x.Parser('celeryd_detach') + p = x.create_parser('celeryd_detach') options, values = p.parse_args([ '--logfile=foo', '--fake', '--enable', 'a', 'b', '-c1', '-d', '2', ]) + print(p.option_list) + print('O: %r V: %r' % (vars(options), values)) self.assertEqual(options.logfile, 'foo') self.assertEqual(values, ['a', 'b']) self.assertEqual(p.leftovers, ['--enable', '-c1', '-d', '2']) diff --git a/celery/tests/bin/test_events.py b/celery/tests/bin/test_events.py index 80e17609d..f49f6f7c3 100644 --- a/celery/tests/bin/test_events.py +++ b/celery/tests/bin/test_events.py @@ -64,7 +64,7 @@ def test_run_cam_detached(self, detached, evcam): self.assertTrue(evcam.called) def test_get_options(self): - self.assertTrue(self.ev.get_options()) + self.assertFalse(self.ev.get_options()) @_old_patch('celery.bin.events', 'events', MockCommand) def test_main(self): From 11ac5fbdc3905ef1205f359e062b63ae1072433b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 25 Nov 2015 15:57:59 -0800 Subject: [PATCH 0841/1103] [docs] Tasks: Prominently display bound task and base class information --- docs/userguide/tasks.rst | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index ca074c685..17e4008ce 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -66,7 +66,6 @@ these can be specified as arguments to the decorator: User.objects.create(username=username, password=password) - .. sidebar:: How do I import the task decorator? And what is "app"? The task decorator is available on your :class:`@Celery` application instance, @@ -98,6 +97,42 @@ these can be specified as arguments to the decorator: def add(x, y): return x + y +Bound tasks +----------- + +A task being bound means the first argument to the task will always +be the task instance (``self``), just like Python bound methods: + +.. code-block:: python + + logger = get_task_logger(__name__) + + @task(bind=True) + def add(self, x, y): + logger.info(self.request.id) + +Bound tasks are needed for retries (using :meth:`@Task.retry`), for +accessing information about the current task request, and for any additional +functionality you add to custom task base classes. + +Task inheritance +---------------- + +The ``base`` argument to the task decorator specifies the base class of the task: + +.. code-block:: python + + import celery + + class MyTask(celery.Task): + + def on_failure(self, exc, task_id, args, kwargs, einfo): + print('{0!r} failed: {1!r}'.format(task_id, exc) + + @task(base=MyTask) + def add(x, y): + raise KeyError() + .. _task-names: Names From b225f93a656d13721b570d0a06775cc4358a7d4e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 25 Nov 2015 16:14:19 -0800 Subject: [PATCH 0842/1103] ISO8601 parser now handles dates without hour/minute/sec. Closes #2062 --- celery/utils/iso8601.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/celery/utils/iso8601.py b/celery/utils/iso8601.py index c951cf6ea..9f9ba9a3a 100644 --- a/celery/utils/iso8601.py +++ b/celery/utils/iso8601.py @@ -69,9 +69,9 @@ def parse_iso8601(datestring): hours = -hours minutes = -minutes tz = FixedOffset(minutes + hours * 60) - frac = groups['fraction'] or 0 return datetime( - int(groups['year']), int(groups['month']), int(groups['day']), - int(groups['hour']), int(groups['minute']), int(groups['second']), - int(frac), tz + int(groups['year']), int(groups['month']), + int(groups['day']), int(groups['hour'] or 0), + int(groups['minute'] or 0), int(groups['second'] or 0), + int(groups['fraction'] or 0), tz ) From 999ad06fb3e7c22fbac15d54f34baf67b9ff9c36 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 25 Nov 2015 17:03:47 -0800 Subject: [PATCH 0843/1103] Update lock example in task-cookbook to not release timed out lock. Closes #2926 --- docs/tutorials/task-cookbook.rst | 44 ++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/docs/tutorials/task-cookbook.rst b/docs/tutorials/task-cookbook.rst index a4c01868f..d5bde5f26 100644 --- a/docs/tutorials/task-cookbook.rst +++ b/docs/tutorials/task-cookbook.rst @@ -23,20 +23,41 @@ a Django model called `Feed`. We ensure that it's not possible for two or more workers to import the same feed at the same time by setting a cache key consisting of the MD5 checksum of the feed URL. -The cache key expires after some time in case something unexpected happens -(you never know, right?) +The cache key expires after some time in case something unexpected happens, +and something always will... + +For this reason your tasks runtime should not exceeed the timeout. + .. code-block:: python from celery import task + from celery.five import monotonic from celery.utils.log import get_task_logger + from contextlib import contextmanager from django.core.cache import cache from hashlib import md5 from djangofeeds.models import Feed logger = get_task_logger(__name__) - LOCK_EXPIRE = 60 * 5 # Lock expires in 5 minutes + LOCK_EXPIRE = 60 * 10 # Lock expires in 10 minutes + + @contextmanager + def memcache_lock(lock_id, oid): + timeout_at = monotonic() + LOCK_EXPIRE - 3 + # cache.add fails if the key already exists + status = cache.add(lock_id, oid, LOCK_EXPIRE) + try: + yield status + finally: + # memcache delete is very slow, but we have to use it to take + # advantage of using add() for atomic locking + if monotonic() < timeout_at: + # do not release the lock if we exceeded the timeout + # to lessen the chance of releasing an expired lock + # owned by someone else. + cache.delete(lock_id) @task(bind=True) def import_feed(self, feed_url): @@ -44,20 +65,9 @@ The cache key expires after some time in case something unexpected happens # of the feed URL. feed_url_hexdigest = md5(feed_url).hexdigest() lock_id = '{0}-lock-{1}'.format(self.name, feed_url_hexdigest) - - # cache.add fails if the key already exists - acquire_lock = lambda: cache.add(lock_id, 'true', LOCK_EXPIRE) - # memcache delete is very slow, but we have to use it to take - # advantage of using add() for atomic locking - release_lock = lambda: cache.delete(lock_id) - logger.debug('Importing feed: %s', feed_url) - if acquire_lock(): - try: - feed = Feed.objects.import_feed(feed_url) - finally: - release_lock() - return feed.url - + with memcache_lock(lock_id, self.app.oid) as acquired: + if acquired: + return Feed.objects.import_feed(feed_url).url logger.debug( 'Feed %s is already being imported by another worker', feed_url) From ed863f219b7fc0e872474355ab4a53c6610ddc2b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 25 Nov 2015 17:06:54 -0800 Subject: [PATCH 0844/1103] Solo pool: Hardcode limit/num_processes to 1 for correct prefetch count calculations. Closes #2925 --- celery/concurrency/solo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/concurrency/solo.py b/celery/concurrency/solo.py index a83f46219..434071908 100644 --- a/celery/concurrency/solo.py +++ b/celery/concurrency/solo.py @@ -22,6 +22,7 @@ class TaskPool(BasePool): def __init__(self, *args, **kwargs): super(TaskPool, self).__init__(*args, **kwargs) self.on_apply = apply_target + self.limit = 1 def _get_info(self): return {'max-concurrency': 1, From 66e94b8abbd913ffd0a2a6a96d283ec367152345 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 25 Nov 2015 17:40:42 -0800 Subject: [PATCH 0845/1103] Fixes bug with argument parsing in master --- celery/bin/base.py | 15 +++++++++------ celery/bin/beat.py | 2 +- celery/bin/celeryd_detach.py | 3 --- celery/bin/events.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/celery/bin/base.py b/celery/bin/base.py index a67e9aa6d..bc0095045 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -426,12 +426,15 @@ def create_parser(self, prog_name, command=None): formatter=HelpFormatter(), description=self.description, ) - parser.option_list.extend(self.preload_options) - self.prepare_arguments(parser) - option_list = self.get_options() - if option_list: - parser.option_lisat.extend(option_list) - parser.option_list.extend(self.app.user_options['preload']) + parser.add_options(self.preload_options) + for typ_ in reversed(type(self).mro()): + try: + prepare_arguments = typ_.prepare_arguments + except AttributeError: + continue + prepare_arguments(self, parser) + parser.add_options(self.get_options() or ()) + parser.add_options(self.app.user_options['preload']) return self.prepare_parser(parser) def prepare_parser(self, parser): diff --git a/celery/bin/beat.py b/celery/bin/beat.py index 911b5f0f9..ebc1cbedc 100644 --- a/celery/bin/beat.py +++ b/celery/bin/beat.py @@ -86,7 +86,7 @@ def prepare_arguments(self, parser): parser.add_option('-S', '--scheduler', dest='scheduler_cls') parser.add_option('-l', '--loglevel', default='WARN') daemon_options(parser, default_pidfile='celerybeat.pid') - parser.option_list.extend(self.app.user_options['beat']) + parser.add_options(self.app.user_options['beat']) def main(app=None): diff --git a/celery/bin/celeryd_detach.py b/celery/bin/celeryd_detach.py index 66ff8a345..43fd5c665 100644 --- a/celery/bin/celeryd_detach.py +++ b/celery/bin/celeryd_detach.py @@ -56,9 +56,6 @@ def __init__(self, *args, **kwargs): self.leftovers = [] OptionParser.__init__(self, *args, **kwargs) - def add_option_group(self, group): - self.option_list.extend(group.option_list) - def _process_long_opt(self, rargs, values): arg = rargs.pop(0) diff --git a/celery/bin/events.py b/celery/bin/events.py index dc92dff0a..4fa7eeb01 100644 --- a/celery/bin/events.py +++ b/celery/bin/events.py @@ -126,7 +126,7 @@ def prepare_arguments(self, parser): parser.add_option('-r', '--maxrate') parser.add_option('-l', '--loglevel', default='INFO') daemon_options(parser, default_pidfile='celeryev.pid') - parser.option_list.extend(self.app.user_options['events']) + parser.add_options(self.app.user_options['events']) def main(): From 9d466a27e604d9e98e49e8c90f44ffb1624763a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aitor=20G=C3=B3mez-Goiri?= Date: Thu, 26 Nov 2015 14:37:31 +0000 Subject: [PATCH 0846/1103] Fixing MaxRetriesExceededError's mentions MaxRetriesExceededError appeared as MaxRetriesExceeded in the documentation. --- docs/userguide/tasks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 17e4008ce..dc8e79ce6 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -497,7 +497,7 @@ but this will not happen if: - An ``exc`` argument was not given. - In this case the :exc:`~@MaxRetriesExceeded` + In this case the :exc:`~@MaxRetriesExceededError` exception will be raised. - There is no current exception @@ -615,7 +615,7 @@ General .. attribute:: Task.max_retries The maximum number of attempted retries before giving up. - If the number of retries exceeds this value a :exc:`~@MaxRetriesExceeded` + If the number of retries exceeds this value a :exc:`~@MaxRetriesExceededError` exception will be raised. *NOTE:* You have to call :meth:`~@Task.retry` manually, as it will not automatically retry on exception.. From c9cac002374efae48b322e0a02c4079ae0b890a7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 27 Nov 2015 11:27:04 -0800 Subject: [PATCH 0847/1103] cosmetics --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 39b043f9c..3c2098230 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,6 +3,7 @@ branch = 1 cover_pylib = 0 include=*celery/* omit = celery.tests.* + [report] omit = */python?.?/* From 2055cbd056f4d2822e0e88b22e36cfca363952a6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 27 Nov 2015 11:33:11 -0800 Subject: [PATCH 0848/1103] AsyncResult.get now supports the on_interval argument --- celery/backends/amqp.py | 5 ++++- celery/backends/base.py | 4 +++- celery/result.py | 25 ++++++++++++++----------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index 16cc9dd7b..ad7cdf226 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -231,7 +231,8 @@ def consume(self, task_id, timeout=None, no_ack=True, on_interval=None): def _many_bindings(self, ids): return [self._create_binding(task_id) for task_id in ids] - def get_many(self, task_ids, timeout=None, no_ack=True, on_message=None, + def get_many(self, task_ids, timeout=None, no_ack=True, + on_message=None, on_interval=None, now=monotonic, getfields=itemgetter('status', 'task_id'), READY_STATES=states.READY_STATES, PROPAGATE_STATES=states.PROPAGATE_STATES, **kwargs): @@ -276,6 +277,8 @@ def _on_message(message): ids.discard(task_id) push_cache(task_id, state) yield task_id, state + if on_interval: + on_interval() def reload_task_result(self, task_id): raise NotImplementedError( diff --git a/celery/backends/base.py b/celery/backends/base.py index 2a2cb613c..c9ecacc2e 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -475,7 +475,7 @@ def _mget_to_results(self, values, keys): } def get_many(self, task_ids, timeout=None, interval=0.5, no_ack=True, - on_message=None, + on_message=None, on_interval=None, READY_STATES=states.READY_STATES): interval = 0.5 if interval is None else interval ids = task_ids if isinstance(task_ids, set) else set(task_ids) @@ -505,6 +505,8 @@ def get_many(self, task_ids, timeout=None, interval=0.5, no_ack=True, yield bytes_to_str(key), value if timeout and iterations * interval >= timeout: raise TimeoutError('Operation timed out ({0})'.format(timeout)) + if on_interval: + on_interval() time.sleep(interval) # don't busy loop. iterations += 1 diff --git a/celery/result.py b/celery/result.py index 42ff01f64..472511b7f 100644 --- a/celery/result.py +++ b/celery/result.py @@ -14,6 +14,7 @@ from contextlib import contextmanager from copy import copy +from amqp import promise from kombu.utils import cached_property from . import current_app @@ -118,7 +119,7 @@ def revoke(self, connection=None, terminate=False, signal=None, reply=wait, timeout=timeout) def get(self, timeout=None, propagate=True, interval=0.5, - no_ack=True, follow_parents=True, callback=None, + no_ack=True, follow_parents=True, callback=None, on_interval=None, EXCEPTION_STATES=states.EXCEPTION_STATES, PROPAGATE_STATES=states.PROPAGATE_STATES): """Wait until task is ready, and return its result. @@ -149,10 +150,12 @@ def get(self, timeout=None, propagate=True, interval=0.5, """ assert_will_not_block() - on_interval = None + _on_interval = promise() if follow_parents and propagate and self.parent: - on_interval = self._maybe_reraise_parent_error - on_interval() + on_interval = promise(self._maybe_reraise_parent_error) + self._maybe_reraise_parent_error() + if on_interval: + _on_interval.then(on_interval) if self._cache: if propagate: @@ -162,7 +165,7 @@ def get(self, timeout=None, propagate=True, interval=0.5, meta = self.backend.wait_for( self.id, timeout=timeout, interval=interval, - on_interval=on_interval, + on_interval=_on_interval, no_ack=no_ack, ) if meta: @@ -579,7 +582,7 @@ def get(self, timeout=None, propagate=True, interval=0.5, ) def join(self, timeout=None, propagate=True, interval=0.5, - callback=None, no_ack=True, on_message=None): + callback=None, no_ack=True, on_message=None, on_interval=None): """Gathers the results of all tasks as a list in order. .. note:: @@ -644,7 +647,7 @@ def join(self, timeout=None, propagate=True, interval=0.5, raise TimeoutError('join operation timed out') value = result.get( timeout=remaining, propagate=propagate, - interval=interval, no_ack=no_ack, + interval=interval, no_ack=no_ack, on_interval=on_interval, ) if callback: callback(result.id, value) @@ -653,7 +656,7 @@ def join(self, timeout=None, propagate=True, interval=0.5, return results def iter_native(self, timeout=None, interval=0.5, no_ack=True, - on_message=None): + on_message=None, on_interval=None): """Backend optimized version of :meth:`iterate`. .. versionadded:: 2.2 @@ -671,12 +674,12 @@ def iter_native(self, timeout=None, interval=0.5, no_ack=True, return self.backend.get_many( {r.id for r in results}, timeout=timeout, interval=interval, no_ack=no_ack, - on_message=on_message, + on_message=on_message, on_interval=on_interval, ) def join_native(self, timeout=None, propagate=True, interval=0.5, callback=None, no_ack=True, - on_message=None): + on_message=None, on_interval=None): """Backend optimized version of :meth:`join`. .. versionadded:: 2.2 @@ -694,7 +697,7 @@ def join_native(self, timeout=None, propagate=True, } acc = None if callback else [None for _ in range(len(self))] for task_id, meta in self.iter_native(timeout, interval, no_ack, - on_message): + on_message, on_interval): value = meta['result'] if propagate and meta['status'] in states.PROPAGATE_STATES: raise value From 5ed5541be2d1797570bb29143fe67d6d05db55c4 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 27 Nov 2015 11:57:09 -0800 Subject: [PATCH 0849/1103] [canvas] Fixes bug with chord upgrade in master. --- celery/canvas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index bc45c65b2..082065b94 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -488,11 +488,11 @@ def prepare_steps(self, args, tasks, # signature instead of a group. tasks.pop() results.pop() - prev_res = prev_prev_res task = chord( task, body=prev_task, task_id=prev_res.task_id, root_id=root_id, app=app, ) + prev_res = prev_prev_res if is_last_task: # chain(task_id=id) means task id is set for the last task # in the chain. If the chord is part of a chord/group From fe60cab15ad71c8ef096a67a6db1419c448fb429 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 27 Nov 2015 11:57:39 -0800 Subject: [PATCH 0850/1103] [canvas] group | task now upgrades to chord early (Issue #2922) --- celery/canvas.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/celery/canvas.py b/celery/canvas.py index 082065b94..c4b47acd0 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -295,6 +295,8 @@ def flatten_links(self): def __or__(self, other): if isinstance(other, group): other = maybe_unroll_group(other) + if isinstance(self, group): + return chord(self, body=other, app=self._app) if not isinstance(self, chain) and isinstance(other, chain): return chain((self,) + other.tasks, app=self._app) elif isinstance(other, chain): From 649f61682d8ab8ea647da2b67fc0d0fc7cf06412 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 27 Nov 2015 12:47:24 -0800 Subject: [PATCH 0851/1103] Redis: Fixes problem with nested chords where parent chord overwrites chord message field. Closes #2922 --- celery/canvas.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index c4b47acd0..07557ed89 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -717,11 +717,12 @@ def _prepared(self, tasks, partial_args, group_id, root_id, app, dict=dict, yield task, task.freeze(group_id=group_id, root_id=root_id) def _apply_tasks(self, tasks, producer=None, app=None, - add_to_parent=None, **options): + add_to_parent=None, chord=None, **options): app = app or self.app with app.producer_or_acquire(producer) as producer: for sig, res in tasks: sig.apply_async(producer=producer, add_to_parent=False, + chord=sig.options.get('chord') or chord, **options) yield res # <-- r.parent, etc set in the frozen result. @@ -868,9 +869,11 @@ def freeze(self, _id=None, group_id=None, chord=None, root_id=None, parent_id=None): if not isinstance(self.tasks, group): self.tasks = group(self.tasks) - self.tasks.freeze(parent_id=parent_id, root_id=root_id) + bodyres = self.body.freeze(_id, parent_id=self.id, root_id=root_id) + self.tasks.freeze(parent_id=parent_id, root_id=root_id, chord=self.body) self.id = self.tasks.id - return self.body.freeze(_id, parent_id=self.id, root_id=root_id) + self.body.set_parent_id(self.id) + return bodyres def set_parent_id(self, parent_id): tasks = self.tasks From 37afd26c317089be103e078011a9cc4c70116858 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 27 Nov 2015 14:26:57 -0800 Subject: [PATCH 0852/1103] Use anon routing for autoqueues (skips routing table). Closes #2484 Issue celery/kombu#236 --- celery/app/amqp.py | 11 +++++++---- funtests/stress/stress/templates.py | 2 -- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 8ea5455a1..2089d7027 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -488,8 +488,11 @@ def send_task_message(producer, name, message, except AttributeError: pass delivery_mode = delivery_mode or default_delivery_mode - exchange = exchange or queue.exchange.name - routing_key = routing_key or queue.routing_key + if not exchange and not routing_key: + exchange, routing_key = '', qname + else: + exchange = exchange or queue.exchange.name or default_exchange + routing_key = routing_key or queue.routing_key or default_rkey if declare is None and queue and not isinstance(queue, Broadcast): declare = [queue] @@ -507,8 +510,8 @@ def send_task_message(producer, name, message, ) ret = producer.publish( body, - exchange=exchange or default_exchange, - routing_key=routing_key or default_rkey, + exchange=exchange, + routing_key=routing_key, serializer=serializer or default_serializer, compression=compression or default_compressor, retry=retry, retry_policy=_rp, diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index e04a15f8b..4a6416e2d 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -61,8 +61,6 @@ class default(object): task_default_queue = CSTRESS_QUEUE task_queues = [ Queue(CSTRESS_QUEUE, - exchange=Exchange(CSTRESS_QUEUE), - routing_key=CSTRESS_QUEUE, durable=not CSTRESS_TRANS, no_ack=CSTRESS_TRANS), ] From 3ca51a88436065c0e17c84ce22fd185a331be84e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 27 Nov 2015 15:38:25 -0800 Subject: [PATCH 0853/1103] Tests passing --- celery/tests/app/test_amqp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/tests/app/test_amqp.py b/celery/tests/app/test_amqp.py index 06104e26b..79fda1e97 100644 --- a/celery/tests/app/test_amqp.py +++ b/celery/tests/app/test_amqp.py @@ -222,7 +222,7 @@ def test_send_task_message__queue_string(self): ) kwargs = prod.publish.call_args[1] self.assertEqual(kwargs['routing_key'], 'foo') - self.assertEqual(kwargs['exchange'], 'foo') + self.assertEqual(kwargs['exchange'], '') def test_send_event_exchange_string(self): evd = Mock(name="evd") From 1b958ef2a0f6122c4db482159211b30a1b7df93e Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Mon, 30 Nov 2015 15:13:26 -0500 Subject: [PATCH 0854/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index b6af4d4b7..aa2ce705f 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -199,3 +199,4 @@ Krzysztof Bujniewicz, 2015/10/21 Sukrit Khera, 2015/10/26 Dave Smith, 2015/10/27 Dennis Brakhane, 2015/10/30 +Chris Harris, 2015/11/27 From 473a90434494a5f2dc5d1603784ae5dea5ab819b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 30 Nov 2015 14:16:35 -0800 Subject: [PATCH 0855/1103] Fixes wrong link to license detail. Closes #2890 --- docs/copyright.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/copyright.rst b/docs/copyright.rst index a81d5cb8d..7a78c9c27 100644 --- a/docs/copyright.rst +++ b/docs/copyright.rst @@ -12,7 +12,7 @@ Copyright |copy| 2009-2015, Ask Solem. All rights reserved. This material may be copied or distributed only subject to the terms and conditions set forth in the `Creative Commons Attribution-ShareAlike 4.0 International` -`_ license. +`_ license. You may share and adapt the material, even for commercial purposes, but you must give the original author credit. From f96234022b1105d90b858ab1f15814251f922ffd Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 1 Dec 2015 11:21:28 -0800 Subject: [PATCH 0856/1103] removes debugging statement --- funtests/stress/stress/app.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/funtests/stress/stress/app.py b/funtests/stress/stress/app.py index ea10c03a5..ac35f0cfe 100644 --- a/funtests/stress/stress/app.py +++ b/funtests/stress/stress/app.py @@ -58,8 +58,6 @@ def _marker(s, sep='-'): @app.task def add(x, y): - import locale - print(locale.getdefaultlocale()) return x + y From 93fb98f0897065bcb878c8e5f714464037813032 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 1 Dec 2015 16:25:35 -0800 Subject: [PATCH 0857/1103] Batches example missing passing request to mark_as_done. Issue #2861 --- celery/contrib/batches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/contrib/batches.py b/celery/contrib/batches.py index 6a0858b08..c1b1b4c9d 100644 --- a/celery/contrib/batches.py +++ b/celery/contrib/batches.py @@ -57,7 +57,7 @@ def wot_api(requests): ) # use mark_as_done to manually return response data for response, request in zip(reponses, requests): - app.backend.mark_as_done(request.id, response) + app.backend.mark_as_done(request.id, response, request) def wot_api_real(urls): From c7d89bd7f18ad8bbbc73b0f9a4b8c0b3729aadd0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 1 Dec 2015 16:35:11 -0800 Subject: [PATCH 0858/1103] Adds appveyor --- appveyor.yml | 53 +++++++++++++++++ extra/appveyor/install.ps1 | 85 ++++++++++++++++++++++++++++ extra/appveyor/run_with_compiler.cmd | 47 +++++++++++++++ 3 files changed, 185 insertions(+) create mode 100644 appveyor.yml create mode 100644 extra/appveyor/install.ps1 create mode 100644 extra/appveyor/run_with_compiler.cmd diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000..867715502 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,53 @@ +environment: + + global: + # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the + # /E:ON and /V:ON options are not enabled in the batch script intepreter + # See: http://stackoverflow.com/a/13751649/163740 + WITH_COMPILER: "cmd /E:ON /V:ON /C .\\extra\\appveyor\\run_with_compiler.cmd" + + matrix: + + # Pre-installed Python versions, which Appveyor may upgrade to + # a later point release. + # See: http://www.appveyor.com/docs/installed-software#python + + - PYTHON: "C:\\Python27" + PYTHON_VERSION: "2.7.x" + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python34" + PYTHON_VERSION: "3.4.x" + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python27-x64" + PYTHON_VERSION: "2.7.x" + PYTHON_ARCH: "64" + WINDOWS_SDK_VERSION: "v7.0" + + - PYTHON: "C:\\Python34-x64" + PYTHON_VERSION: "3.4.x" + PYTHON_ARCH: "64" + WINDOWS_SDK_VERSION: "v7.1" + + +init: + - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" + +install: + - "powershell extra\\appveyor\\install.ps1" + - "%PYTHON%/Scripts/pip.exe install -U setuptools" + +build: off + +test_script: + - "%WITH_COMPILER% %PYTHON%/python setup.py test" + +after_test: + - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" + +artifacts: + - path: dist\* + +#on_success: +# - TODO: upload the content of dist/*.whl to a public wheelhouse diff --git a/extra/appveyor/install.ps1 b/extra/appveyor/install.ps1 new file mode 100644 index 000000000..3f0562825 --- /dev/null +++ b/extra/appveyor/install.ps1 @@ -0,0 +1,85 @@ +# Sample script to install Python and pip under Windows +# Authors: Olivier Grisel and Kyle Kastner +# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ + +$BASE_URL = "https://www.python.org/ftp/python/" +$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" +$GET_PIP_PATH = "C:\get-pip.py" + + +function DownloadPython ($python_version, $platform_suffix) { + $webclient = New-Object System.Net.WebClient + $filename = "python-" + $python_version + $platform_suffix + ".msi" + $url = $BASE_URL + $python_version + "/" + $filename + + $basedir = $pwd.Path + "\" + $filepath = $basedir + $filename + if (Test-Path $filename) { + Write-Host "Reusing" $filepath + return $filepath + } + + # Download and retry up to 5 times in case of network transient errors. + Write-Host "Downloading" $filename "from" $url + $retry_attempts = 3 + for($i=0; $i -lt $retry_attempts; $i++){ + try { + $webclient.DownloadFile($url, $filepath) + break + } + Catch [Exception]{ + Start-Sleep 1 + } + } + Write-Host "File saved at" $filepath + return $filepath +} + + +function InstallPython ($python_version, $architecture, $python_home) { + Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home + if (Test-Path $python_home) { + Write-Host $python_home "already exists, skipping." + return $false + } + if ($architecture -eq "32") { + $platform_suffix = "" + } else { + $platform_suffix = ".amd64" + } + $filepath = DownloadPython $python_version $platform_suffix + Write-Host "Installing" $filepath "to" $python_home + $args = "/qn /i $filepath TARGETDIR=$python_home" + Write-Host "msiexec.exe" $args + Start-Process -FilePath "msiexec.exe" -ArgumentList $args -Wait -Passthru + Write-Host "Python $python_version ($architecture) installation complete" + return $true +} + + +function InstallPip ($python_home) { + $pip_path = $python_home + "/Scripts/pip.exe" + $python_path = $python_home + "/python.exe" + if (-not(Test-Path $pip_path)) { + Write-Host "Installing pip..." + $webclient = New-Object System.Net.WebClient + $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH) + Write-Host "Executing:" $python_path $GET_PIP_PATH + Start-Process -FilePath "$python_path" -ArgumentList "$GET_PIP_PATH" -Wait -Passthru + } else { + Write-Host "pip already installed." + } +} + +function InstallPackage ($python_home, $pkg) { + $pip_path = $python_home + "/Scripts/pip.exe" + & $pip_path install $pkg +} + +function main () { + InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON + InstallPip $env:PYTHON + InstallPackage $env:PYTHON wheel +} + +main diff --git a/extra/appveyor/run_with_compiler.cmd b/extra/appveyor/run_with_compiler.cmd new file mode 100644 index 000000000..3a472bc83 --- /dev/null +++ b/extra/appveyor/run_with_compiler.cmd @@ -0,0 +1,47 @@ +:: To build extensions for 64 bit Python 3, we need to configure environment +:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) +:: +:: To build extensions for 64 bit Python 2, we need to configure environment +:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) +:: +:: 32 bit builds do not require specific environment configurations. +:: +:: Note: this script needs to be run with the /E:ON and /V:ON flags for the +:: cmd interpreter, at least for (SDK v7.0) +:: +:: More details at: +:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows +:: http://stackoverflow.com/a/13751649/163740 +:: +:: Author: Olivier Grisel +:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ +@ECHO OFF + +SET COMMAND_TO_RUN=%* +SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows + +SET MAJOR_PYTHON_VERSION="%PYTHON_VERSION:~0,1%" +IF %MAJOR_PYTHON_VERSION% == "2" ( + SET WINDOWS_SDK_VERSION="v7.0" +) ELSE IF %MAJOR_PYTHON_VERSION% == "3" ( + SET WINDOWS_SDK_VERSION="v7.1" +) ELSE ( + ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" + EXIT 1 +) + +IF "%PYTHON_ARCH%"=="64" ( + ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture + SET DISTUTILS_USE_SDK=1 + SET MSSdk=1 + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 +) ELSE ( + ECHO Using default MSVC build environment for 32 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 +) From f7b5c4000925be71805080b3d62739bb3018a50d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Dec 2015 13:21:39 -0800 Subject: [PATCH 0859/1103] [appveyor] must install requirements/dev.txt --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index 867715502..07c259a04 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -37,6 +37,7 @@ init: install: - "powershell extra\\appveyor\\install.ps1" - "%PYTHON%/Scripts/pip.exe install -U setuptools" + - "%PYTHON%/Scripts/pip.exe install -r requirements/dev.txt" build: off From bf9756f17a655860cc3292b0813a479849f5cd2b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Dec 2015 16:12:32 -0800 Subject: [PATCH 0860/1103] Windows test fixes --- celery/concurrency/asynpool.py | 6 +++++- celery/tests/backends/test_mongodb.py | 4 ++++ celery/tests/concurrency/test_prefork.py | 8 ++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 781370a16..7e544fd01 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -428,6 +428,7 @@ def __init__(self, processes=None, synack=False, def _event_process_exit(self, hub, fd): # This method is called whenever the process sentinel is readable. + print('>>> HUB REMOVE PROCESS: %r' %(fd,)) hub.remove(fd) self.maintain_pool() @@ -614,6 +615,7 @@ def on_process_down(proc): remove_reader(proc.sentinel) waiting_to_start.discard(proc) self._active_writes.discard(proc.inqW_fd) + print('>>> REMOVE WRITER: %r' % (proc.inqW_fd,)) remove_writer(proc.inqW_fd) remove_reader(proc.outqR_fd) if proc.synqR_fd: @@ -694,7 +696,9 @@ def on_poll_start(): # noqa [hub_add(fd, None, WRITE | ERR, consolidate=True) for fd in diff(active_writes)] else: - [hub_remove(fd) for fd in diff(active_writes)] + fds = diff(active_writes) + print('>>> REMOVING ALL: %r' % (fds,)) + [hub_remove(fd) for fd in fds] self.on_poll_start = on_poll_start def on_inqueue_close(fd, proc): diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index 70c7a9aa6..6419878e6 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -402,6 +402,10 @@ def test_prepare_client_options(self): class test_MongoBackend_no_mock(AppCase): + def setup(self): + if pymongo is None: + raise SkipTest('pymongo is not installed.') + def test_encode_decode(self): backend = MongoBackend(app=self.app) data = {'foo': 1} diff --git a/celery/tests/concurrency/test_prefork.py b/celery/tests/concurrency/test_prefork.py index bd405eb03..3e63fe0f9 100644 --- a/celery/tests/concurrency/test_prefork.py +++ b/celery/tests/concurrency/test_prefork.py @@ -198,6 +198,10 @@ def setup(self): class test_AsynPool(PoolCase): + def setup(self): + if sys.platform == 'win32': + raise SkipTest('win32: skip') + def test_gen_not_started(self): def gen(): @@ -303,6 +307,10 @@ def test_Worker(self): class test_ResultHandler(PoolCase): + def setup(self): + if sys.platform == 'win32': + raise SkipTest('win32: skip') + def test_process_result(self): x = asynpool.ResultHandler( Mock(), Mock(), {}, Mock(), From 2b9cf7b216032942eacfdc7fe143b5ab7b88f770 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Dec 2015 16:19:18 -0800 Subject: [PATCH 0861/1103] forgot to import sys --- celery/tests/concurrency/test_prefork.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/tests/concurrency/test_prefork.py b/celery/tests/concurrency/test_prefork.py index 3e63fe0f9..b317d6821 100644 --- a/celery/tests/concurrency/test_prefork.py +++ b/celery/tests/concurrency/test_prefork.py @@ -3,6 +3,7 @@ import errno import os import socket +import sys from itertools import cycle From dc9db5ff5a08bf4cf3c967cc797e3bb1dbef613b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 3 Dec 2015 13:05:51 -0800 Subject: [PATCH 0862/1103] Attempts to fix Windows CI --- celery/tests/backends/test_database.py | 9 +++++++++ celery/tests/backends/test_filesystem.py | 5 ++++- celery/tests/utils/test_platforms.py | 4 ++++ celery/tests/worker/test_request.py | 4 ++-- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/celery/tests/backends/test_database.py b/celery/tests/backends/test_database.py index 5c2fcba6e..5e716723d 100644 --- a/celery/tests/backends/test_database.py +++ b/celery/tests/backends/test_database.py @@ -22,6 +22,7 @@ import sqlalchemy # noqa except ImportError: DatabaseBackend = Task = TaskSet = retry = None # noqa + SessionManager = session_cleanup = None # noqa else: from celery.backends.database import ( DatabaseBackend, retry, session_cleanup, @@ -39,6 +40,10 @@ def __init__(self, data): class test_session_cleanup(AppCase): + def setup(self): + if session_cleanup is None: + raise SkipTest('slqlalchemy not installed') + def test_context(self): session = Mock(name='session') with session_cleanup(session): @@ -215,6 +220,10 @@ def test_TaskSet__repr__(self): class test_SessionManager(AppCase): + def setup(self): + if SessionManager is None: + raise SkipTest('sqlalchemy not installed') + def test_after_fork(self): s = SessionManager() self.assertFalse(s.forked) diff --git a/celery/tests/backends/test_filesystem.py b/celery/tests/backends/test_filesystem.py index 87639da07..c6019b678 100644 --- a/celery/tests/backends/test_filesystem.py +++ b/celery/tests/backends/test_filesystem.py @@ -3,6 +3,7 @@ import os import shutil +import sys import tempfile from celery import states @@ -10,12 +11,14 @@ from celery.exceptions import ImproperlyConfigured from celery.utils import uuid -from celery.tests.case import AppCase +from celery.tests.case import AppCase, SkipTest class test_FilesystemBackend(AppCase): def setup(self): + if sys.platform == 'win32': + raise SkiptTest('win32: skip') self.directory = tempfile.mkdtemp() self.url = 'file://' + self.directory self.path = self.directory.encode('ascii') diff --git a/celery/tests/utils/test_platforms.py b/celery/tests/utils/test_platforms.py index 2864dccf4..4dd6704f9 100644 --- a/celery/tests/utils/test_platforms.py +++ b/celery/tests/utils/test_platforms.py @@ -61,6 +61,10 @@ def test_short_opt(self): class test_fd_by_path(Case): + def setUp(self): + if sys.platform == 'win32': + raise SkipTest('win32: skip') + def test_finds(self): test_file = tempfile.NamedTemporaryFile() try: diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index 01a0941f2..5b50ff389 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -696,7 +696,7 @@ def test_fast_trace_task(self): message.content_type, message.content_encoding) self.assertFalse(failed) self.assertEqual(res, repr(4 ** 4)) - self.assertTrue(runtime) + self.assertIsNotNone(runtime) self.assertIsInstance(runtime, numbers.Real) finally: reset_worker_optimizations() @@ -708,7 +708,7 @@ def test_fast_trace_task(self): ) self.assertFalse(failed) self.assertEqual(res, repr(4 ** 4)) - self.assertTrue(runtime) + self.assertIsNotNone(runtime) self.assertIsInstance(runtime, numbers.Real) def test_trace_task_ret(self): From 4868f43448e4afbd71224d575e6b7acd1ea5c799 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 3 Dec 2015 15:04:03 -0800 Subject: [PATCH 0863/1103] Fixes typo --- celery/tests/backends/test_filesystem.py | 2 +- celery/tests/worker/test_worker.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/celery/tests/backends/test_filesystem.py b/celery/tests/backends/test_filesystem.py index c6019b678..b8ff0d5ca 100644 --- a/celery/tests/backends/test_filesystem.py +++ b/celery/tests/backends/test_filesystem.py @@ -18,7 +18,7 @@ class test_FilesystemBackend(AppCase): def setup(self): if sys.platform == 'win32': - raise SkiptTest('win32: skip') + raise SkipTest('win32: skip') self.directory = tempfile.mkdtemp() self.url = 'file://' + self.directory self.path = self.directory.encode('ascii') diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index 874d5def6..5fb734270 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -2,6 +2,7 @@ import os import socket +import sys from collections import deque from datetime import datetime, timedelta @@ -1185,6 +1186,7 @@ def timers(self): pool = components.Pool(w) pool.create(w) pool.register_with_event_loop(w, w.hub) - self.assertIsInstance(w.semaphore, LaxBoundedSemaphore) + if sys.platform != 'win32': + self.assertIsInstance(w.semaphore, LaxBoundedSemaphore) P = w.pool P.start() From def92d580fe7a2c42fcc5f47feed802fd6f7ff48 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 3 Dec 2015 18:20:26 -0800 Subject: [PATCH 0864/1103] Windows tests are actually creating pools --- celery/tests/worker/test_worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index 5fb734270..b65663e2d 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -1188,5 +1188,5 @@ def timers(self): pool.register_with_event_loop(w, w.hub) if sys.platform != 'win32': self.assertIsInstance(w.semaphore, LaxBoundedSemaphore) - P = w.pool - P.start() + P = w.pool + P.start() From 6b4922efcb5ab2390a7126136e0ed5be0199781b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Dec 2015 13:55:32 -0800 Subject: [PATCH 0865/1103] Oops, removes debugging print statements --- celery/concurrency/asynpool.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 7e544fd01..781370a16 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -428,7 +428,6 @@ def __init__(self, processes=None, synack=False, def _event_process_exit(self, hub, fd): # This method is called whenever the process sentinel is readable. - print('>>> HUB REMOVE PROCESS: %r' %(fd,)) hub.remove(fd) self.maintain_pool() @@ -615,7 +614,6 @@ def on_process_down(proc): remove_reader(proc.sentinel) waiting_to_start.discard(proc) self._active_writes.discard(proc.inqW_fd) - print('>>> REMOVE WRITER: %r' % (proc.inqW_fd,)) remove_writer(proc.inqW_fd) remove_reader(proc.outqR_fd) if proc.synqR_fd: @@ -696,9 +694,7 @@ def on_poll_start(): # noqa [hub_add(fd, None, WRITE | ERR, consolidate=True) for fd in diff(active_writes)] else: - fds = diff(active_writes) - print('>>> REMOVING ALL: %r' % (fds,)) - [hub_remove(fd) for fd in fds] + [hub_remove(fd) for fd in diff(active_writes)] self.on_poll_start = on_poll_start def on_inqueue_close(fd, proc): From c2c81499137646c3e179aeab69f18b26b30818db Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Dec 2015 16:15:41 -0800 Subject: [PATCH 0866/1103] Implements app.control.autoscale required by inspect autoscale. Closes #2950 --- celery/app/control.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/celery/app/control.py b/celery/app/control.py index 10baf59e9..7058025e0 100644 --- a/celery/app/control.py +++ b/celery/app/control.py @@ -282,6 +282,15 @@ def pool_shrink(self, n=1, destination=None, **kwargs): """ return self.broadcast('pool_shrink', {'n': n}, destination, **kwargs) + def autoscale(self, max, min, destination=None, **kwargs): + """Change worker(s) autoscale setting. + + Supports the same arguments as :meth:`broadcast`. + + """ + return self.broadcast( + 'autoscale', {'max': max, 'min': min}, destination, **kwargs) + def broadcast(self, command, arguments=None, destination=None, connection=None, reply=False, timeout=1, limit=None, callback=None, channel=None, **extra_kwargs): From 4ff2df2b967d95c6e7b0b521ce8b57509a5be4d2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Dec 2015 16:32:22 -0800 Subject: [PATCH 0867/1103] Raise helpful error when backend class is a Python module. Closes #2945 --- celery/backends/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/celery/backends/__init__.py b/celery/backends/__init__.py index 91ad500c4..2f5b07b52 100644 --- a/celery/backends/__init__.py +++ b/celery/backends/__init__.py @@ -9,7 +9,9 @@ from __future__ import absolute_import import sys +import types +from celery.exceptions import ImproperlyConfigured from celery.local import Proxy from celery._state import current_app from celery.five import reraise @@ -47,10 +49,14 @@ def get_backend_cls(backend=None, loader=None): loader = loader or current_app.loader aliases = dict(BACKEND_ALIASES, **loader.override_backends) try: - return symbol_by_name(backend, aliases) + cls = symbol_by_name(backend, aliases) except ValueError as exc: - reraise(ValueError, ValueError(UNKNOWN_BACKEND.format( - backend, exc)), sys.exc_info()[2]) + reraise(ImproperlyConfigured, ImproperlyConfigured( + UNKNOWN_BACKEND.format(backend, exc)), sys.exc_info()[2]) + if isinstance(cls, types.ModuleType): + raise ImproperlyConfigured(UNKNOWN_BACKEND.format( + backend, 'is a Python module, not a backend class.')) + return cls def get_backend_by_url(backend=None, loader=None): From 208d8d07e1a84ca6f1da3028809ee45b948f22fd Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Dec 2015 17:21:48 -0800 Subject: [PATCH 0868/1103] Fixes exceptions deserialzation with amqp result backend join_native (Issue #2409) --- celery/backends/amqp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index ad7cdf226..f88b711aa 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -176,7 +176,8 @@ def get_task_meta(self, task_id, backlog_limit=1000): raise self.BacklogLimitExceeded(task_id) if latest: - payload = self._cache[task_id] = latest.payload + payload = self._cache[task_id] = self.meta_from_decoded( + latest.payload) latest.requeue() return payload else: From fa479458cc7ede43fa463e72b201541f4b1f9606 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Dec 2015 17:34:09 -0800 Subject: [PATCH 0869/1103] Fixes tests --- celery/tests/backends/test_backends.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/tests/backends/test_backends.py b/celery/tests/backends/test_backends.py index c6a936b93..29915b290 100644 --- a/celery/tests/backends/test_backends.py +++ b/celery/tests/backends/test_backends.py @@ -3,6 +3,7 @@ from celery import backends from celery.backends.amqp import AMQPBackend from celery.backends.cache import CacheBackend +from celery.exceptions import ImproperlyConfigured from celery.tests.case import AppCase, depends_on_current_app, patch @@ -36,5 +37,5 @@ def test_backend_by_url(self, url='redis://localhost/1'): def test_sym_raises_ValuError(self): with patch('celery.backends.symbol_by_name') as sbn: sbn.side_effect = ValueError() - with self.assertRaises(ValueError): + with self.assertRaises(ImproperlyConfigured): backends.get_backend_cls('xxx.xxx:foo', self.app.loader) From a4a5c2a7947e2ff073e9ebec87fb3c07f15759ed Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Dec 2015 18:09:59 -0800 Subject: [PATCH 0870/1103] Adds app.current_worker_task property. Closes #2100 --- celery/app/base.py | 10 +++++++--- celery/app/builtins.py | 4 ++-- celery/canvas.py | 4 ++-- celery/tests/app/test_builtins.py | 12 ++++++------ 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 3774b9cce..5968459dc 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -635,12 +635,12 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, options = router.route(options, route_name or name, args, kwargs) if root_id is None: - parent, have_parent = get_current_worker_task(), True + parent, have_parent = self.current_worker_task, True if parent: root_id = parent.request.root_id or parent.request.id if parent_id is None: if not have_parent: - parent, have_parent = get_current_worker_task(), True + parent, have_parent = self.current_worker_task, True if parent: parent_id = parent.request.id @@ -661,7 +661,7 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, result = (result_cls or self.AsyncResult)(task_id) if add_to_parent: if not have_parent: - parent, have_parent = get_current_worker_task(), True + parent, have_parent = self.current_worker_task, True if parent: parent.add_trail(result) return result @@ -1025,6 +1025,10 @@ def current_task(self): :const:`None`.""" return _task_stack.top + @property + def current_worker_task(self): + return get_current_worker_task() + @cached_property def oid(self): return oid_from(self) diff --git a/celery/app/builtins.py b/celery/app/builtins.py index 53cf11925..5d3993474 100644 --- a/celery/app/builtins.py +++ b/celery/app/builtins.py @@ -9,7 +9,7 @@ """ from __future__ import absolute_import -from celery._state import get_current_worker_task, connect_on_app_finalize +from celery._state import connect_on_app_finalize from celery.utils.log import get_logger __all__ = [] @@ -157,7 +157,7 @@ def group(self, tasks, result, group_id, partial_args, add_to_parent=True): with app.producer_or_acquire() as producer: [stask.apply_async(group_id=group_id, producer=producer, add_to_parent=False) for stask in taskit] - parent = get_current_worker_task() + parent = app.current_worker_task if add_to_parent and parent: parent.add_trail(result) return result diff --git a/celery/canvas.py b/celery/canvas.py index 07557ed89..d5dab233a 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -22,7 +22,7 @@ from kombu.utils import cached_property, fxrange, reprcall, uuid -from celery._state import current_app, get_current_worker_task +from celery._state import current_app from celery.local import try_import from celery.result import GroupResult from celery.utils import abstract @@ -761,7 +761,7 @@ def apply_async(self, args=(), kwargs=None, add_to_parent=True, if len(result) == 1 and isinstance(result[0], GroupResult): result = result[0] - parent_task = get_current_worker_task() + parent_task = app.current_worker_task if add_to_parent and parent_task: parent_task.add_trail(result) return result diff --git a/celery/tests/app/test_builtins.py b/celery/tests/app/test_builtins.py index b6539935a..73601734b 100644 --- a/celery/tests/app/test_builtins.py +++ b/celery/tests/app/test_builtins.py @@ -111,21 +111,21 @@ def mock_group(self, *tasks): task.clone.attach_mock(Mock(), 'apply_async') return g, result - @patch('celery.app.builtins.get_current_worker_task') - def test_task(self, get_current_worker_task): + @patch('celery.app.base.Celery.current_worker_task') + def test_task(self, current_worker_task): g, result = self.mock_group(self.add.s(2), self.add.s(4)) self.task(g.tasks, result, result.id, (2,)).results g.tasks[0].clone().apply_async.assert_called_with( group_id=result.id, producer=self.app.producer_or_acquire(), add_to_parent=False, ) - get_current_worker_task().add_trail.assert_called_with(result) + current_worker_task.add_trail.assert_called_with(result) - @patch('celery.app.builtins.get_current_worker_task') - def test_task__disable_add_to_parent(self, get_current_worker_task): + @patch('celery.app.base.Celery.current_worker_task') + def test_task__disable_add_to_parent(self, current_worker_task): g, result = self.mock_group(self.add.s(2, 2), self.add.s(4, 4)) self.task(g.tasks, result, result.id, None, add_to_parent=False) - self.assertFalse(get_current_worker_task().add_trail.called) + self.assertFalse(current_worker_task.add_trail.called) class test_chain(BuiltinsCase): From 2a47f425c70aad44c0c7c5385148c64b1dbfaa1b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Dec 2015 18:29:19 -0800 Subject: [PATCH 0871/1103] Fixes celery graph on Python3. Closes #2133 --- celery/bootsteps.py | 4 +++- celery/datastructures.py | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/celery/bootsteps.py b/celery/bootsteps.py index fa9c71b14..edc7d563f 100644 --- a/celery/bootsteps.py +++ b/celery/bootsteps.py @@ -13,6 +13,7 @@ from kombu.common import ignore_errors from kombu.utils import symbol_by_name +from kombu.utils.encoding import bytes_to_str from .datastructures import DependencyGraph, GraphFormatter from .five import values, with_metaclass @@ -58,7 +59,8 @@ class StepFormatter(GraphFormatter): def label(self, step): return step and '{0}{1}'.format( self._get_prefix(step), - (step.label or _label(step)).encode('utf-8', 'ignore'), + bytes_to_str( + (step.label or _label(step)).encode('utf-8', 'ignore')), ) def _get_prefix(self, step): diff --git a/celery/datastructures.py b/celery/datastructures.py index cc4330870..d6812cf57 100644 --- a/celery/datastructures.py +++ b/celery/datastructures.py @@ -17,7 +17,7 @@ from itertools import chain from billiard.einfo import ExceptionInfo # noqa -from kombu.utils.encoding import safe_str +from kombu.utils.encoding import safe_str, bytes_to_str from kombu.utils.limits import TokenBucket # noqa from celery.five import items @@ -288,7 +288,9 @@ def to_dot(self, fh, formatter=None): """ seen = set() draw = formatter or self.formatter - P = partial(print, file=fh) + + def P(s): + print(bytes_to_str(s), file=fh) def if_not_seen(fun, obj): if draw.label(obj) not in seen: From e65115bfc897850f975eedc8ed241d22ebce0cda Mon Sep 17 00:00:00 2001 From: Ryan Luckie Date: Tue, 8 Dec 2015 11:13:03 -0600 Subject: [PATCH 0872/1103] Reword for clarity --- docs/configuration.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index c3a7f9b1c..0db88c06e 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1594,8 +1594,8 @@ certificate authority: .. warning:: - Be careful using ``broker_use_ssl=True``, it is possible that your default - configuration do not validate the server cert at all, please read Python + Be careful using ``broker_use_ssl=True``. It is possible that your default + configuration will not validate the server cert at all. Please read Python `ssl module security considerations `_. From 3e4cce1688424406638eeca5cd401b4e2f91c41b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 8 Dec 2015 12:19:53 -0800 Subject: [PATCH 0873/1103] Worker: In master the worker crashed if a message could not be decoded --- celery/worker/consumer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 984826518..776feaafc 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -479,7 +479,10 @@ def on_task_received(message): except TypeError: return on_unknown_message(None, message) except KeyError: - payload = message.payload + try: + payload = message.decode() + except Exception as exc: + return self.on_decode_error(message, exc) try: type_, payload = payload['task'], payload # protocol v1 except (TypeError, KeyError): From fff558072ddc97c544dc3a08c2ad2b91b3472886 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 8 Dec 2015 12:20:15 -0800 Subject: [PATCH 0874/1103] Stress: Templates did not support lower-case settings --- funtests/stress/stress/templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index 4a6416e2d..91f7d53f9 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -40,7 +40,7 @@ def mixin_template(template, conf): cls = symbol_by_name(templates[template]) conf.update(dict( (k, v) for k, v in items(vars(cls)) - if k.isupper() and not k.startswith('_') + if not k.startswith('_') )) From 5ae95c8a01d0f7a012a71c779062b3176c1fcf15 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 8 Dec 2015 12:20:48 -0800 Subject: [PATCH 0875/1103] Prefork: Fixes task execution when using pickle and protocol1. Closes #2942 --- celery/app/trace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/app/trace.py b/celery/app/trace.py index 5634a867f..d887e57f0 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -486,7 +486,7 @@ def _trace_task_ret(name, uuid, request, body, content_type, body, content_type, content_encoding, accept=accept, ) else: - args, kwargs = body + args, kwargs, embed = body hostname = socket.gethostname() request.update({ 'args': args, 'kwargs': kwargs, @@ -508,7 +508,7 @@ def _fast_trace_task(task, uuid, request, body, content_type, body, content_type, content_encoding, accept=accept, ) else: - args, kwargs = body + args, kwargs, embed = body request.update({ 'args': args, 'kwargs': kwargs, 'hostname': hostname, 'is_eager': False, From 7b876989921968897c06d690e04d2025576d56f0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 8 Dec 2015 14:09:59 -0800 Subject: [PATCH 0876/1103] Fixes bug with configuration key prefix --- celery/datastructures.py | 38 +++++++++++++++---------- celery/tests/app/test_app.py | 4 +-- celery/utils/text.py | 4 +++ docs/django/first-steps-with-django.rst | 2 +- examples/django/proj/celery.py | 2 +- 5 files changed, 31 insertions(+), 19 deletions(-) diff --git a/celery/datastructures.py b/celery/datastructures.py index d6812cf57..0580de559 100644 --- a/celery/datastructures.py +++ b/celery/datastructures.py @@ -22,6 +22,7 @@ from celery.five import items from celery.utils.functional import LRUCache, first, uniq # noqa +from celery.utils.text import match_case try: from django.utils.functional import LazyObject, LazySettings @@ -462,14 +463,17 @@ def __init__(self, changes, defaults=None, key_t=None, prefix=None): defaults=defaults, key_t=key_t, _order=[changes] + defaults, - prefix=prefix, + prefix=prefix.rstrip('_') + '_' if prefix else prefix, ) + def _to_keys(self, key): + prefix = self.prefix + if prefix: + pkey = prefix + key if not key.startswith(prefix) else key + return match_case(pkey, prefix), self._key(key) + return self._key(key), + def _key(self, key): - if self.prefix: - key = self.prefix + key - if self.prefix.isupper(): - key = key.upper() return self.key_t(key) if self.key_t is not None else key def add_defaults(self, d): @@ -478,23 +482,27 @@ def add_defaults(self, d): self._order.insert(1, d) def __getitem__(self, key): - key = self._key(key) - for d in self._order: - try: - return d[key] - except KeyError: - pass + keys = self._to_keys(key) + for k in keys: + for d in self._order: + try: + return d[k] + except KeyError: + pass + if len(keys) > 1: + raise KeyError( + 'Key not found: {0!r} (with prefix: {0!r})'.format(*keys)) raise KeyError(key) def __setitem__(self, key, value): self.changes[self._key(key)] = value def first(self, *keys): - return first(None, (self.get(self._key(key)) for key in keys)) + return first(None, (self.get(key) for key in keys)) def get(self, key, default=None): try: - return self[self._key(key)] + return self[key] except KeyError: return default @@ -511,8 +519,8 @@ def update(self, *args, **kwargs): return self.changes.update(*args, **kwargs) def __contains__(self, key): - key = self._key(key) - return any(key in m for m in self._order) + keys = self._to_keys(key) + return any(any(k in m for k in keys) for m in self._order) def __bool__(self): return any(self._order) diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index 304037265..34970799c 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -599,7 +599,7 @@ class Config(object): CELERY_TASK_ALWAYS_EAGER = 44 CELERY_TASK_DEFAULT_DELIVERY_MODE = 301 - self.app.config_from_object(Config(), namespace='CELERY_') + self.app.config_from_object(Config(), namespace='CELERY') self.assertEqual(self.app.conf.task_always_eager, 44) def test_config_from_object__namespace_lowercase(self): @@ -608,7 +608,7 @@ class Config(object): celery_task_always_eager = 44 celery_task_default_delivery_mode = 301 - self.app.config_from_object(Config(), namespace='celery_') + self.app.config_from_object(Config(), namespace='celery') self.assertEqual(self.app.conf.task_always_eager, 44) def test_config_from_object__mixing_new_and_old(self): diff --git a/celery/utils/text.py b/celery/utils/text.py index 6bf34bf59..2920ad782 100644 --- a/celery/utils/text.py +++ b/celery/utils/text.py @@ -90,3 +90,7 @@ def pretty(value, width=80, nl_width=80, sep='\n', **kw): ) else: return pformat(value, width=width, **kw) + + +def match_case(s, other): + return s.upper() if other.isupper() else s.lower() diff --git a/docs/django/first-steps-with-django.rst b/docs/django/first-steps-with-django.rst index 70786d81c..d033f0741 100644 --- a/docs/django/first-steps-with-django.rst +++ b/docs/django/first-steps-with-django.rst @@ -90,7 +90,7 @@ or execv: .. code-block:: python - app.config_from_object('django.conf:settings', namespace='CELERY_') + app.config_from_object('django.conf:settings', namespace='CELERY') Next, a common practice for reusable apps is to define all tasks in a separate ``tasks.py`` module, and Celery does have a way to diff --git a/examples/django/proj/celery.py b/examples/django/proj/celery.py index 02020c6eb..d7ea41a48 100644 --- a/examples/django/proj/celery.py +++ b/examples/django/proj/celery.py @@ -11,7 +11,7 @@ # Using a string here means the worker will not have to # pickle the object when using Windows. -app.config_from_object('django.conf:settings', namespace='CELERY_') +app.config_from_object('django.conf:settings', namespace='CELERY') # load task modules from all registered Django app configs. app.autodiscover_tasks() From ebbfa84cc11b2916a0b4ee941ef2ba5dfa9e07f9 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 8 Dec 2015 14:11:17 -0800 Subject: [PATCH 0877/1103] Fixes build --- celery/tests/tasks/test_trace.py | 4 ++-- celery/tests/worker/test_loops.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/tests/tasks/test_trace.py b/celery/tests/tasks/test_trace.py index a1b9e1ace..a8090ab2d 100644 --- a/celery/tests/tasks/test_trace.py +++ b/celery/tests/tasks/test_trace.py @@ -274,7 +274,7 @@ def test_trace_exception(self): def test_trace_task_ret__no_content_type(self): _trace_task_ret( - self.add.name, 'id1', {}, ((2, 2), {}), None, None, + self.add.name, 'id1', {}, ((2, 2), {}, {}), None, None, app=self.app, ) @@ -283,7 +283,7 @@ def test_fast_trace_task__no_content_type(self): self.add.name, self.add, app=self.app, ) _fast_trace_task( - self.add.name, 'id1', {}, ((2, 2), {}), None, None, + self.add.name, 'id1', {}, ((2, 2), {}, {}), None, None, app=self.app, _loc=[self.app.tasks, {}, 'hostname'] ) diff --git a/celery/tests/worker/test_loops.py b/celery/tests/worker/test_loops.py index f8dc07f7b..ada763873 100644 --- a/celery/tests/worker/test_loops.py +++ b/celery/tests/worker/test_loops.py @@ -177,7 +177,7 @@ def test_on_task_message_missing_name(self): x, on_task, msg, strategy = self.task_context(self.add.s(2, 2)) msg.headers.pop('task') on_task(msg) - x.on_unknown_message.assert_called_with(msg.payload, msg) + x.on_unknown_message.assert_called_with(msg.decode(), msg) def test_on_task_not_registered(self): x, on_task, msg, strategy = self.task_context(self.add.s(2, 2)) From b274229c57cd2bd3efbe92f833c6c8c5bb768770 Mon Sep 17 00:00:00 2001 From: Eric Zarowny Date: Tue, 8 Dec 2015 16:33:24 -0800 Subject: [PATCH 0878/1103] change celerybeat generic init script to report service as down when no pid file can be found --- extra/generic-init.d/celerybeat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/generic-init.d/celerybeat b/extra/generic-init.d/celerybeat index 85785caa5..5d221e630 100755 --- a/extra/generic-init.d/celerybeat +++ b/extra/generic-init.d/celerybeat @@ -254,7 +254,7 @@ check_status () { local failed= local pid_file=$CELERYBEAT_PID_FILE if [ ! -e $pid_file ]; then - echo "${SCRIPT_NAME} is up: no pid file found" + echo "${SCRIPT_NAME} is down: no pid file found" failed=true elif [ ! -r $pid_file ]; then echo "${SCRIPT_NAME} is in unknown state, user cannot read pid file." From f3ac3173ec7c1b9cf704b267582658350e5e08b7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 8 Dec 2015 17:48:45 -0800 Subject: [PATCH 0879/1103] Use kombu.pools for connection/producer pools --- celery/app/amqp.py | 9 +++------ celery/app/base.py | 30 ++++++++++++------------------ celery/tests/app/test_app.py | 21 ++++----------------- 3 files changed, 19 insertions(+), 41 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 2089d7027..c6ab2e241 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -15,9 +15,9 @@ from datetime import timedelta from weakref import WeakValueDictionary +from kombu import pools from kombu import Connection, Consumer, Exchange, Producer, Queue from kombu.common import Broadcast -from kombu.pools import ProducerPool from kombu.utils import cached_property from kombu.utils.functional import maybe_list @@ -567,11 +567,8 @@ def router(self): @property def producer_pool(self): if self._producer_pool is None: - self._producer_pool = ProducerPool( - self.app.pool, - limit=self.app.pool.limit, - Producer=self.Producer, - ) + self._producer_pool = pools.producers[self.app.connection()] + self._producer_pool.limit = self.app.pool.limit return self._producer_pool publisher_pool = producer_pool # compat alias diff --git a/celery/app/base.py b/celery/app/base.py index 5968459dc..047bc2c88 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -17,10 +17,7 @@ from functools import wraps from amqp import starpromise -try: - from billiard.util import register_after_fork -except ImportError: # pragma: no cover - register_after_fork = None +from kombu import pools from kombu.clocks import LamportClock from kombu.common import oid_from from kombu.utils import cached_property, uuid @@ -56,6 +53,11 @@ # Load all builtin tasks from . import builtins # noqa +try: + from billiard.util import register_after_fork +except ImportError: # pragma: no cover + register_after_fork = None + __all__ = ['Celery'] _EXECV = os.environ.get('FORKED_BY_MULTIPROCESSING') @@ -276,8 +278,7 @@ def __exit__(self, *exc_info): self.close() def close(self): - """Close any open pool connections and do any other steps necessary - to clean up after the application. + """Clean up after the application. Only necessary for dynamically created apps for which you can use the with statement instead:: @@ -286,7 +287,7 @@ def close(self): with app.connection() as conn: pass """ - self._maybe_close_pool() + self._pool = None _deregister_app(self) def on_init(self): @@ -828,16 +829,8 @@ def _load_config(self): return self._conf def _after_fork(self, obj_): - self._maybe_close_pool() - - def _maybe_close_pool(self): - if self._pool: - self._pool.force_close_all() - self._pool = None - amqp = self.__dict__.get('amqp') - if amqp is not None and amqp._producer_pool is not None: - amqp._producer_pool.force_close_all() - amqp._producer_pool = None + self._pool = None + pools.reset() def signature(self, *args, **kwargs): """Return a new :class:`~celery.canvas.Signature` bound to this app. @@ -1016,7 +1009,8 @@ def pool(self): if self._pool is None: _ensure_after_fork() limit = self.conf.broker_pool_limit - self._pool = self.connection().Pool(limit=limit) + pools.set_limit(limit) + self._pool = pools.connections[self.connection()] return self._pool @property diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index 34970799c..6e7408799 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -196,20 +196,6 @@ def test_connection_or_acquire(self): with self.app.connection_or_acquire(pool=False): self.assertFalse(self.app.pool._dirty) - def test_maybe_close_pool(self): - cpool = self.app._pool = Mock() - amqp = self.app.__dict__['amqp'] = Mock() - ppool = amqp._producer_pool - self.app._maybe_close_pool() - cpool.force_close_all.assert_called_with() - ppool.force_close_all.assert_called_with() - self.assertIsNone(self.app._pool) - self.assertIsNone(self.app.__dict__['amqp']._producer_pool) - - self.app._pool = Mock() - self.app._maybe_close_pool() - self.app._maybe_close_pool() - def test_using_v1_reduce(self): self.app._using_v1_reduce = True self.assertTrue(loads(dumps(self.app))) @@ -790,11 +776,12 @@ def my_failover_strategy(it): my_failover_strategy, ) - def test_after_fork(self): - p = self.app._pool = Mock() + @patch('kombu.pools.reset') + def test_after_fork(self, reset): + self.app._pool = Mock() self.app._after_fork(self.app) - p.force_close_all.assert_called_with() self.assertIsNone(self.app._pool) + reset.assert_called_with() self.app._after_fork(self.app) def test_global_after_fork(self): From a5f5b63a4b92a15245aa80473badca655de13db2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 8 Dec 2015 17:49:40 -0800 Subject: [PATCH 0880/1103] flakes --- celery/canvas.py | 3 ++- celery/datastructures.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index d5dab233a..192e2b02a 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -870,7 +870,8 @@ def freeze(self, _id=None, group_id=None, chord=None, if not isinstance(self.tasks, group): self.tasks = group(self.tasks) bodyres = self.body.freeze(_id, parent_id=self.id, root_id=root_id) - self.tasks.freeze(parent_id=parent_id, root_id=root_id, chord=self.body) + self.tasks.freeze( + parent_id=parent_id, root_id=root_id, chord=self.body) self.id = self.tasks.id self.body.set_parent_id(self.id) return bodyres diff --git a/celery/datastructures.py b/celery/datastructures.py index 0580de559..e889e5e8b 100644 --- a/celery/datastructures.py +++ b/celery/datastructures.py @@ -13,7 +13,6 @@ from collections import defaultdict, Mapping, MutableMapping, MutableSet from heapq import heapify, heappush, heappop -from functools import partial from itertools import chain from billiard.einfo import ExceptionInfo # noqa From 9f526499398d433f4463efbfbfb7e98b3a12f308 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 9 Dec 2015 13:01:32 -0800 Subject: [PATCH 0881/1103] Fixes typo signalled -> signaled --- celery/app/task.py | 2 +- docs/configuration.rst | 2 +- docs/whatsnew-3.0.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index bbd1d85e6..e14d4e625 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -223,7 +223,7 @@ class Task(object): #: Even if :attr:`acks_late` is enabled, the worker will #: acknowledge tasks when the worker process executing them abrubtly - #: exits or is signalled (e.g. :sig:`KILL`/:sig:`INT`, etc). + #: exits or is signaled (e.g. :sig:`KILL`/:sig:`INT`, etc). #: #: Setting this to true allows the message to be requeued instead, #: so that the task will execute again by the same worker, or another diff --git a/docs/configuration.rst b/docs/configuration.rst index c3a7f9b1c..aead2c6a6 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -447,7 +447,7 @@ task_reject_on_worker_lost Even if :setting:`task_acks_late` is enabled, the worker will acknowledge tasks when the worker process executing them abrubtly -exits or is signalled (e.g. :sig:`KILL`/:sig:`INT`, etc). +exits or is signaled (e.g. :sig:`KILL`/:sig:`INT`, etc). Setting this to true allows the message to be requeued instead, so that the task will execute again by the same worker, or another diff --git a/docs/whatsnew-3.0.rst b/docs/whatsnew-3.0.rst index dc1320e27..165bb54ab 100644 --- a/docs/whatsnew-3.0.rst +++ b/docs/whatsnew-3.0.rst @@ -709,7 +709,7 @@ In Other News - New :setting:`CELERYD_WORKER_LOST_WAIT` to control the timeout in seconds before :exc:`billiard.WorkerLostError` is raised - when a worker can not be signalled (Issue #595). + when a worker can not be signaled (Issue #595). Contributed by Brendon Crawford. From 6dac87f0f67657be8fee56e13c00869bfceb8570 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 9 Dec 2015 14:39:09 -0800 Subject: [PATCH 0882/1103] Fixes a bug with using kombu pools and after forkers and cleans up after forkers --- celery/app/base.py | 62 +++++++++++++---------------- celery/backends/database/session.py | 15 +++---- celery/tests/app/test_app.py | 50 +++++++++-------------- 3 files changed, 54 insertions(+), 73 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 047bc2c88..1bbc13362 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -20,7 +20,7 @@ from kombu import pools from kombu.clocks import LamportClock from kombu.common import oid_from -from kombu.utils import cached_property, uuid +from kombu.utils import cached_property, register_after_fork, uuid from celery import platforms from celery import signals @@ -40,6 +40,7 @@ from celery.utils.dispatch import Signal from celery.utils.functional import first, maybe_list, head_from_fun from celery.utils.imports import instantiate, symbol_by_name +from celery.utils.log import get_logger from celery.utils.objects import FallbackContext, mro_lookup from .annotations import prepare as prepare_annotations @@ -53,13 +54,10 @@ # Load all builtin tasks from . import builtins # noqa -try: - from billiard.util import register_after_fork -except ImportError: # pragma: no cover - register_after_fork = None - __all__ = ['Celery'] +logger = get_logger(__name__) + _EXECV = os.environ.get('FORKED_BY_MULTIPROCESSING') BUILTIN_FIXUPS = { 'celery.fixups.django:fixup', @@ -71,8 +69,6 @@ Please set this variable and make it point to a configuration module.""" -_after_fork_registered = False - def app_has_custom(app, attr): return mro_lookup(app.__class__, attr, stop=(Celery, object), @@ -85,30 +81,11 @@ def _unpickle_appattr(reverse_name, args): return get_current_app()._rgetattr(reverse_name)(*args) -def _global_after_fork(obj): - # Previously every app would call: - # `register_after_fork(app, app._after_fork)` - # but this created a leak as `register_after_fork` stores concrete object - # references and once registered an object cannot be removed without - # touching and iterating over the private afterfork registry list. - # - # See Issue #1949 - from celery import _state - from multiprocessing import util as mputil - for app in _state._apps: - try: - app._after_fork(obj) - except Exception as exc: - if mputil._logger: - mputil._logger.info( - 'after forker raised exception: %r', exc, exc_info=1) - - -def _ensure_after_fork(): - global _after_fork_registered - _after_fork_registered = True - if register_after_fork is not None: - register_after_fork(_global_after_fork, _global_after_fork) +def _after_fork_cleanup_app(app): + try: + app._after_fork() + except Exception as exc: + logger.info('after forker raised exception: %r', exc, exc_info=1) class PendingConfiguration(UserDict, AttributeDictMixin): @@ -180,6 +157,7 @@ class Celery(object): _pool = None _conf = None builtin_fixups = BUILTIN_FIXUPS + _after_fork_registered = False #: Signal sent when app is loading configuration. on_configure = None @@ -190,6 +168,9 @@ class Celery(object): #: Signal sent after app has been finalized. on_after_finalize = None + #: Signal sent by every new process after fork. + on_after_fork = None + def __init__(self, main=None, loader=None, backend=None, amqp=None, events=None, log=None, control=None, set_as_current=True, tasks=None, broker=None, include=None, @@ -254,6 +235,7 @@ def __init__(self, main=None, loader=None, backend=None, self.on_configure = Signal() self.on_after_configure = Signal() self.on_after_finalize = Signal() + self.on_after_fork = Signal() self.on_init() _register_app(self) @@ -271,6 +253,12 @@ def set_default(self): """Makes this the default app for all threads.""" set_default_app(self) + def _ensure_after_fork(self): + if not self._after_fork_registered: + self._after_fork_registered = True + if register_after_fork is not None: + register_after_fork(self, _after_fork_cleanup_app) + def __enter__(self): return self @@ -828,9 +816,13 @@ def _load_config(self): self.on_after_configure.send(sender=self, source=self._conf) return self._conf - def _after_fork(self, obj_): + def _after_fork(self): self._pool = None - pools.reset() + try: + self.__dict__['amqp']._producer_pool = None + except (AttributeError, KeyError): + pass + self.on_after_fork.send(sender=self) def signature(self, *args, **kwargs): """Return a new :class:`~celery.canvas.Signature` bound to this app. @@ -1007,7 +999,7 @@ def pool(self): """ if self._pool is None: - _ensure_after_fork() + self._ensure_after_fork() limit = self.conf.broker_pool_limit pools.set_limit(limit) self._pool = pools.connections[self.connection()] diff --git a/celery/backends/database/session.py b/celery/backends/database/session.py index 17cdc8982..451c735c6 100644 --- a/celery/backends/database/session.py +++ b/celery/backends/database/session.py @@ -8,21 +8,22 @@ """ from __future__ import absolute_import -try: - from billiard.util import register_after_fork -except ImportError: # pragma: no cover - register_after_fork = None - from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import NullPool +from kombu.utils import register_after_fork + ResultModelBase = declarative_base() __all__ = ['SessionManager'] +def _after_fork_cleanup_session(session): + session._after_fork() + + class SessionManager(object): def __init__(self): @@ -31,9 +32,9 @@ def __init__(self): self.forked = False self.prepared = False if register_after_fork is not None: - register_after_fork(self, self._after_fork) + register_after_fork(self, _after_fork_cleanup_session) - def _after_fork(self,): + def _after_fork(self): self.forked = True def get_engine(self, dburi, **kwargs): diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index 6e7408799..7a8a415a2 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -776,46 +776,34 @@ def my_failover_strategy(it): my_failover_strategy, ) - @patch('kombu.pools.reset') - def test_after_fork(self, reset): + def test_after_fork(self): self.app._pool = Mock() - self.app._after_fork(self.app) + self.app.on_after_fork = Mock(name='on_after_fork') + self.app._after_fork() self.assertIsNone(self.app._pool) - reset.assert_called_with() - self.app._after_fork(self.app) + self.app.on_after_fork.send.assert_called_with(sender=self.app) + self.app._after_fork() def test_global_after_fork(self): - app = Mock(name='app') - prev, _state._apps = _state._apps, [app] - try: - obj = Mock(name='obj') - _appbase._global_after_fork(obj) - app._after_fork.assert_called_with(obj) - finally: - _state._apps = prev - - @patch('multiprocessing.util', create=True) - def test_global_after_fork__raises(self, util): - app = Mock(name='app') - prev, _state._apps = _state._apps, [app] - try: - obj = Mock(name='obj') - exc = app._after_fork.side_effect = KeyError() - _appbase._global_after_fork(obj) - util._logger.info.assert_called_with( - 'after forker raised exception: %r', exc, exc_info=1) - util._logger = None - _appbase._global_after_fork(obj) - finally: - _state._apps = prev + self.app._after_fork = Mock(name='_after_fork') + _appbase._after_fork_cleanup_app(self.app) + self.app._after_fork.assert_called_with() + + @patch('celery.app.base.logger') + def test_after_fork_cleanup_app__raises(self, logger): + self.app._after_fork = Mock(name='_after_fork') + exc = self.app._after_fork.side_effect = KeyError() + _appbase._after_fork_cleanup_app(self.app) + logger.info.assert_called_with( + 'after forker raised exception: %r', exc, exc_info=1) def test_ensure_after_fork__no_multiprocessing(self): prev, _appbase.register_after_fork = ( _appbase.register_after_fork, None) try: - _appbase._after_fork_registered = False - _appbase._ensure_after_fork() - self.assertTrue(_appbase._after_fork_registered) + self.app._after_fork_registered = False + self.app._ensure_after_fork() + self.assertTrue(self.app._after_fork_registered) finally: _appbase.register_after_fork = prev From 682bcec3651e57b381ee5689a9746b31029c5a3d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 9 Dec 2015 15:13:35 -0800 Subject: [PATCH 0883/1103] Redis new_join: Chord error should call link_error callbacks (Issue #2796) --- celery/backends/redis.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index ae8f7fd82..00bc01227 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -222,19 +222,18 @@ def on_chord_part_return(self, request, state, result, propagate=None): except Exception as exc: error('Chord callback for %r raised: %r', request.group, exc, exc_info=1) - app._tasks[callback.task].backend.fail_from_current_stack( - callback.id, - exc=ChordError('Callback error: {0!r}'.format(exc)), + return self.chord_error_from_stack( + callback, + ChordError('Callback error: {0!r}'.format(exc)), ) except ChordError as exc: error('Chord %r raised: %r', request.group, exc, exc_info=1) - app._tasks[callback.task].backend.fail_from_current_stack( - callback.id, exc=exc, - ) + return self.chord_error_from_stack(callback, exc) except Exception as exc: error('Chord %r raised: %r', request.group, exc, exc_info=1) - app._tasks[callback.task].backend.fail_from_current_stack( - callback.id, exc=ChordError('Join error: {0!r}'.format(exc)), + return self.chord_error_from_stack( + callback, + ChordError('Join error: {0!r}'.format(exc)), ) def _create_client(self, socket_timeout=None, socket_connect_timeout=None, From 8c346495bfaaacbdce8bd47c57ab60306c1121d0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 11:18:46 -0800 Subject: [PATCH 0884/1103] Updates donate button --- docs/.templates/sidebarintro.html | 16 +++++++--------- docs/.templates/sidebarlogo.html | 16 +++++++--------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/docs/.templates/sidebarintro.html b/docs/.templates/sidebarintro.html index 16cca544a..cc68b8f24 100644 --- a/docs/.templates/sidebarintro.html +++ b/docs/.templates/sidebarintro.html @@ -2,14 +2,12 @@

diff --git a/docs/.templates/sidebarlogo.html b/docs/.templates/sidebarlogo.html index 16cca544a..cc68b8f24 100644 --- a/docs/.templates/sidebarlogo.html +++ b/docs/.templates/sidebarlogo.html @@ -2,14 +2,12 @@

From 0252652a2055719f5451206330f9107038d3b8c8 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 11:44:42 -0800 Subject: [PATCH 0885/1103] Adds sponsored by Robinhood logo --- docs/.templates/sidebarintro.html | 7 ++++++- docs/.templates/sidebarlogo.html | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/.templates/sidebarintro.html b/docs/.templates/sidebarintro.html index cc68b8f24..8eb9fea26 100644 --- a/docs/.templates/sidebarintro.html +++ b/docs/.templates/sidebarintro.html @@ -1,5 +1,5 @@ + +
+ Sponsored by: + +
diff --git a/docs/.templates/sidebarlogo.html b/docs/.templates/sidebarlogo.html index cc68b8f24..8eb9fea26 100644 --- a/docs/.templates/sidebarlogo.html +++ b/docs/.templates/sidebarlogo.html @@ -1,5 +1,5 @@ + +
+ Sponsored by: + +
From f733d93cae73cd60d614846c9c745b5b1fdae5e1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 11:48:52 -0800 Subject: [PATCH 0886/1103] Adds link to Robinhood --- docs/.templates/sidebarintro.html | 4 +++- docs/.templates/sidebarlogo.html | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/.templates/sidebarintro.html b/docs/.templates/sidebarintro.html index 8eb9fea26..2c5b83e3b 100644 --- a/docs/.templates/sidebarintro.html +++ b/docs/.templates/sidebarintro.html @@ -14,5 +14,7 @@
Sponsored by: - + + +
diff --git a/docs/.templates/sidebarlogo.html b/docs/.templates/sidebarlogo.html index 8eb9fea26..2c5b83e3b 100644 --- a/docs/.templates/sidebarlogo.html +++ b/docs/.templates/sidebarlogo.html @@ -14,5 +14,7 @@
Sponsored by: - + + +
From b95d02e429ccf3cbc151aa6f0a83a982b98b4334 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 11:52:57 -0800 Subject: [PATCH 0887/1103] Trying to fix RTD build --- requirements/docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/docs.txt b/requirements/docs.txt index b0bdf1c0c..d2b1b673a 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,3 +1,3 @@ Sphinx -r extras/sqlalchemy.txt --r dev.txt +-r -U dev.txt From b03e3dfff306fe493d4fd42765b694ccbd9b9af7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 11:55:37 -0800 Subject: [PATCH 0888/1103] Trying to fix RTD build again --- requirements/docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/docs.txt b/requirements/docs.txt index d2b1b673a..b66874546 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,3 +1,3 @@ Sphinx -r extras/sqlalchemy.txt --r -U dev.txt +-U -r dev.txt From 9005f60e4806bdec0643ca81a7ddaee18ae30755 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 11:58:25 -0800 Subject: [PATCH 0889/1103] Oh well --- requirements/docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/docs.txt b/requirements/docs.txt index b66874546..b0bdf1c0c 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,3 +1,3 @@ Sphinx -r extras/sqlalchemy.txt --U -r dev.txt +-r dev.txt From 7d4c9bc3267903b8c7edd1ce1ce2bc5b39e2094f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 13:12:50 -0800 Subject: [PATCH 0890/1103] Worker: Fixes on_unknown_message for proto1 --- celery/worker/consumer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 776feaafc..ba558f1aa 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -439,7 +439,10 @@ def on_unknown_message(self, body, message): def on_unknown_task(self, body, message, exc): error(UNKNOWN_TASK_ERROR, exc, dump_body(message, body), exc_info=True) - id_, name = message.headers['id'], message.headers['task'] + try: + id_, name = message.headers['id'], message.headers['task'] + except KeyError: # proto1 + id_, name = body['id'], body['task'] message.reject_log_error(logger, self.connection_errors) self.app.backend.mark_as_failure(id_, NotRegistered(name)) if self.event_dispatcher: From 7e82a328347ec6cacb28e418c89813e3106b710e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 13:13:24 -0800 Subject: [PATCH 0891/1103] Canvas: Fixes repr for immutable tasks --- celery/canvas.py | 8 ++++---- celery/tests/tasks/test_canvas.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 192e2b02a..a98859ac3 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -193,8 +193,8 @@ def apply(self, args=(), kwargs={}, **options): args, kwargs, options = self._merge(args, kwargs, options) return self.type.apply(args, kwargs, **options) - def _merge(self, args=(), kwargs={}, options={}): - if self.immutable: + def _merge(self, args=(), kwargs={}, options={}, force=False): + if self.immutable and not force: return (self.args, self.kwargs, dict(self.options, **options) if options else self.options) return (tuple(args) + tuple(self.args) if args else self.args, @@ -323,7 +323,7 @@ def __json__(self): return dict(self) def reprcall(self, *args, **kwargs): - args, kwargs, _ = self._merge(args, kwargs, {}) + args, kwargs, _ = self._merge(args, kwargs, {}, force=True) return reprcall(self['task'], args, kwargs) def election(self): @@ -840,7 +840,7 @@ def __iter__(self): return iter(self.tasks) def __repr__(self): - return repr(self.tasks) + return 'group({0.tasks!r})'.format(self) @property def app(self): diff --git a/celery/tests/tasks/test_canvas.py b/celery/tests/tasks/test_canvas.py index e8ba66e22..c56394e7d 100644 --- a/celery/tests/tasks/test_canvas.py +++ b/celery/tests/tasks/test_canvas.py @@ -476,7 +476,7 @@ class test_group(CanvasCase): def test_repr(self): x = group([self.add.s(2, 2), self.add.s(4, 4)]) - self.assertEqual(repr(x), repr(x.tasks)) + self.assertTrue(repr(x)) def test_reverse(self): x = group([self.add.s(2, 2), self.add.s(4, 4)]) From 67c4d3e12b90b416ed4714f390220736040ffff1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 13:14:03 -0800 Subject: [PATCH 0892/1103] Canvas: `group | group` is now unrolled into single group (Issue #2573) --- celery/canvas.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index a98859ac3..7a811753f 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -293,10 +293,13 @@ def flatten_links(self): ))) def __or__(self, other): - if isinstance(other, group): - other = maybe_unroll_group(other) if isinstance(self, group): + if isinstance(other, group): + return group(_chain(self.tasks, other.tasks), app=self.app) return chord(self, body=other, app=self._app) + elif isinstance(other, group): + other = maybe_unroll_group(other) + if not isinstance(self, chain) and isinstance(other, chain): return chain((self,) + other.tasks, app=self._app) elif isinstance(other, chain): From 7eab3f59b64e60bbfd9688f05296ce303b9beccc Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 13:14:45 -0800 Subject: [PATCH 0893/1103] Canvas: Chord does not always pass app to group() --- celery/canvas.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 7a811753f..a84adc99b 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -871,7 +871,7 @@ def __init__(self, header, body=None, task='celery.chord', def freeze(self, _id=None, group_id=None, chord=None, root_id=None, parent_id=None): if not isinstance(self.tasks, group): - self.tasks = group(self.tasks) + self.tasks = group(self.tasks, app=self.app) bodyres = self.body.freeze(_id, parent_id=self.id, root_id=root_id) self.tasks.freeze( parent_id=parent_id, root_id=root_id, chord=self.body) @@ -924,7 +924,7 @@ def apply_async(self, args=(), kwargs={}, task_id=None, body = body.clone(**options) app = self._get_app(body) tasks = (self.tasks.clone() if isinstance(self.tasks, group) - else group(self.tasks)) + else group(self.tasks, app=app)) if app.conf.task_always_eager: return self.apply((), kwargs, body=body, task_id=task_id, **options) @@ -933,7 +933,7 @@ def apply_async(self, args=(), kwargs={}, task_id=None, def apply(self, args=(), kwargs={}, propagate=True, body=None, **options): body = self.body if body is None else body tasks = (self.tasks.clone() if isinstance(self.tasks, group) - else group(self.tasks)) + else group(self.tasks, app=self.app)) return body.apply( args=(tasks.apply().get(propagate=propagate),), ) From 1de0d5d5aec945abdfb055fc38600290a05d10b5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 13:15:05 -0800 Subject: [PATCH 0894/1103] Chord: Chords containing groups with iterators now work. regen does not work with dequeue(regen(it)), as deque seems to use some C magic instead of __iter__ for copying the list, so the iterator ends up being consumed. --- celery/canvas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index a84adc99b..b00df8e7d 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -939,7 +939,7 @@ def apply(self, args=(), kwargs={}, propagate=True, body=None, **options): ) def _traverse_tasks(self, tasks, value=None): - stack = deque(tasks) + stack = deque(list(tasks)) while stack: task = stack.popleft() if isinstance(task, group): From 2d6d660ee678623adc66c3c2745279d0cfc68d86 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 13:44:10 -0800 Subject: [PATCH 0895/1103] Worker: inspect active and friends must copy active_requests when using threads. Closes #2567 --- celery/tests/worker/test_control.py | 7 +++++-- celery/worker/control.py | 15 ++++++++++----- celery/worker/pidbox.py | 7 ++++++- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/celery/tests/worker/test_control.py b/celery/tests/worker/test_control.py index 691e6e51d..dcabfb6f2 100644 --- a/celery/tests/worker/test_control.py +++ b/celery/tests/worker/test_control.py @@ -126,6 +126,7 @@ def mytask(): def create_state(self, **kwargs): kwargs.setdefault('app', self.app) kwargs.setdefault('hostname', hostname) + kwargs.setdefault('tset', set) return AttributeDict(kwargs) def create_panel(self, **kwargs): @@ -481,14 +482,16 @@ def test_revoke(self): def test_revoke_terminate(self): request = Mock() request.id = tid = uuid() + state = self.create_state() + state.consumer = Mock() worker_state.reserved_requests.add(request) try: - r = control.revoke(Mock(), tid, terminate=True) + r = control.revoke(state, tid, terminate=True) self.assertIn(tid, revoked) self.assertTrue(request.terminate.call_count) self.assertIn('terminate:', r['ok']) # unknown task id only revokes - r = control.revoke(Mock(), uuid(), terminate=True) + r = control.revoke(state, uuid(), terminate=True) self.assertIn('tasks unknown', r['ok']) finally: worker_state.reserved_requests.discard(request) diff --git a/celery/worker/control.py b/celery/worker/control.py index 69bd42d00..669f047d4 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -54,10 +54,12 @@ def query_task(state, ids, **kwargs): ids = maybe_list(ids) return dict({ req.id: ('reserved', req.info()) - for req in _find_requests_by_id(ids, worker_state.reserved_requests) + for req in _find_requests_by_id( + ids, state.tset(worker_state.reserved_requests)) }, **{ req.id: ('active', req.info()) - for req in _find_requests_by_id(ids, worker_state.active_requests) + for req in _find_requests_by_id( + ids, state.tset(worker_state.active_requests)) }) @@ -76,7 +78,7 @@ def revoke(state, task_id, terminate=False, signal=None, **kwargs): # so need to consume the items first, then terminate after. requests = set(_find_requests_by_id( task_ids, - worker_state.reserved_requests, + state.tset(worker_state.reserved_requests), )) for request in requests: if request.id not in terminated: @@ -197,7 +199,10 @@ def prepare_entries(): @Panel.register def dump_reserved(state, safe=False, **kwargs): - reserved = worker_state.reserved_requests - worker_state.active_requests + reserved = ( + state.tset(worker_state.reserved_requests) - + state.tset(worker_state.active_requests) + ) if not reserved: return [] return [request.info(safe=safe) for request in reserved] @@ -206,7 +211,7 @@ def dump_reserved(state, safe=False, **kwargs): @Panel.register def dump_active(state, safe=False, **kwargs): return [request.info(safe=safe) - for request in worker_state.active_requests] + for request in state.tset(worker_state.active_requests)] @Panel.register diff --git a/celery/worker/pidbox.py b/celery/worker/pidbox.py index 4a5ae1704..72bdd3714 100644 --- a/celery/worker/pidbox.py +++ b/celery/worker/pidbox.py @@ -7,6 +7,7 @@ from kombu.utils.encoding import safe_str from celery.datastructures import AttributeDict +from celery.utils.functional import pass1 from celery.utils.log import get_logger from . import control @@ -26,7 +27,11 @@ def __init__(self, c): self.node = c.app.control.mailbox.Node( safe_str(c.hostname), handlers=control.Panel.data, - state=AttributeDict(app=c.app, hostname=c.hostname, consumer=c), + state=AttributeDict( + app=c.app, + hostname=c.hostname, + consumer=c, + tset=pass1 if c.controller.use_eventloop else set), ) self._forward_clock = self.c.app.clock.forward From 59968352cb2ee3be1a5a8f5b5f28d2ca00d5caff Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 14:03:37 -0800 Subject: [PATCH 0896/1103] Removes reference to @periodic_task in FAQ --- docs/faq.rst | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index c2ae478d5..cf45f5f80 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -810,19 +810,9 @@ Can I schedule tasks to execute at a specific time? **Answer**: Yes. You can use the `eta` argument of :meth:`Task.apply_async`. -Or to schedule a periodic task at a specific time, use the -:class:`celery.schedules.crontab` schedule behavior: +See also :ref:`guide-beat`. -.. code-block:: python - - from celery.schedules import crontab - from celery.task import periodic_task - - @periodic_task(run_every=crontab(hour=7, minute=30, day_of_week="mon")) - def every_monday_morning(): - print("This is run every Monday morning at 7:30") - .. _faq-safe-worker-shutdown: How can I safely shut down the worker? From 830e216835e6d35bae4f3c93e9964cca4afabc37 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 14:07:25 -0800 Subject: [PATCH 0897/1103] Stress: Use pyamqp:// by default --- funtests/stress/stress/templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index 91f7d53f9..75118d06f 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -51,7 +51,7 @@ def template_names(): @template() class default(object): accept_content = ['json'] - broker_url = os.environ.get('CSTRESS_BROKER', 'amqp://') + broker_url = os.environ.get('CSTRESS_BROKER', 'pyamqp://') broker_heartbeat = 30 result_backend = os.environ.get('CSTRESS_BACKEND', 'rpc://') result_serializer = 'json' From 91ee16b4fbfd9485e452293a27da54729a3ce86e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 14:15:16 -0800 Subject: [PATCH 0898/1103] Worker: Removes pickle deprecated startup warning --- celery/apps/worker.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/celery/apps/worker.py b/celery/apps/worker.py index af1ec025d..49905d926 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -44,26 +44,6 @@ is_jython = sys.platform.startswith('java') is_pypy = hasattr(sys, 'pypy_version_info') -W_PICKLE_DEPRECATED = """ -Starting from version 4.0 Celery will refuse to accept pickle by default. - -The pickle serializer is a security concern as it may give attackers -the ability to execute any command. It's important to secure -your broker from unauthorized access when using pickle, so we think -that enabling pickle should require a deliberate action and not be -the default choice. - -If you depend on pickle then you should set a setting to disable this -warning and to be sure that everything will continue working -when you upgrade to Celery 4.0:: - - accept_content = ['pickle', 'json', 'msgpack', 'yaml'] - -You must only enable the serializers that you will actually use. - -""" - - def active_thread_count(): from threading import enumerate return sum(1 for t in enumerate() @@ -159,9 +139,6 @@ def on_start(self): sender=self.hostname, instance=self, conf=app.conf, ) - if not app.conf.value_set_for('accept_content'): # pragma: no cover - warnings.warn(CDeprecationWarning(W_PICKLE_DEPRECATED)) - if self.purge: self.purge_messages() From a0269898e54d6d4b6d10b0b9038bb67a23523b68 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 14:31:55 -0800 Subject: [PATCH 0899/1103] Chain.link_error should not overwrite individual errbacks. Closes #2547 --- celery/canvas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index b00df8e7d..1d25cc9ce 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -527,7 +527,7 @@ def prepare_steps(self, args, tasks, task.set_parent_id(parent_id) if link_error: - task.set(link_error=link_error) + task.link_error(link_error) tasks.append(task) results.append(res) From 2cc25f53b2a553d3a15851ede223d7395aa2f9ff Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 14:49:31 -0800 Subject: [PATCH 0900/1103] Ability to configure maxlen of result/args/kwargs repr. Closes #2540 --- celery/app/amqp.py | 11 +++++++++-- celery/app/task.py | 3 +++ celery/app/trace.py | 4 +++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index c6ab2e241..d8c9e1324 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -235,6 +235,13 @@ class AMQP(object): # and instead send directly to the queue named in the routing key. autoexchange = None + #: Max size of positional argument representation used for + #: logging purposes. + argsrepr_maxsize = 1024 + + #: Max size of keyword argument representation used for logging purposes. + kwargsrepr_maxsize = 1024 + def __init__(self, app): self.app = app self.task_protocols = { @@ -318,8 +325,8 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, eta = eta and eta.isoformat() expires = expires and expires.isoformat() - argsrepr = saferepr(args) - kwargsrepr = saferepr(kwargs) + argsrepr = saferepr(args, self.argsrepr_maxsize) + kwargsrepr = saferepr(kwargs, self.kwargsrepr_maxsize) if JSON_NEEDS_UNICODE_KEYS: # pragma: no cover if callbacks: diff --git a/celery/app/task.py b/celery/app/task.py index e14d4e625..9e196d36c 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -244,6 +244,9 @@ class Task(object): #: Default task expiry time. expires = None + #: Max length of result representation used in logs and events. + resultrepr_maxsize = 1024 + #: Task request stack, the current request will be the topmost. request_stack = None diff --git a/celery/app/trace.py b/celery/app/trace.py index d887e57f0..cb07f84ad 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -37,6 +37,7 @@ from celery.five import monotonic from celery.utils.log import get_logger from celery.utils.objects import mro_lookup +from celery.utils.saferepr import saferepr from celery.utils.serialization import ( get_pickleable_exception, get_pickled_exception, get_pickleable_etype, ) @@ -292,6 +293,7 @@ def build_tracer(name, task, loader=None, hostname=None, store_errors=True, push_task = _task_stack.push pop_task = _task_stack.pop _does_info = logger.isEnabledFor(logging.INFO) + resultrepr_maxsize = task.resultrepr_maxsize prerun_receivers = signals.task_prerun.receivers postrun_receivers = signals.task_postrun.receivers @@ -423,7 +425,7 @@ def trace_task(uuid, args, kwargs, request=None): send_success(sender=task, result=retval) if _does_info: T = monotonic() - time_start - Rstr = truncate(safe_repr(R), 256) + Rstr = saferepr(R, resultrepr_maxsize) info(LOG_SUCCESS, { 'id': uuid, 'name': name, 'return_value': Rstr, 'runtime': T, From 9881af4b5108a6aa422f3de1d2cef5312693e1a5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 14:54:13 -0800 Subject: [PATCH 0901/1103] flakes --- celery/apps/worker.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/celery/apps/worker.py b/celery/apps/worker.py index 49905d926..a67389bd8 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -16,7 +16,6 @@ import os import platform as _platform import sys -import warnings from functools import partial @@ -26,9 +25,7 @@ from celery import VERSION_BANNER, platforms, signals from celery.app import trace -from celery.exceptions import ( - CDeprecationWarning, WorkerShutdown, WorkerTerminate, -) +from celery.exceptions import WorkerShutdown, WorkerTerminate from celery.five import string, string_t from celery.loaders.app import AppLoader from celery.platforms import EX_FAILURE, EX_OK, check_privileges @@ -44,6 +41,7 @@ is_jython = sys.platform.startswith('java') is_pypy = hasattr(sys, 'pypy_version_info') + def active_thread_count(): from threading import enumerate return sum(1 for t in enumerate() From 8866282482942cb3ebcb214dca25fa0cc2cbb284 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 15:04:05 -0800 Subject: [PATCH 0902/1103] Fixes build --- celery/canvas.py | 4 +++- celery/tests/worker/test_loops.py | 2 +- celery/tests/worker/test_worker.py | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 1d25cc9ce..f737068f5 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -21,6 +21,7 @@ from itertools import chain as _chain from kombu.utils import cached_property, fxrange, reprcall, uuid +from kombu.utils.functional import maybe_list from celery._state import current_app from celery.local import try_import @@ -527,7 +528,8 @@ def prepare_steps(self, args, tasks, task.set_parent_id(parent_id) if link_error: - task.link_error(link_error) + for errback in maybe_list(link_error): + task.link_error(errback) tasks.append(task) results.append(res) diff --git a/celery/tests/worker/test_loops.py b/celery/tests/worker/test_loops.py index ada763873..2f08f9866 100644 --- a/celery/tests/worker/test_loops.py +++ b/celery/tests/worker/test_loops.py @@ -61,7 +61,7 @@ def __init__(self, app, heartbeat=None, on_task_message=None, self.Hub = self.hub self.blueprint.state = RUN # need this for create_task_handler - _consumer = Consumer(Mock(), timer=Mock(), app=app) + _consumer = Consumer(Mock(), timer=Mock(), controller=Mock(), app=app) _consumer.on_task_message = on_task_message or [] self.obj.create_task_handler = _consumer.create_task_handler self.on_unknown_message = self.obj.on_unknown_message = Mock( diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index b65663e2d..e018d51dc 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -62,6 +62,7 @@ def __init__(self, *args, **kwargs): kwargs.setdefault('without_mingle', True) # disable Mingle step kwargs.setdefault('without_gossip', True) # disable Gossip step kwargs.setdefault('without_heartbeat', True) # disable Heart step + kwargs.setdefault('controller', Mock()) super(Consumer, self).__init__(*args, **kwargs) @@ -71,6 +72,7 @@ class _MyKombuConsumer(Consumer): def __init__(self, *args, **kwargs): kwargs.setdefault('pool', BasePool(2)) + kwargs.setdefault('controller', Mock()) super(_MyKombuConsumer, self).__init__(*args, **kwargs) def restart_heartbeat(self): From 2beef9fe589ee1dfb2fcc56d6b8ecfc66b5012d9 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 15:19:53 -0800 Subject: [PATCH 0903/1103] flakes --- celery/canvas.py | 1 - 1 file changed, 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index f737068f5..299b38e9c 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -21,7 +21,6 @@ from itertools import chain as _chain from kombu.utils import cached_property, fxrange, reprcall, uuid -from kombu.utils.functional import maybe_list from celery._state import current_app from celery.local import try_import From d80ad64dceaf443c4168593866b6b4de95c0aab3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 15:21:11 -0800 Subject: [PATCH 0904/1103] WorkController.__repr__ failed if not fully setup. Closes #2514 --- celery/worker/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/worker/__init__.py b/celery/worker/__init__.py index f038c01c1..e85721b95 100644 --- a/celery/worker/__init__.py +++ b/celery/worker/__init__.py @@ -332,7 +332,8 @@ def stats(self): def __repr__(self): return ''.format( - self=self, state=self.blueprint.human_state(), + self=self, + state=self.blueprint.human_state() if self.blueprint else 'INIT', ) def __str__(self): From e6fb53488e017ce3b4bd9aab1437b0ea1c2ef9fc Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 15:35:26 -0800 Subject: [PATCH 0905/1103] Worker: Now calls errbacks for tasks even when result stored by parent process. Closes #2510 --- celery/app/trace.py | 26 +++++++++++++------------- celery/backends/base.py | 16 ++++++++++++---- celery/tests/backends/test_base.py | 2 ++ celery/tests/tasks/test_trace.py | 1 + celery/worker/request.py | 9 ++++++++- 5 files changed, 36 insertions(+), 18 deletions(-) diff --git a/celery/app/trace.py b/celery/app/trace.py index cb07f84ad..fa0599cd0 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -141,15 +141,17 @@ def __init__(self, state, retval=None): self.state = state self.retval = retval - def handle_error_state(self, task, req, eager=False): + def handle_error_state(self, task, req, + eager=False, call_errbacks=True): store_errors = not eager if task.ignore_result: store_errors = task.store_errors_even_if_ignored - return { RETRY: self.handle_retry, FAILURE: self.handle_failure, - }[self.state](task, req, store_errors=store_errors) + }[self.state](task, req, + store_errors=store_errors, + call_errbacks=call_errbacks) def handle_reject(self, task, req, **kwargs): self._log_error(task, req, ExceptionInfo()) @@ -157,7 +159,7 @@ def handle_reject(self, task, req, **kwargs): def handle_ignore(self, task, req, **kwargs): self._log_error(task, req, ExceptionInfo()) - def handle_retry(self, task, req, store_errors=True): + def handle_retry(self, task, req, store_errors=True, **kwargs): """Handle retry exception.""" # the exception raised is the Retry semi-predicate, # and it's exc' attribute is the original exception raised (if any). @@ -180,7 +182,7 @@ def handle_retry(self, task, req, store_errors=True): finally: del(tb) - def handle_failure(self, task, req, store_errors=True): + def handle_failure(self, task, req, store_errors=True, call_errbacks=True): """Handle exception.""" type_, _, tb = sys.exc_info() try: @@ -189,7 +191,9 @@ def handle_failure(self, task, req, store_errors=True): einfo.exception = get_pickleable_exception(einfo.exception) einfo.type = get_pickleable_etype(einfo.type) task.backend.mark_as_failure( - req.id, exc, einfo.traceback, req, store_errors, + req.id, exc, einfo.traceback, + request=req, store_result=store_errors, + call_errbacks=call_errbacks, ) task.on_failure(exc, req.id, req.args, req.kwargs, einfo) signals.task_failure.send(sender=task, task_id=req.id, @@ -306,13 +310,9 @@ def on_error(request, exc, uuid, state=FAILURE, call_errbacks=True): if propagate: raise I = Info(state, exc) - R = I.handle_error_state(task, request, eager=eager) - if call_errbacks: - root_id = request.root_id or uuid - group( - [signature(errback, app=app) - for errback in request.errbacks or []], app=app, - ).apply_async((uuid,), parent_id=uuid, root_id=root_id) + R = I.handle_error_state( + task, request, eager=eager, call_errbacks=call_errbacks, + ) return I, R, I.state, I.retval def trace_task(uuid, args, kwargs, request=None): diff --git a/celery/backends/base.py b/celery/backends/base.py index c9ecacc2e..a82ac4060 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -26,7 +26,7 @@ from kombu.utils.encoding import bytes_to_str, ensure_bytes, from_utf8 from celery import states -from celery import current_app, maybe_signature +from celery import current_app, group, maybe_signature from celery.app import current_task from celery.exceptions import ChordError, TimeoutError, TaskRevokedError from celery.five import items @@ -121,14 +121,22 @@ def mark_as_done(self, task_id, result, self.on_chord_part_return(request, state, result) def mark_as_failure(self, task_id, exc, - traceback=None, request=None, store_result=True, + traceback=None, request=None, + store_result=True, call_errbacks=True, state=states.FAILURE): """Mark task as executed with failure. Stores the exception.""" if store_result: self.store_result(task_id, exc, state, traceback=traceback, request=request) - if request and request.chord: - self.on_chord_part_return(request, state, exc) + if request: + if request.chord: + self.on_chord_part_return(request, state, exc) + if call_errbacks: + root_id = request.root_id or task_id + group( + [self.app.signature(errback) + for errback in request.errbacks or []], app=self.app, + ).apply_async((task_id,), parent_id=task_id, root_id=root_id) def mark_as_revoked(self, task_id, reason='', request=None, store_result=True, state=states.REVOKED): diff --git a/celery/tests/backends/test_base.py b/celery/tests/backends/test_base.py index 273600c60..86b4f1b4f 100644 --- a/celery/tests/backends/test_base.py +++ b/celery/tests/backends/test_base.py @@ -270,6 +270,7 @@ def test_mark_as_failure__chord(self): b = BaseBackend(app=self.app) b._store_result = Mock() request = Mock(name='request') + request.errbacks = [] b.on_chord_part_return = Mock() exc = KeyError() b.mark_as_failure('id', exc, request=request) @@ -279,6 +280,7 @@ def test_mark_as_revoked__chord(self): b = BaseBackend(app=self.app) b._store_result = Mock() request = Mock(name='request') + request.errbacks = [] b.on_chord_part_return = Mock() b.mark_as_revoked('id', 'revoked', request=request) b.on_chord_part_return.assert_called_with(request, states.REVOKED, ANY) diff --git a/celery/tests/tasks/test_trace.py b/celery/tests/tasks/test_trace.py index a8090ab2d..47563a73b 100644 --- a/celery/tests/tasks/test_trace.py +++ b/celery/tests/tasks/test_trace.py @@ -319,6 +319,7 @@ def test_handle_error_state(self): x.handle_failure.assert_called_with( self.add_cast, self.add_cast.request, store_errors=self.add_cast.store_errors_even_if_ignored, + call_errbacks=True, ) @patch('celery.app.trace.ExceptionInfo') diff --git a/celery/worker/request.py b/celery/worker/request.py index 1c01d5a79..e22489713 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -466,11 +466,18 @@ def _payload(self): @cached_property def chord(self): - # used by backend.on_chord_part_return when failures reported + # used by backend.mark_as_failure when failure is reported # by parent process _, _, embed = self._payload return embed.get('chord') + @cached_property + def errbacks(self): + # used by backend.mark_as_failure when failure is reported + # by parent process + _, _, embed = self._payload + return embed.get('errbacks') + @cached_property def group(self): # used by backend.on_chord_part_return when failures reported From 4b009835fd8b53b5ca415213eda871b024a9b25e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 15:57:52 -0800 Subject: [PATCH 0906/1103] Removes unused but confusing AsyncResult.task_name. --- celery/result.py | 9 ++++----- celery/tests/tasks/test_result.py | 3 +-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/celery/result.py b/celery/result.py index 472511b7f..4c1e14a1e 100644 --- a/celery/result.py +++ b/celery/result.py @@ -76,7 +76,8 @@ class AsyncResult(ResultBase): #: The task result backend to use. backend = None - def __init__(self, id, backend=None, task_name=None, + def __init__(self, id, backend=None, + task_name=None, # deprecated app=None, parent=None): if id is None: raise ValueError( @@ -84,7 +85,6 @@ def __init__(self, id, backend=None, task_name=None, self.app = app_or_default(app or self.app) self.id = id self.backend = backend or self.app.backend - self.task_name = task_name self.parent = parent self._cache = None @@ -306,14 +306,14 @@ def __ne__(self, other): def __copy__(self): return self.__class__( - self.id, self.backend, self.task_name, self.app, self.parent, + self.id, self.backend, None, self.app, self.parent, ) def __reduce__(self): return self.__class__, self.__reduce_args__() def __reduce_args__(self): - return self.id, self.backend, self.task_name, None, self.parent + return self.id, self.backend, None, None, self.parent def __del__(self): self._cache = None @@ -826,7 +826,6 @@ def restore(self, id, backend=None): class EagerResult(AsyncResult): """Result that we know has already been executed.""" - task_name = None def __init__(self, id, ret_value, state, traceback=None): self.id = id diff --git a/celery/tests/tasks/test_result.py b/celery/tests/tasks/test_result.py index 433e081b4..789e81c67 100644 --- a/celery/tests/tasks/test_result.py +++ b/celery/tests/tasks/test_result.py @@ -174,10 +174,9 @@ def test_eq_not_implemented(self): @depends_on_current_app def test_reduce(self): - a1 = self.app.AsyncResult('uuid', task_name=self.mytask.name) + a1 = self.app.AsyncResult('uuid') restored = pickle.loads(pickle.dumps(a1)) self.assertEqual(restored.id, 'uuid') - self.assertEqual(restored.task_name, self.mytask.name) a2 = self.app.AsyncResult('uuid') self.assertEqual(pickle.loads(pickle.dumps(a2)).id, 'uuid') From 62f5bf04144220478335a52e11664f450ba242ab Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 15:59:59 -0800 Subject: [PATCH 0907/1103] task.http: JSON must always be unicode. Closes #2499 --- celery/task/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/task/http.py b/celery/task/http.py index 8d5a5e51d..0c1246185 100644 --- a/celery/task/http.py +++ b/celery/task/http.py @@ -17,6 +17,7 @@ from urlparse import urlparse, parse_qsl # noqa from kombu.utils import json +from kombu.utils.encoding import bytes_to_str from celery import shared_task, __version__ as celery_version from celery.five import items, reraise @@ -155,7 +156,7 @@ def dispatch(self): else: params = urlencode(utf8dict(items(self.task_kwargs))) raw_response = self.make_request(str(url), self.method, params) - return extract_response(raw_response) + return extract_response(bytes_to_str(raw_response)) @property def http_headers(self): From 097dd74f677068c23833be4896ba5b658c0bf4d2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 16:08:58 -0800 Subject: [PATCH 0908/1103] Worker direct queues are no longer auto_delete=True (INCOMPATIBLE) Closes #2492 --- celery/utils/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/celery/utils/__init__.py b/celery/utils/__init__.py index 5661f6dfd..bfd96e8fb 100644 --- a/celery/utils/__init__.py +++ b/celery/utils/__init__.py @@ -52,10 +52,10 @@ MP_MAIN_FILE = os.environ.get('MP_MAIN_FILE') or None #: Exchange for worker direct queues. -WORKER_DIRECT_EXCHANGE = Exchange('C.dq') +WORKER_DIRECT_EXCHANGE = Exchange('C.dq2') #: Format for worker direct queue names. -WORKER_DIRECT_QUEUE_FORMAT = '{hostname}.dq' +WORKER_DIRECT_QUEUE_FORMAT = '{hostname}.dq2' #: Separator for worker node name and hostname. NODENAME_SEP = '@' @@ -75,9 +75,11 @@ def worker_direct(hostname): """ if isinstance(hostname, Queue): return hostname - return Queue(WORKER_DIRECT_QUEUE_FORMAT.format(hostname=hostname), - WORKER_DIRECT_EXCHANGE, - hostname, auto_delete=True) + return Queue( + WORKER_DIRECT_QUEUE_FORMAT.format(hostname=hostname), + WORKER_DIRECT_EXCHANGE, + hostname, + ) def warn_deprecated(description=None, deprecation=None, From ee650d0c2f4ab35437026e04a358b09709b330b0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 16:15:33 -0800 Subject: [PATCH 0909/1103] Signal: cannot use id() for sender=str. Closes #2475 --- celery/utils/dispatch/signal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/celery/utils/dispatch/signal.py b/celery/utils/dispatch/signal.py index 7d4b337a9..36f042e01 100644 --- a/celery/utils/dispatch/signal.py +++ b/celery/utils/dispatch/signal.py @@ -5,7 +5,7 @@ import weakref from . import saferef -from celery.five import range +from celery.five import range, text_t from celery.local import PromiseProxy, Proxy __all__ = ['Signal'] @@ -16,6 +16,9 @@ def _make_id(target): # pragma: no cover if isinstance(target, Proxy): target = target._get_current_object() + if isinstance(target, (bytes, text_t)): + # see Issue #2475 + return target if hasattr(target, '__func__'): return (id(target.__self__), id(target.__func__)) return id(target) From 4a279d41ee2000a75e7a8efc677c18c8778fb183 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 16:17:08 -0800 Subject: [PATCH 0910/1103] Fixes typo in calling guide. Closes #2479 --- docs/userguide/calling.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index 47cc7e1af..bd0e8e0c3 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -449,7 +449,7 @@ Though this particular example is much better expressed as a group: >>> from celery import group >>> numbers = [(2, 2), (4, 4), (8, 8), (16, 16)] - >>> res = group(add.s(i) for i in numbers).apply_async() + >>> res = group(add.s(i, j) for i, j in numbers).apply_async() >>> res.get() [4, 8, 16, 32] From c136e7f0cbefe093bb07c9413dbd5a4d7f8d380d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 16:39:13 -0800 Subject: [PATCH 0911/1103] Disable events completely if without-gossip+without-heartbeat. Closes #2483 --- celery/worker/consumer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index ba558f1aa..fa62f2251 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -542,9 +542,14 @@ def info(self, c, params='N/A'): class Events(bootsteps.StartStopStep): requires = (Connection,) - def __init__(self, c, send_events=None, **kwargs): - self.send_events = True + def __init__(self, c, send_events=True, + without_heartbeat=False, without_gossip=False, **kwargs): self.groups = None if send_events else ['worker'] + self.send_events = ( + send_events or + not without_gossip or + not without_heartbeat + ) c.event_dispatcher = None def start(self, c): From 151696c5166f68539c0bf661d6a2837e43677d23 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 17:45:08 -0800 Subject: [PATCH 0912/1103] Disables the local client result cache by default (Issue #2461) --- celery/app/defaults.py | 2 +- docs/configuration.rst | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index ae40b2ae5..a6f9b8b69 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -173,7 +173,7 @@ def __repr__(self): backend=Option(type='string'), cache_max=Option( - 100, + -1, type='int', old={'celery_max_cached_results'}, ), compression=Option(type='str'), diff --git a/docs/configuration.rst b/docs/configuration.rst index 0772ade1e..1a4ebe880 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -594,12 +594,16 @@ Default is to expire after 1 day. result_cache_max ~~~~~~~~~~~~~~~~ -Result backends caches ready results used by the client. +Enables client caching of results, which can be useful for the old "amqp" +backend where the result is unavailable as soon as one result instance +consumes it. This is the total number of results to cache before older results are evicted. -The default is 5000. 0 or None means no limit, and a value of :const:`-1` +A value of 0 or None means no limit, and a value of :const:`-1` will disable the cache. +Disabled by default. + .. _conf-database-result-backend: Database backend settings From 46c17e99092441c4ba69f9bcd4ce5a0e23754e06 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 23:04:44 -0800 Subject: [PATCH 0913/1103] Cache the value of gethostname --- celery/app/trace.py | 8 ++++---- celery/bin/multi.py | 5 ++--- celery/tests/bin/test_base.py | 2 +- celery/tests/bin/test_multi.py | 10 +++++----- celery/utils/__init__.py | 11 +++++++---- celery/worker/consumer.py | 4 ++-- celery/worker/request.py | 5 ++--- 7 files changed, 23 insertions(+), 22 deletions(-) diff --git a/celery/app/trace.py b/celery/app/trace.py index fa0599cd0..7fd459f01 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -17,7 +17,6 @@ import logging import os -import socket import sys from collections import namedtuple @@ -35,6 +34,7 @@ from celery.app.task import Task as BaseTask, Context from celery.exceptions import Ignore, Reject, Retry, InvalidTaskError from celery.five import monotonic +from celery.utils import gethostname from celery.utils.log import get_logger from celery.utils.objects import mro_lookup from celery.utils.saferepr import saferepr @@ -273,7 +273,7 @@ def build_tracer(name, task, loader=None, hostname=None, store_errors=True, track_started = task.track_started track_started = not eager and (task.track_started and not ignore_result) publish_result = not eager and not ignore_result - hostname = hostname or socket.gethostname() + hostname = hostname or gethostname() loader_task_init = loader.on_task_init loader_cleanup = loader.on_process_cleanup @@ -489,7 +489,7 @@ def _trace_task_ret(name, uuid, request, body, content_type, ) else: args, kwargs, embed = body - hostname = socket.gethostname() + hostname = gethostname() request.update({ 'args': args, 'kwargs': kwargs, 'hostname': hostname, 'is_eager': False, @@ -537,7 +537,7 @@ def report_internal_error(task, exc): def setup_worker_optimizations(app, hostname=None): global trace_task_ret - hostname = hostname or socket.gethostname() + hostname = hostname or gethostname() # make sure custom Task.__call__ methods that calls super # will not mess up the request/task stack. diff --git a/celery/bin/multi.py b/celery/bin/multi.py index 1191ffd94..39919c42a 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -100,7 +100,6 @@ import os import shlex import signal -import socket import sys from collections import OrderedDict, defaultdict, namedtuple @@ -115,7 +114,7 @@ from celery.five import items from celery.platforms import Pidfile, IS_WINDOWS from celery.utils import term -from celery.utils import host_format, node_format, nodesplit +from celery.utils import gethostname, host_format, node_format, nodesplit from celery.utils.text import pluralize __all__ = ['MultiTool'] @@ -480,7 +479,7 @@ def multi_args(p, cmd='celery worker', append='', prefix='', suffix=''): cmd = options.pop('--cmd', cmd) append = options.pop('--append', append) hostname = options.pop('--hostname', - options.pop('-n', socket.gethostname())) + options.pop('-n', gethostname())) prefix = options.pop('--prefix', prefix) or '' suffix = options.pop('--suffix', suffix) or hostname suffix = '' if suffix in ('""', "''") else suffix diff --git a/celery/tests/bin/test_base.py b/celery/tests/bin/test_base.py index fd6657f40..f8a8b5e58 100644 --- a/celery/tests/bin/test_base.py +++ b/celery/tests/bin/test_base.py @@ -258,7 +258,7 @@ def test_ask(self): def test_host_format(self): cmd = MockCommand(app=self.app) - with patch('socket.gethostname') as hn: + with patch('celery.utils.gethostname') as hn: hn.return_value = 'blacktron.example.com' self.assertEqual(cmd.host_format(''), '') self.assertEqual( diff --git a/celery/tests/bin/test_multi.py b/celery/tests/bin/test_multi.py index d99052058..5e18a9b90 100644 --- a/celery/tests/bin/test_multi.py +++ b/celery/tests/bin/test_multi.py @@ -67,7 +67,7 @@ def test_parse(self): class test_multi_args(AppCase): - @patch('socket.gethostname') + @patch('celery.bin.multi.gethostname') def test_parse(self, gethostname): gethostname.return_value = 'example.com' p = NamespacedOptionParser([ @@ -298,7 +298,7 @@ def read_pid(self): Pidfile.side_effect = pids @patch('celery.bin.multi.Pidfile') - @patch('socket.gethostname') + @patch('celery.bin.multi.gethostname') def test_getpids(self, gethostname, Pidfile): gethostname.return_value = 'e.com' self.prepare_pidfile_for_getpids(Pidfile) @@ -336,7 +336,7 @@ def test_getpids(self, gethostname, Pidfile): nodes = self.t.getpids(p, 'celery worker', callback=None) @patch('celery.bin.multi.Pidfile') - @patch('socket.gethostname') + @patch('celery.bin.multi.gethostname') @patch('celery.bin.multi.sleep') def test_shutdown_nodes(self, slepp, gethostname, Pidfile): gethostname.return_value = 'e.com' @@ -415,7 +415,7 @@ def test_show(self): self.t.show(['foo', 'bar', 'baz'], 'celery worker') self.assertTrue(self.fh.getvalue()) - @patch('socket.gethostname') + @patch('celery.bin.multi.gethostname') def test_get(self, gethostname): gethostname.return_value = 'e.com' self.t.get(['xuzzy@e.com', 'foo', 'bar', 'baz'], 'celery worker') @@ -423,7 +423,7 @@ def test_get(self, gethostname): self.t.get(['foo@e.com', 'foo', 'bar', 'baz'], 'celery worker') self.assertTrue(self.fh.getvalue()) - @patch('socket.gethostname') + @patch('celery.bin.multi.gethostname') def test_names(self, gethostname): gethostname.return_value = 'e.com' self.t.names(['foo', 'bar', 'baz'], 'celery worker') diff --git a/celery/utils/__init__.py b/celery/utils/__init__.py index bfd96e8fb..fdbb21ec0 100644 --- a/celery/utils/__init__.py +++ b/celery/utils/__init__.py @@ -26,6 +26,8 @@ from celery.exceptions import CPendingDeprecationWarning, CDeprecationWarning from celery.five import WhateverIO, items, reraise, string_t +from .functional import memoize + __all__ = ['worker_direct', 'warn_deprecated', 'deprecated', 'lpmerge', 'is_iterable', 'isatty', 'cry', 'maybe_reraise', 'strtobool', 'jsonify', 'gen_task_name', 'nodename', 'nodesplit', @@ -33,7 +35,6 @@ PY3 = sys.version_info[0] == 3 - PENDING_DEPRECATION_FMT = """ {description} is scheduled for deprecation in \ version {deprecation} and removal in version v{removal}. \ @@ -63,6 +64,8 @@ NODENAME_DEFAULT = 'celery' RE_FORMAT = re.compile(r'%(\w)') +gethostname = memoize(1, Cache=dict)(socket.gethostname) + def worker_direct(hostname): """Return :class:`kombu.Queue` that is a direct route to @@ -327,7 +330,7 @@ def nodename(name, hostname): def anon_nodename(hostname=None, prefix='gen'): return nodename(''.join([prefix, str(os.getpid())]), - hostname or socket.gethostname()) + hostname or gethostname()) def nodesplit(nodename): @@ -340,7 +343,7 @@ def nodesplit(nodename): def default_nodename(hostname): name, host = nodesplit(hostname or '') - return nodename(name or NODENAME_DEFAULT, host or socket.gethostname()) + return nodename(name or NODENAME_DEFAULT, host or gethostname()) def node_format(s, nodename, **extra): @@ -357,7 +360,7 @@ def _fmt_process_index(prefix='', default='0'): def host_format(s, host=None, name=None, **extra): - host = host or socket.gethostname() + host = host or gethostname() hname, _, domain = host.partition('.') name = name or hname keys = dict({ diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index fa62f2251..fbbc820ae 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -14,7 +14,6 @@ import kombu import logging import os -import socket from collections import defaultdict from functools import partial @@ -36,6 +35,7 @@ from celery.app.trace import build_tracer from celery.canvas import signature from celery.exceptions import InvalidTaskError, NotRegistered +from celery.utils import gethostname from celery.utils.functional import noop from celery.utils.log import get_logger from celery.utils.text import truncate @@ -172,7 +172,7 @@ def __init__(self, on_task_request, self.app = app self.controller = controller self.init_callback = init_callback - self.hostname = hostname or socket.gethostname() + self.hostname = hostname or gethostname() self.pid = os.getpid() self.pool = pool self.timer = timer diff --git a/celery/worker/request.py b/celery/worker/request.py index e22489713..020454b9f 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -10,7 +10,6 @@ from __future__ import absolute_import, unicode_literals import logging -import socket import sys from datetime import datetime @@ -27,7 +26,7 @@ ) from celery.five import string from celery.platforms import signals as _signals -from celery.utils import cached_property +from celery.utils import cached_property, gethostname from celery.utils.functional import noop from celery.utils.log import get_logger from celery.utils.timeutils import maybe_iso8601, timezone, maybe_make_aware @@ -120,7 +119,7 @@ def __init__(self, message, on_ack=noop, self.kwargsrepr = headers.get('kwargsrepr', '') self.on_ack = on_ack self.on_reject = on_reject - self.hostname = hostname or socket.gethostname() + self.hostname = hostname or gethostname() self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[type] From 0d10f9c71b055cec6a3b699d6a71948be988dd30 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Dec 2015 23:04:55 -0800 Subject: [PATCH 0914/1103] Fixes build --- celery/tests/backends/test_amqp.py | 3 +++ celery/tests/tasks/test_result.py | 1 + 2 files changed, 4 insertions(+) diff --git a/celery/tests/backends/test_amqp.py b/celery/tests/backends/test_amqp.py index ac7a1c0d1..640733f1c 100644 --- a/celery/tests/backends/test_amqp.py +++ b/celery/tests/backends/test_amqp.py @@ -29,6 +29,9 @@ def __init__(self, data): class test_AMQPBackend(AppCase): + def setup(self): + self.app.conf.result_cache_max = 100 + def create_backend(self, **opts): opts = dict(dict(serializer='pickle', persistent=True), **opts) return AMQPBackend(self.app, **opts) diff --git a/celery/tests/tasks/test_result.py b/celery/tests/tasks/test_result.py index 789e81c67..bf39668c5 100644 --- a/celery/tests/tasks/test_result.py +++ b/celery/tests/tasks/test_result.py @@ -49,6 +49,7 @@ def make_mock_group(app, size=10): class test_AsyncResult(AppCase): def setup(self): + self.app.conf.result_cache_max = 100 self.app.conf.result_serializer = 'pickle' self.task1 = mock_task('task1', states.SUCCESS, 'the') self.task2 = mock_task('task2', states.SUCCESS, 'quick') From 1bcdfde9fc6dca5d8be0393d1046615d9502a2dd Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Dec 2015 13:34:24 -0800 Subject: [PATCH 0915/1103] Task protocol 2: Adds new "origin" message header for hostname of task sender --- celery/app/amqp.py | 5 ++++- celery/app/task.py | 1 + docs/internals/protocol.rst | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index d8c9e1324..bcd3c8139 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -24,6 +24,7 @@ from celery import signals from celery.five import items, string_t from celery.local import try_import +from celery.utils import anon_nodename from celery.utils.saferepr import saferepr from celery.utils.text import indent as textindent from celery.utils.timeutils import maybe_make_aware, to_utc @@ -303,7 +304,8 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, callbacks=None, errbacks=None, reply_to=None, time_limit=None, soft_time_limit=None, create_sent_event=False, root_id=None, parent_id=None, - shadow=None, chain=None, now=None, timezone=None): + shadow=None, chain=None, now=None, timezone=None, + origin=None): args = args or () kwargs = kwargs or {} if not isinstance(args, (list, tuple)): @@ -350,6 +352,7 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, 'parent_id': parent_id, 'argsrepr': argsrepr, 'kwargsrepr': kwargsrepr, + 'origin': origin or anon_nodename() }, properties={ 'correlation_id': task_id, diff --git a/celery/app/task.py b/celery/app/task.py index 9e196d36c..5aac03058 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -92,6 +92,7 @@ class Context(object): callbacks = None errbacks = None timelimit = None + origin = None _children = None # see property _protected = 0 diff --git a/docs/internals/protocol.rst b/docs/internals/protocol.rst index 623d9b184..8a6922d65 100644 --- a/docs/internals/protocol.rst +++ b/docs/internals/protocol.rst @@ -48,6 +48,7 @@ Definition 'timelimit': (soft, hard), 'argsrepr': str repr(args), 'kwargsrepr': str repr(kwargs), + 'origin': str nodename, } body = ( @@ -70,6 +71,10 @@ This example sends a task message using version 2 of the protocol: # chain: add(add(add(2, 2), 4), 8) == 2 + 2 + 4 + 8 + import json + import os + import socket + task_id = uuid() args = (2, 2) kwargs = {} @@ -80,6 +85,7 @@ This example sends a task message using version 2 of the protocol: 'task': 'proj.tasks.add', 'argsrepr': repr(args), 'kwargsrepr': repr(kwargs), + 'origin': '@'.join([os.getpid(), socket.gethostname()]) } properties={ 'correlation_id': task_id, From 8454428b0f17ef4fc2c6e61614eee8a9b539c371 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Dec 2015 13:52:15 -0800 Subject: [PATCH 0916/1103] Events: Adds new task-rejected event for basic.reject --- celery/events/state.py | 12 +++++++----- celery/states.py | 7 +++++-- celery/worker/request.py | 1 + docs/userguide/monitoring.rst | 10 ++++++++++ 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/celery/events/state.py b/celery/events/state.py index cfb12ecb9..19800f79a 100644 --- a/celery/events/state.py +++ b/celery/events/state.py @@ -205,14 +205,14 @@ def id(self): class Task(object): """Task State.""" name = received = sent = started = succeeded = failed = retried = \ - revoked = args = kwargs = eta = expires = retries = worker = result = \ - exception = timestamp = runtime = traceback = exchange = \ - routing_key = root_id = parent_id = client = None + revoked = rejected = args = kwargs = eta = expires = retries = \ + worker = result = exception = timestamp = runtime = traceback = \ + exchange = routing_key = root_id = parent_id = client = None state = states.PENDING clock = 0 _fields = ( - 'uuid', 'name', 'state', 'received', 'sent', 'started', + 'uuid', 'name', 'state', 'received', 'sent', 'started', 'rejected', 'succeeded', 'failed', 'retried', 'revoked', 'args', 'kwargs', 'eta', 'expires', 'retries', 'worker', 'result', 'exception', 'timestamp', 'runtime', 'traceback', 'exchange', 'routing_key', @@ -254,7 +254,7 @@ def event(self, type_, timestamp=None, local_received=None, fields=None, PENDING=states.PENDING, RECEIVED=states.RECEIVED, STARTED=states.STARTED, FAILURE=states.FAILURE, RETRY=states.RETRY, SUCCESS=states.SUCCESS, - REVOKED=states.REVOKED): + REVOKED=states.REVOKED, REJECTED=states.REJECTED): fields = fields or {} if type_ == 'sent': state, self.sent = PENDING, timestamp @@ -270,6 +270,8 @@ def event(self, type_, timestamp=None, local_received=None, fields=None, state, self.succeeded = SUCCESS, timestamp elif type_ == 'revoked': state, self.revoked = REVOKED, timestamp + elif type_ == 'rejected': + state, self.rejected = REJECTED, timestamp else: state = type_.upper() diff --git a/celery/states.py b/celery/states.py index 592c08b5f..0525375b2 100644 --- a/celery/states.py +++ b/celery/states.py @@ -72,6 +72,7 @@ 'REVOKED', 'STARTED', 'RECEIVED', + 'REJECTED', 'RETRY', 'PENDING'] @@ -126,7 +127,7 @@ def __le__(self, other): #: Task state is unknown (assumed pending since you know the id). PENDING = 'PENDING' -#: Task was received by a worker. +#: Task was received by a worker (only used in events). RECEIVED = 'RECEIVED' #: Task was started by a worker (:setting:`task_track_started`). STARTED = 'STARTED' @@ -136,13 +137,15 @@ def __le__(self, other): FAILURE = 'FAILURE' #: Task was revoked. REVOKED = 'REVOKED' +#: Task was rejected (only used in events). +REJECTED = 'REJECTED' #: Task is waiting for retry. RETRY = 'RETRY' IGNORED = 'IGNORED' REJECTED = 'REJECTED' READY_STATES = frozenset({SUCCESS, FAILURE, REVOKED}) -UNREADY_STATES = frozenset({PENDING, RECEIVED, STARTED, RETRY}) +UNREADY_STATES = frozenset({PENDING, RECEIVED, STARTED, REJECTED, RETRY}) EXCEPTION_STATES = frozenset({RETRY, FAILURE, REVOKED}) PROPAGATE_STATES = frozenset({FAILURE, REVOKED}) diff --git a/celery/worker/request.py b/celery/worker/request.py index 020454b9f..824965684 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -389,6 +389,7 @@ def reject(self, requeue=False): if not self.acknowledged: self.on_reject(logger, self.connection_errors, requeue) self.acknowledged = True + self.send_event('task-rejected', requeue=requeue) def info(self, safe=False): return {'id': self.id, diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index 8652f6bec..c3df06960 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -696,6 +696,16 @@ task-failed Sent if the execution of the task failed. +.. event:: task-rejected + +task-rejected +~~~~~~~~~~~~~ + +:signature: ``task-rejected(uuid, requeued)`` + +The task was rejected by the worker, possibly to be requeued or moved to a +dead letter queue. + .. event:: task-revoked task-revoked From ddc7de1fd900ab880f7359eae8602f35e42c0263 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Dec 2015 13:53:32 -0800 Subject: [PATCH 0917/1103] Worker: task.reject_on_worker_lost must not send task-failed event. --- celery/worker/request.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/worker/request.py b/celery/worker/request.py index 824965684..9bac2ec8b 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -365,6 +365,7 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): ) if reject: self.reject(requeue=requeue) + send_failed_event = False else: self.acknowledge() From b02ad4d3fab12d50123e00ec6cbb0a2c4147d976 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Dec 2015 15:40:18 -0800 Subject: [PATCH 0918/1103] Thread Pool: Set default app for all threads. Closes #2701 --- celery/concurrency/base.py | 5 +++-- celery/concurrency/threads.py | 3 +++ celery/worker/components.py | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/celery/concurrency/base.py b/celery/concurrency/base.py index 4913ffb27..4b2e7a15d 100644 --- a/celery/concurrency/base.py +++ b/celery/concurrency/base.py @@ -74,13 +74,14 @@ class BasePool(object): task_join_will_block = True body_can_be_buffer = False - def __init__(self, limit=None, putlocks=True, - forking_enable=True, callbacks_propagate=(), **options): + def __init__(self, limit=None, putlocks=True, forking_enable=True, + callbacks_propagate=(), app=None, **options): self.limit = limit self.putlocks = putlocks self.options = options self.forking_enable = forking_enable self.callbacks_propagate = callbacks_propagate + self.app = app def on_start(self): pass diff --git a/celery/concurrency/threads.py b/celery/concurrency/threads.py index fee901ecf..cb1d4b8d7 100644 --- a/celery/concurrency/threads.py +++ b/celery/concurrency/threads.py @@ -34,6 +34,9 @@ def __init__(self, *args, **kwargs): super(TaskPool, self).__init__(*args, **kwargs) def on_start(self): + # make sure all threads have the same current_app. + self.app.set_default() + self._pool = self.ThreadPool(self.limit) # threadpool stores all work requests until they are processed # we don't need this dict, and it occupies way too much memory. diff --git a/celery/worker/components.py b/celery/worker/components.py index 1856710a4..469db8995 100644 --- a/celery/worker/components.py +++ b/celery/worker/components.py @@ -170,6 +170,7 @@ def create(self, w, semaphore=None, max_restarts=None, forking_enable=forking_enable, semaphore=semaphore, sched_strategy=self.optimization, + app=w.app, ) _set_task_join_will_block(pool.task_join_will_block) return pool From 6b08111d64eaff4e64313d8c4ad3f9aa5b071ea0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Dec 2015 15:57:04 -0800 Subject: [PATCH 0919/1103] Docs: Routing: Link to AMQP spec. Closes #2858 --- docs/userguide/routing.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/userguide/routing.rst b/docs/userguide/routing.rst index 6e882ad70..0e72f406b 100644 --- a/docs/userguide/routing.rst +++ b/docs/userguide/routing.rst @@ -313,6 +313,8 @@ Related API commands Declares an exchange by name. + See :meth:`amqp:Channel.exchange_declare `. + :keyword passive: Passive means the exchange won't be created, but you can use this to check if the exchange already exists. @@ -327,22 +329,31 @@ Related API commands Declares a queue by name. + See :meth:`amqp:Channel.queue_declare ` + Exclusive queues can only be consumed from by the current connection. Exclusive also implies `auto_delete`. .. method:: queue.bind(queue_name, exchange_name, routing_key) Binds a queue to an exchange with a routing key. + Unbound queues will not receive messages, so this is necessary. + See :meth:`amqp:Channel.queue_bind ` + .. method:: queue.delete(name, if_unused=False, if_empty=False) Deletes a queue and its binding. + See :meth:`amqp:Channel.queue_delete ` + .. method:: exchange.delete(name, if_unused=False) Deletes an exchange. + See :meth:`amqp:Channel.exchange_delete ` + .. note:: Declaring does not necessarily mean "create". When you declare you From 66d0de753735248aeec4e992061850a39531f7c9 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Dec 2015 16:14:05 -0800 Subject: [PATCH 0920/1103] Autodiscover: Fixes error "Empty module name". Closes #2908 --- celery/loaders/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/celery/loaders/base.py b/celery/loaders/base.py index 02ec1624a..0223297eb 100644 --- a/celery/loaders/base.py +++ b/celery/loaders/base.py @@ -286,6 +286,8 @@ def find_related_module(package, related_name): importlib.import_module(package) except ImportError: package, _, _ = package.rpartition('.') + if not package: + raise try: pkg_path = importlib.import_module(package).__path__ From 208a2eeb521f5b9ffcae1e2b3527f4454a695f46 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Dec 2015 18:14:42 -0800 Subject: [PATCH 0921/1103] Fixes build --- celery/tests/concurrency/test_threads.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/celery/tests/concurrency/test_threads.py b/celery/tests/concurrency/test_threads.py index 2eb5e3882..1edeb5664 100644 --- a/celery/tests/concurrency/test_threads.py +++ b/celery/tests/concurrency/test_threads.py @@ -20,31 +20,31 @@ def test_without_threadpool(self): with mask_modules('threadpool'): with self.assertRaises(ImportError): - TaskPool() + TaskPool(app=self.app) def test_with_threadpool(self): with mock_module('threadpool'): - x = TaskPool() + x = TaskPool(app=self.app) self.assertTrue(x.ThreadPool) self.assertTrue(x.WorkRequest) def test_on_start(self): with mock_module('threadpool'): - x = TaskPool() + x = TaskPool(app=self.app) x.on_start() self.assertTrue(x._pool) self.assertIsInstance(x._pool.workRequests, NullDict) def test_on_stop(self): with mock_module('threadpool'): - x = TaskPool() + x = TaskPool(app=self.app) x.on_start() x.on_stop() x._pool.dismissWorkers.assert_called_with(x.limit, do_join=True) def test_on_apply(self): with mock_module('threadpool'): - x = TaskPool() + x = TaskPool(app=self.app) x.on_start() callback = Mock() accept_callback = Mock() From 7612d78ead19e4449cab2d939864da144596cf65 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Dec 2015 18:15:21 -0800 Subject: [PATCH 0922/1103] Adds support for the new billiard REMAP_SIGTERM envvar. Closes #2839 Requires celery/billiard@6b4ff8470a22e8d98f4219bc2828cdcae4381473 --- celery/app/control.py | 4 +++- celery/concurrency/prefork.py | 6 +++++- celery/worker/control.py | 3 ++- celery/worker/request.py | 3 ++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/celery/app/control.py b/celery/app/control.py index 7058025e0..9caa6942d 100644 --- a/celery/app/control.py +++ b/celery/app/control.py @@ -11,6 +11,8 @@ import warnings +from billiard.common import TERM_SIGNAME + from kombu.pidbox import Mailbox from kombu.utils import cached_property @@ -151,7 +153,7 @@ def election(self, id, topic, action=None, connection=None): }) def revoke(self, task_id, destination=None, terminate=False, - signal='SIGTERM', **kwargs): + signal=TERM_SIGNAME, **kwargs): """Tell all (or specific) workers to revoke a task by id. If a task is revoked, the workers will ignore the task and diff --git a/celery/concurrency/prefork.py b/celery/concurrency/prefork.py index dac9f2111..173316e6d 100644 --- a/celery/concurrency/prefork.py +++ b/celery/concurrency/prefork.py @@ -10,6 +10,7 @@ import os +from billiard.common import REMAP_SIGTERM, TERM_SIGNAME from billiard import forking_enable from billiard.pool import RUN, CLOSE, Pool as BlockingPool @@ -32,7 +33,10 @@ } #: List of signals to ignore when a child process starts. -WORKER_SIGIGNORE = {'SIGINT'} +if REMAP_SIGTERM: + WORKER_SIGIGNORE = {'SIGINT', TERM_SIGNAME} +else: + WORKER_SIGIGNORE = {'SIGINT'} logger = get_logger(__name__) warning, debug = logger.warning, logger.debug diff --git a/celery/worker/control.py b/celery/worker/control.py index 669f047d4..1fb73d442 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -11,6 +11,7 @@ import io import tempfile +from billiard.common import TERM_SIGNAME from kombu.utils.encoding import safe_repr from celery.exceptions import WorkerShutdown @@ -73,7 +74,7 @@ def revoke(state, task_id, terminate=False, signal=None, **kwargs): revoked.update(task_ids) if terminate: - signum = _signals.signum(signal or 'TERM') + signum = _signals.signum(signal or TERM_SIGNAME) # reserved_requests changes size during iteration # so need to consume the items first, then terminate after. requests = set(_find_requests_by_id( diff --git a/celery/worker/request.py b/celery/worker/request.py index 9bac2ec8b..06b210d47 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -15,6 +15,7 @@ from datetime import datetime from weakref import ref +from billiard.common import TERM_SIGNAME from kombu.utils.encoding import safe_repr, safe_str from celery import signals @@ -234,7 +235,7 @@ def maybe_expire(self): return True def terminate(self, pool, signal=None): - signal = _signals.signum(signal or 'TERM') + signal = _signals.signum(signal or TERM_SIGNAME) if self.time_start: pool.terminate_job(self.worker_pid, signal) self._announce_revoked('terminated', True, signal, False) From c8bd72fe2195a7346bf36860d8561c309f76cfdb Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Dec 2015 18:45:05 -0800 Subject: [PATCH 0923/1103] Sanitize result backend in celery report output. Closes #2812 --- celery/app/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/utils.py b/celery/app/utils.py index 396d06538..f3e3f33e2 100644 --- a/celery/app/utils.py +++ b/celery/app/utils.py @@ -300,7 +300,7 @@ def bugreport(app): py_v=_platform.python_version(), driver_v=driver_v, transport=transport, - results=app.conf.result_backend or 'disabled', + results=maybe_sanitize_url(app.conf.result_backend or 'disabled'), human_settings=app.conf.humanize(), loader=qualname(app.loader.__class__), ) From 7ec89a6bf0da853fc9f7e3e9911faf86880178d6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Dec 2015 20:01:41 -0800 Subject: [PATCH 0924/1103] Ensure threads/greenlets do not use the broker connection at the same time. Closes #2755 --- celery/tests/worker/test_control.py | 5 +++++ celery/tests/worker/test_loops.py | 33 ++++++++++++++++++++++++----- celery/worker/consumer.py | 27 ++++++++++++++++++++--- celery/worker/control.py | 10 ++++++--- celery/worker/loops.py | 2 ++ 5 files changed, 66 insertions(+), 11 deletions(-) diff --git a/celery/tests/worker/test_control.py b/celery/tests/worker/test_control.py index dcabfb6f2..e8356f534 100644 --- a/celery/tests/worker/test_control.py +++ b/celery/tests/worker/test_control.py @@ -48,6 +48,10 @@ def __init__(self, app): from celery.concurrency.base import BasePool self.pool = BasePool(10) self.task_buckets = defaultdict(lambda: None) + self.hub = None + + def call_soon(self, p, *args, **kwargs): + return p(*args, **kwargs) class test_Pidbox(AppCase): @@ -345,6 +349,7 @@ class MockConsumer(object): queues = [] cancelled = [] consuming = False + hub = Mock(name='hub') def add_queue(self, queue): self.queues.append(queue.name) diff --git a/celery/tests/worker/test_loops.py b/celery/tests/worker/test_loops.py index 2f08f9866..0d2c7d3da 100644 --- a/celery/tests/worker/test_loops.py +++ b/celery/tests/worker/test_loops.py @@ -3,6 +3,7 @@ import errno import socket +from amqp import promise from kombu.async import Hub, READ, WRITE, ERR from celery.bootsteps import CLOSE, RUN @@ -18,6 +19,22 @@ from celery.tests.case import AppCase, Mock, task_message_from_sig +class PromiseEqual(object): + + def __init__(self, fun, *args, **kwargs): + self.fun = fun + self.args = args + self.kwargs = kwargs + + def __eq__(self, other): + return (other.fun == self.fun and + other.args == self.args and + other.kwargs == self.kwargs) + + def __repr__(self): + return ''.format(self) + + class X(object): def __init__(self, app, heartbeat=None, on_task_message=None, @@ -61,7 +78,8 @@ def __init__(self, app, heartbeat=None, on_task_message=None, self.Hub = self.hub self.blueprint.state = RUN # need this for create_task_handler - _consumer = Consumer(Mock(), timer=Mock(), controller=Mock(), app=app) + self._consumer = _consumer = Consumer( + Mock(), timer=Mock(), controller=Mock(), app=app) _consumer.on_task_message = on_task_message or [] self.obj.create_task_handler = _consumer.create_task_handler self.on_unknown_message = self.obj.on_unknown_message = Mock( @@ -157,20 +175,25 @@ def task_context(self, sig, **kwargs): return x, on_task, message, strategy def test_on_task_received(self): - _, on_task, msg, strategy = self.task_context(self.add.s(2, 2)) + x, on_task, msg, strategy = self.task_context(self.add.s(2, 2)) on_task(msg) strategy.assert_called_with( - msg, None, msg.ack_log_error, msg.reject_log_error, [], + msg, None, + PromiseEqual(x._consumer.call_soon, msg.ack_log_error), + PromiseEqual(x._consumer.call_soon, msg.reject_log_error), [], ) def test_on_task_received_executes_on_task_message(self): cbs = [Mock(), Mock(), Mock()] - _, on_task, msg, strategy = self.task_context( + x, on_task, msg, strategy = self.task_context( self.add.s(2, 2), on_task_message=cbs, ) on_task(msg) strategy.assert_called_with( - msg, None, msg.ack_log_error, msg.reject_log_error, cbs, + msg, None, + PromiseEqual(x._consumer.call_soon, msg.ack_log_error), + PromiseEqual(x._consumer.call_soon, msg.reject_log_error), + cbs, ) def test_on_task_message_missing_name(self): diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index fbbc820ae..dd7d3fc5c 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -21,6 +21,7 @@ from operator import itemgetter from time import sleep +from amqp.promise import ppartial, promise from billiard.common import restart_state from billiard.exceptions import RestartFreqExceeded from kombu.async.semaphore import DummyLock @@ -213,12 +214,29 @@ def __init__(self, on_task_request, # connect again. self.app.conf.broker_connection_timeout = None + self._pending_operations = [] + self.steps = [] self.blueprint = self.Blueprint( app=self.app, on_close=self.on_close, ) self.blueprint.apply(self, **dict(worker_options or {}, **kwargs)) + def call_soon(self, p, *args, **kwargs): + p = ppartial(p, *args, **kwargs) + if self.hub: + return self.hub.call_soon(p) + self._pending_operations.append(p) + return p + + def perform_pending_operations(self): + if not self.hub: + while self._pending_operations: + try: + self._pending_operations.pop()() + except Exception as exc: + error('Pending callback raised: %r', exc, exc_info=1) + def bucket_for_task(self, type): limit = rate(getattr(type, 'rate_limit', None)) return TokenBucket(limit, capacity=1) if limit else None @@ -466,12 +484,13 @@ def update_strategies(self): task.__trace__ = build_tracer(name, task, loader, self.hostname, app=self.app) - def create_task_handler(self): + def create_task_handler(self, promise=promise): strategies = self.strategies on_unknown_message = self.on_unknown_message on_unknown_task = self.on_unknown_task on_invalid_task = self.on_invalid_task callbacks = self.on_task_message + call_soon = self.call_soon def on_task_received(message): # payload will only be set for v1 protocol, since v2 @@ -497,8 +516,10 @@ def on_task_received(message): else: try: strategy( - message, payload, message.ack_log_error, - message.reject_log_error, callbacks, + message, payload, + promise(call_soon, (message.ack_log_error,)), + promise(call_soon, (message.reject_log_error,)), + callbacks, ) except InvalidTaskError as exc: return on_invalid_task(payload, message, exc) diff --git a/celery/worker/control.py b/celery/worker/control.py index 1fb73d442..1d4b8e711 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -345,14 +345,18 @@ def shutdown(state, msg='Got shutdown from remote', **kwargs): @Panel.register def add_consumer(state, queue, exchange=None, exchange_type=None, routing_key=None, **options): - state.consumer.add_task_queue(queue, exchange, exchange_type, - routing_key, **options) + state.consumer.call_soon( + state.consumer.add_task_queue, + queue, exchange, exchange_type, routing_key, **options + ) return {'ok': 'add consumer {0}'.format(queue)} @Panel.register def cancel_consumer(state, queue=None, **_): - state.consumer.cancel_task_queue(queue) + state.consumer.call_soon( + state.consumer.cancel_task_queue, queue, + ) return {'ok': 'no longer consuming from {0}'.format(queue)} diff --git a/celery/worker/loops.py b/celery/worker/loops.py index 8dcc9be62..8365f221f 100644 --- a/celery/worker/loops.py +++ b/celery/worker/loops.py @@ -104,6 +104,7 @@ def synloop(obj, connection, consumer, blueprint, hub, qos, """Fallback blocking event loop for transports that doesn't support AIO.""" on_task_received = obj.create_task_handler() + perform_pending_operations = obj.perform_pending_operations consumer.on_message = on_task_received consumer.consume() @@ -114,6 +115,7 @@ def synloop(obj, connection, consumer, blueprint, hub, qos, if qos.prev != qos.value: qos.update() try: + perform_pending_operations() connection.drain_events(timeout=2.0) except socket.timeout: pass From 8d9fc98dd11784e074f8020feb37ff430bb938d4 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Dec 2015 20:02:21 -0800 Subject: [PATCH 0925/1103] flakes --- celery/tests/worker/test_loops.py | 1 - 1 file changed, 1 deletion(-) diff --git a/celery/tests/worker/test_loops.py b/celery/tests/worker/test_loops.py index 0d2c7d3da..95eaa95eb 100644 --- a/celery/tests/worker/test_loops.py +++ b/celery/tests/worker/test_loops.py @@ -3,7 +3,6 @@ import errno import socket -from amqp import promise from kombu.async import Hub, READ, WRITE, ERR from celery.bootsteps import CLOSE, RUN From d62e9d83bf1d00b272a6c612bd4b539aa3170f81 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Dec 2015 21:57:50 -0800 Subject: [PATCH 0926/1103] Revert "Adds link to Robinhood" This reverts commit f733d93cae73cd60d614846c9c745b5b1fdae5e1. --- docs/.templates/sidebarintro.html | 4 +--- docs/.templates/sidebarlogo.html | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/.templates/sidebarintro.html b/docs/.templates/sidebarintro.html index 2c5b83e3b..8eb9fea26 100644 --- a/docs/.templates/sidebarintro.html +++ b/docs/.templates/sidebarintro.html @@ -14,7 +14,5 @@
Sponsored by: - - - +
diff --git a/docs/.templates/sidebarlogo.html b/docs/.templates/sidebarlogo.html index 2c5b83e3b..8eb9fea26 100644 --- a/docs/.templates/sidebarlogo.html +++ b/docs/.templates/sidebarlogo.html @@ -14,7 +14,5 @@
Sponsored by: - - - +
From cd125e6e5c455125bb0aa0a8034039b70981aefe Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Dec 2015 21:58:05 -0800 Subject: [PATCH 0927/1103] Revert "Adds sponsored by Robinhood logo" This reverts commit 0252652a2055719f5451206330f9107038d3b8c8. --- docs/.templates/sidebarintro.html | 7 +------ docs/.templates/sidebarlogo.html | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/.templates/sidebarintro.html b/docs/.templates/sidebarintro.html index 8eb9fea26..cc68b8f24 100644 --- a/docs/.templates/sidebarintro.html +++ b/docs/.templates/sidebarintro.html @@ -1,5 +1,5 @@ - -
- Sponsored by: - -
diff --git a/docs/.templates/sidebarlogo.html b/docs/.templates/sidebarlogo.html index 8eb9fea26..cc68b8f24 100644 --- a/docs/.templates/sidebarlogo.html +++ b/docs/.templates/sidebarlogo.html @@ -1,5 +1,5 @@ - -
- Sponsored by: - -
From 9044a23c5a9bcc2a82658a72ecc89c324f685471 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Dec 2015 22:58:14 -0800 Subject: [PATCH 0928/1103] Signal handlers should not be able to propagate exceptions. Closes #2738 --- celery/tests/utils/test_dispatcher.py | 84 +++++++++++++++------------ celery/utils/dispatch/signal.py | 39 ++----------- 2 files changed, 54 insertions(+), 69 deletions(-) diff --git a/celery/tests/utils/test_dispatcher.py b/celery/tests/utils/test_dispatcher.py index 72a36f3b3..9a3dcd8ab 100644 --- a/celery/tests/utils/test_dispatcher.py +++ b/celery/tests/utils/test_dispatcher.py @@ -57,18 +57,22 @@ def _testIsClean(self, signal): def test_exact(self): a_signal.connect(receiver_1_arg, sender=self) - expected = [(receiver_1_arg, 'test')] - result = a_signal.send(sender=self, val='test') - self.assertEqual(result, expected) - a_signal.disconnect(receiver_1_arg, sender=self) + try: + expected = [(receiver_1_arg, 'test')] + result = a_signal.send(sender=self, val='test') + self.assertEqual(result, expected) + finally: + a_signal.disconnect(receiver_1_arg, sender=self) self._testIsClean(a_signal) def test_ignored_sender(self): a_signal.connect(receiver_1_arg) - expected = [(receiver_1_arg, 'test')] - result = a_signal.send(sender=self, val='test') - self.assertEqual(result, expected) - a_signal.disconnect(receiver_1_arg) + try: + expected = [(receiver_1_arg, 'test')] + result = a_signal.send(sender=self, val='test') + self.assertEqual(result, expected) + finally: + a_signal.disconnect(receiver_1_arg) self._testIsClean(a_signal) def test_garbage_collected(self): @@ -83,19 +87,22 @@ def test_garbage_collected(self): def test_multiple_registration(self): a = Callable() - a_signal.connect(a) - a_signal.connect(a) - a_signal.connect(a) - a_signal.connect(a) - a_signal.connect(a) - a_signal.connect(a) - result = a_signal.send(sender=self, val='test') - self.assertEqual(len(result), 1) - self.assertEqual(len(a_signal.receivers), 1) - del a - del result - garbage_collect() - self._testIsClean(a_signal) + result = None + try: + a_signal.connect(a) + a_signal.connect(a) + a_signal.connect(a) + a_signal.connect(a) + a_signal.connect(a) + a_signal.connect(a) + result = a_signal.send(sender=self, val='test') + self.assertEqual(len(result), 1) + self.assertEqual(len(a_signal.receivers), 1) + finally: + del a + del result + garbage_collect() + self._testIsClean(a_signal) def test_uid_registration(self): @@ -106,9 +113,11 @@ def uid_based_receiver_2(**kwargs): pass a_signal.connect(uid_based_receiver_1, dispatch_uid='uid') - a_signal.connect(uid_based_receiver_2, dispatch_uid='uid') - self.assertEqual(len(a_signal.receivers), 1) - a_signal.disconnect(dispatch_uid='uid') + try: + a_signal.connect(uid_based_receiver_2, dispatch_uid='uid') + self.assertEqual(len(a_signal.receivers), 1) + finally: + a_signal.disconnect(dispatch_uid='uid') self._testIsClean(a_signal) def test_robust(self): @@ -117,22 +126,25 @@ def fails(val, **kwargs): raise ValueError('this') a_signal.connect(fails) - result = a_signal.send_robust(sender=self, val='test') - err = result[0][1] - self.assertTrue(isinstance(err, ValueError)) - self.assertEqual(err.args, ('this',)) - a_signal.disconnect(fails) + try: + a_signal.send(sender=self, val='test') + finally: + a_signal.disconnect(fails) self._testIsClean(a_signal) def test_disconnection(self): receiver_1 = Callable() receiver_2 = Callable() receiver_3 = Callable() - a_signal.connect(receiver_1) - a_signal.connect(receiver_2) - a_signal.connect(receiver_3) - a_signal.disconnect(receiver_1) - del receiver_2 - garbage_collect() - a_signal.disconnect(receiver_3) + try: + try: + a_signal.connect(receiver_1) + a_signal.connect(receiver_2) + a_signal.connect(receiver_3) + finally: + a_signal.disconnect(receiver_1) + del receiver_2 + garbage_collect() + finally: + a_signal.disconnect(receiver_3) self._testIsClean(a_signal) diff --git a/celery/utils/dispatch/signal.py b/celery/utils/dispatch/signal.py index 36f042e01..2f0d6c832 100644 --- a/celery/utils/dispatch/signal.py +++ b/celery/utils/dispatch/signal.py @@ -7,9 +7,12 @@ from celery.five import range, text_t from celery.local import PromiseProxy, Proxy +from celery.utils.log import get_logger __all__ = ['Signal'] +logger = get_logger(__name__) + WEAKREF_TYPES = (weakref.ReferenceType, saferef.BoundMethodWeakref) @@ -165,42 +168,12 @@ def send(self, sender, **named): if not self.receivers: return responses - for receiver in self._live_receivers(_make_id(sender)): - response = receiver(signal=self, sender=sender, **named) - responses.append((receiver, response)) - return responses - - def send_robust(self, sender, **named): - """Send signal from sender to all connected receivers catching errors. - - :param sender: The sender of the signal. Can be any python object - (normally one registered with a connect if you actually want - something to occur). - - :keyword \*\*named: Named arguments which will be passed to receivers. - These arguments must be a subset of the argument names defined in - :attr:`providing_args`. - - :returns: a list of tuple pairs: `[(receiver, response), … ]`. - - :raises DispatcherKeyError: - - if any receiver raises an error (specifically any subclass of - :exc:`Exception`), the error instance is returned as the result - for that receiver. - - """ - responses = [] - if not self.receivers: - return responses - - # Call each receiver with whatever arguments it can accept. - # Return a list of tuple pairs [(receiver, response), … ]. for receiver in self._live_receivers(_make_id(sender)): try: response = receiver(signal=self, sender=sender, **named) - except Exception as err: - responses.append((receiver, err)) + except Exception as exc: + logger.error('Signal handler %r raised: %r', + receiver, exc, exc_info=1) else: responses.append((receiver, response)) return responses From 069d36db5183149a02cc94ac1255432765b448a9 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Dec 2015 23:24:17 -0800 Subject: [PATCH 0929/1103] Fixes typos in docs for inspect stats total values. Closes #2730 --- docs/userguide/workers.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index cbe93aee7..6a78c8438 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -1132,8 +1132,8 @@ The output will include the following fields: - ``total`` - List of task names and a total number of times that task have been - executed since worker start. + Map of task names and the total number of tasks with that type + the worker has accepted since startup. Additional Commands From 6946fb74b699056878702c239eadf09751d4311f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 14 Dec 2015 12:06:18 -0800 Subject: [PATCH 0930/1103] Only use anon exchange for direct exchanges --- celery/app/amqp.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index bcd3c8139..de1f9b68a 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -476,7 +476,7 @@ def send_task_message(producer, name, message, retry=None, retry_policy=None, serializer=None, delivery_mode=None, compression=None, declare=None, - headers=None, **kwargs): + headers=None, exchange_type=None, **kwargs): retry = default_retry if retry is None else retry headers2, properties, body, sent_event = message if headers: @@ -492,13 +492,21 @@ def send_task_message(producer, name, message, qname, queue = queue, queues[queue] else: qname = queue.name + if delivery_mode is None: try: delivery_mode = queue.exchange.delivery_mode except AttributeError: pass delivery_mode = delivery_mode or default_delivery_mode - if not exchange and not routing_key: + + if exchange_type is None: + try: + exchange_type = queue.exchange.type + except AttributeError: + exchange_type = 'direct' + + if not exchange and not routing_key and exchange_type == 'direct': exchange, routing_key = '', qname else: exchange = exchange or queue.exchange.name or default_exchange From 816ff0310296aad7d44cd6e07004f770c007e226 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 14 Dec 2015 12:07:05 -0800 Subject: [PATCH 0931/1103] Remote control: Use producer pool for requests/replies. Also fixes worker EINTR error --- celery/app/control.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/celery/app/control.py b/celery/app/control.py index 9caa6942d..4444e0551 100644 --- a/celery/app/control.py +++ b/celery/app/control.py @@ -15,6 +15,7 @@ from kombu.pidbox import Mailbox from kombu.utils import cached_property +from kombu.utils.functional import lazy from celery.exceptions import DuplicateNodenameWarning from celery.utils.text import pluralize @@ -128,7 +129,12 @@ class Control(object): def __init__(self, app=None): self.app = app - self.mailbox = self.Mailbox('celery', type='fanout', accept=['json']) + self.mailbox = self.Mailbox( + 'celery', + type='fanout', + accept=['json'], + producer_pool=lazy(lambda: self.app.amqp.producer_pool), + ) @cached_property def inspect(self): From 8432fdc5f0ccc166af6065fc7904fc3afdbf2ebb Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 16 Dec 2015 16:09:53 -0800 Subject: [PATCH 0932/1103] Backends: Use state in API consistently over status - Backend.store_result() -> status argument renamed to state - Backend.get_status() -> renamed to Backend.get_state() --- celery/backends/amqp.py | 8 ++++---- celery/backends/base.py | 23 ++++++++++++----------- celery/backends/cassandra.py | 6 +++--- celery/backends/database/__init__.py | 6 +++--- celery/backends/mongodb.py | 6 +++--- celery/contrib/abortable.py | 4 ++-- celery/result.py | 6 +++--- celery/tests/backends/test_amqp.py | 4 ++-- celery/tests/backends/test_base.py | 8 ++++---- celery/tests/backends/test_cache.py | 14 +++++++------- celery/tests/backends/test_database.py | 14 +++++++------- celery/tests/backends/test_filesystem.py | 4 ++-- celery/tests/backends/test_redis.py | 4 ++-- 13 files changed, 54 insertions(+), 53 deletions(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index f88b711aa..853200bc3 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -111,16 +111,16 @@ def destination_for(self, task_id, request): return self.rkey(task_id), request.correlation_id or task_id return self.rkey(task_id), task_id - def store_result(self, task_id, result, status, + def store_result(self, task_id, result, state, traceback=None, request=None, **kwargs): - """Send task return value and status.""" + """Send task return value and state.""" routing_key, correlation_id = self.destination_for(task_id, request) if not routing_key: return with self.app.amqp.producer_pool.acquire(block=True) as producer: producer.publish( - {'task_id': task_id, 'status': status, - 'result': self.encode_result(result, status), + {'task_id': task_id, 'status': state, + 'result': self.encode_result(result, state), 'traceback': traceback, 'children': self.current_task_children(request)}, exchange=self.exchange, diff --git a/celery/backends/base.py b/celery/backends/base.py index a82ac4060..8a30ec044 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -263,8 +263,8 @@ def prepare_persistent(self, enabled=None): p = self.app.conf.result_persistent return self.persistent if p is None else p - def encode_result(self, result, status): - if status in self.EXCEPTION_STATES and isinstance(result, Exception): + def encode_result(self, result, state): + if state in self.EXCEPTION_STATES and isinstance(result, Exception): return self.prepare_exception(result) else: return self.prepare_value(result) @@ -272,11 +272,11 @@ def encode_result(self, result, status): def is_cached(self, task_id): return task_id in self._cache - def store_result(self, task_id, result, status, + def store_result(self, task_id, result, state, traceback=None, request=None, **kwargs): """Update task state and result.""" - result = self.encode_result(result, status) - self._store_result(task_id, result, status, traceback, + result = self.encode_result(result, state) + self._store_result(task_id, result, state, traceback, request=request, **kwargs) return result @@ -287,9 +287,10 @@ def forget(self, task_id): def _forget(self, task_id): raise NotImplementedError('backend does not implement forget.') - def get_status(self, task_id): - """Get the status of a task.""" + def get_state(self, task_id): + """Get the state of a task.""" return self.get_task_meta(task_id)['status'] + get_status = get_state # XXX compat def get_traceback(self, task_id): """Get the traceback for a failed task.""" @@ -521,9 +522,9 @@ def get_many(self, task_ids, timeout=None, interval=0.5, no_ack=True, def _forget(self, task_id): self.delete(self.get_key_for_task(task_id)) - def _store_result(self, task_id, result, status, + def _store_result(self, task_id, result, state, traceback=None, request=None, **kwargs): - meta = {'status': status, 'result': result, 'traceback': traceback, + meta = {'status': state, 'result': result, 'traceback': traceback, 'children': self.current_task_children(request)} self.set(self.get_key_for_task(task_id), self.encode(meta)) return result @@ -639,5 +640,5 @@ def _is_disabled(self, *args, **kwargs): raise NotImplementedError( 'No result backend configured. ' 'Please see the documentation for more information.') - wait_for = get_status = get_result = get_traceback = _is_disabled - get_many = _is_disabled + get_state = get_status = get_result = get_traceback = _is_disabled + wait_for = get_many = _is_disabled diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index 3caa7d255..d406be1df 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -188,14 +188,14 @@ def _get_connection(self, write=False): self._session = None raise # we did fail after all - reraise - def _store_result(self, task_id, result, status, + def _store_result(self, task_id, result, state, traceback=None, request=None, **kwargs): - """Store return value and status of an executed task.""" + """Store return value and state of an executed task.""" self._get_connection(write=True) self._session.execute(self._write_stmt, ( task_id, - status, + state, buf_t(self.encode(result)), self.app.now(), buf_t(self.encode(traceback)), diff --git a/celery/backends/database/__init__.py b/celery/backends/database/__init__.py index bbd570a71..858092619 100644 --- a/celery/backends/database/__init__.py +++ b/celery/backends/database/__init__.py @@ -106,9 +106,9 @@ def ResultSession(self, session_manager=SessionManager()): ) @retry - def _store_result(self, task_id, result, status, + def _store_result(self, task_id, result, state, traceback=None, max_retries=3, **kwargs): - """Store return value and status of an executed task.""" + """Store return value and state of an executed task.""" session = self.ResultSession() with session_cleanup(session): task = list(session.query(Task).filter(Task.task_id == task_id)) @@ -118,7 +118,7 @@ def _store_result(self, task_id, result, status, session.add(task) session.flush() task.result = result - task.status = status + task.status = state task.traceback = traceback session.commit() return result diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index fe863ea56..8935d0d81 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -181,12 +181,12 @@ def decode(self, data): return data return super(MongoBackend, self).decode(data) - def _store_result(self, task_id, result, status, + def _store_result(self, task_id, result, state, traceback=None, request=None, **kwargs): - """Store return value and status of an executed task.""" + """Store return value and state of an executed task.""" meta = {'_id': task_id, - 'status': status, + 'status': state, 'result': self.encode(result), 'date_done': datetime.utcnow(), 'traceback': self.encode(traceback), diff --git a/celery/contrib/abortable.py b/celery/contrib/abortable.py index dcdc61566..eaacebde7 100644 --- a/celery/contrib/abortable.py +++ b/celery/contrib/abortable.py @@ -132,9 +132,9 @@ def abort(self): """ # TODO: store_result requires all four arguments to be set, - # but only status should be updated here + # but only state should be updated here return self.backend.store_result(self.id, result=None, - status=ABORTED, traceback=None) + state=ABORTED, traceback=None) class AbortableTask(Task): diff --git a/celery/result.py b/celery/result.py index 4c1e14a1e..1dfbb69df 100644 --- a/celery/result.py +++ b/celery/result.py @@ -170,8 +170,8 @@ def get(self, timeout=None, propagate=True, interval=0.5, ) if meta: self._maybe_set_cache(meta) - status = meta['status'] - if status in PROPAGATE_STATES and propagate: + state = meta['status'] + if state in PROPAGATE_STATES and propagate: raise meta['result'] if callback is not None: callback(self.id, meta['result']) @@ -395,7 +395,7 @@ def state(self): """ return self._get_task_meta()['status'] - status = state + status = state # XXX compat @property def task_id(self): diff --git a/celery/tests/backends/test_amqp.py b/celery/tests/backends/test_amqp.py index 640733f1c..64c4fa721 100644 --- a/celery/tests/backends/test_amqp.py +++ b/celery/tests/backends/test_amqp.py @@ -57,7 +57,7 @@ def test_mark_as_done(self): tid = uuid() tb1.mark_as_done(tid, 42) - self.assertEqual(tb2.get_status(tid), states.SUCCESS) + self.assertEqual(tb2.get_state(tid), states.SUCCESS) self.assertEqual(tb2.get_result(tid), 42) self.assertTrue(tb2._cache.get(tid)) self.assertTrue(tb2.get_result(tid), 42) @@ -92,7 +92,7 @@ def test_mark_as_failure(self): except KeyError as exception: einfo = ExceptionInfo() tb1.mark_as_failure(tid3, exception, traceback=einfo.traceback) - self.assertEqual(tb2.get_status(tid3), states.FAILURE) + self.assertEqual(tb2.get_state(tid3), states.FAILURE) self.assertIsInstance(tb2.get_result(tid3), KeyError) self.assertEqual(tb2.get_traceback(tid3), einfo.traceback) diff --git a/celery/tests/backends/test_base.py b/celery/tests/backends/test_base.py index 86b4f1b4f..226bb0d7a 100644 --- a/celery/tests/backends/test_base.py +++ b/celery/tests/backends/test_base.py @@ -337,9 +337,9 @@ def test_get_store_delete_result(self): tid = uuid() self.b.mark_as_done(tid, 'Hello world') self.assertEqual(self.b.get_result(tid), 'Hello world') - self.assertEqual(self.b.get_status(tid), states.SUCCESS) + self.assertEqual(self.b.get_state(tid), states.SUCCESS) self.b.forget(tid) - self.assertEqual(self.b.get_status(tid), states.PENDING) + self.assertEqual(self.b.get_state(tid), states.PENDING) def test_strip_prefix(self): x = self.b.get_key_for_task('x1b34') @@ -529,7 +529,7 @@ def test_chord_apply_fallback(self): def test_get_missing_meta(self): self.assertIsNone(self.b.get_result('xxx-missing')) - self.assertEqual(self.b.get_status('xxx-missing'), states.PENDING) + self.assertEqual(self.b.get_state('xxx-missing'), states.PENDING) def test_save_restore_delete_group(self): tid = uuid() @@ -583,4 +583,4 @@ def test_store_result(self): def test_is_disabled(self): with self.assertRaises(NotImplementedError): - DisabledBackend(self.app).get_status('foo') + DisabledBackend(self.app).get_state('foo') diff --git a/celery/tests/backends/test_cache.py b/celery/tests/backends/test_cache.py index e5e2fce74..ee3291295 100644 --- a/celery/tests/backends/test_cache.py +++ b/celery/tests/backends/test_cache.py @@ -41,11 +41,11 @@ def test_no_backend(self): CacheBackend(backend=None, app=self.app) def test_mark_as_done(self): - self.assertEqual(self.tb.get_status(self.tid), states.PENDING) + self.assertEqual(self.tb.get_state(self.tid), states.PENDING) self.assertIsNone(self.tb.get_result(self.tid)) self.tb.mark_as_done(self.tid, 42) - self.assertEqual(self.tb.get_status(self.tid), states.SUCCESS) + self.assertEqual(self.tb.get_state(self.tid), states.SUCCESS) self.assertEqual(self.tb.get_result(self.tid), 42) def test_is_pickled(self): @@ -61,7 +61,7 @@ def test_mark_as_failure(self): raise KeyError('foo') except KeyError as exception: self.tb.mark_as_failure(self.tid, exception) - self.assertEqual(self.tb.get_status(self.tid), states.FAILURE) + self.assertEqual(self.tb.get_state(self.tid), states.FAILURE) self.assertIsInstance(self.tb.get_result(self.tid), KeyError) def test_apply_chord(self): @@ -219,7 +219,7 @@ def test_memcache_unicode_key(self): cache._imp = [None] task_id, result = string(uuid()), 42 b = cache.CacheBackend(backend='memcache', app=self.app) - b.store_result(task_id, result, status=states.SUCCESS) + b.store_result(task_id, result, state=states.SUCCESS) self.assertEqual(b.get_result(task_id), result) def test_memcache_bytes_key(self): @@ -230,7 +230,7 @@ def test_memcache_bytes_key(self): cache._imp = [None] task_id, result = str_to_bytes(uuid()), 42 b = cache.CacheBackend(backend='memcache', app=self.app) - b.store_result(task_id, result, status=states.SUCCESS) + b.store_result(task_id, result, state=states.SUCCESS) self.assertEqual(b.get_result(task_id), result) def test_pylibmc_unicode_key(self): @@ -240,7 +240,7 @@ def test_pylibmc_unicode_key(self): cache._imp = [None] task_id, result = string(uuid()), 42 b = cache.CacheBackend(backend='memcache', app=self.app) - b.store_result(task_id, result, status=states.SUCCESS) + b.store_result(task_id, result, state=states.SUCCESS) self.assertEqual(b.get_result(task_id), result) def test_pylibmc_bytes_key(self): @@ -250,5 +250,5 @@ def test_pylibmc_bytes_key(self): cache._imp = [None] task_id, result = str_to_bytes(uuid()), 42 b = cache.CacheBackend(backend='memcache', app=self.app) - b.store_result(task_id, result, status=states.SUCCESS) + b.store_result(task_id, result, state=states.SUCCESS) self.assertEqual(b.get_result(task_id), result) diff --git a/celery/tests/backends/test_database.py b/celery/tests/backends/test_database.py index 5e716723d..0dbbacd11 100644 --- a/celery/tests/backends/test_database.py +++ b/celery/tests/backends/test_database.py @@ -90,7 +90,7 @@ def test_missing_dburi_raises_ImproperlyConfigured(self): def test_missing_task_id_is_PENDING(self): tb = DatabaseBackend(self.uri, app=self.app) - self.assertEqual(tb.get_status('xxx-does-not-exist'), states.PENDING) + self.assertEqual(tb.get_state('xxx-does-not-exist'), states.PENDING) def test_missing_task_meta_is_dict_with_pending(self): tb = DatabaseBackend(self.uri, app=self.app) @@ -106,11 +106,11 @@ def test_mark_as_done(self): tid = uuid() - self.assertEqual(tb.get_status(tid), states.PENDING) + self.assertEqual(tb.get_state(tid), states.PENDING) self.assertIsNone(tb.get_result(tid)) tb.mark_as_done(tid, 42) - self.assertEqual(tb.get_status(tid), states.SUCCESS) + self.assertEqual(tb.get_state(tid), states.SUCCESS) self.assertEqual(tb.get_result(tid), 42) def test_is_pickled(self): @@ -128,13 +128,13 @@ def test_mark_as_started(self): tb = DatabaseBackend(self.uri, app=self.app) tid = uuid() tb.mark_as_started(tid) - self.assertEqual(tb.get_status(tid), states.STARTED) + self.assertEqual(tb.get_state(tid), states.STARTED) def test_mark_as_revoked(self): tb = DatabaseBackend(self.uri, app=self.app) tid = uuid() tb.mark_as_revoked(tid) - self.assertEqual(tb.get_status(tid), states.REVOKED) + self.assertEqual(tb.get_state(tid), states.REVOKED) def test_mark_as_retry(self): tb = DatabaseBackend(self.uri, app=self.app) @@ -145,7 +145,7 @@ def test_mark_as_retry(self): import traceback trace = '\n'.join(traceback.format_stack()) tb.mark_as_retry(tid, exception, traceback=trace) - self.assertEqual(tb.get_status(tid), states.RETRY) + self.assertEqual(tb.get_state(tid), states.RETRY) self.assertIsInstance(tb.get_result(tid), KeyError) self.assertEqual(tb.get_traceback(tid), trace) @@ -159,7 +159,7 @@ def test_mark_as_failure(self): import traceback trace = '\n'.join(traceback.format_stack()) tb.mark_as_failure(tid3, exception, traceback=trace) - self.assertEqual(tb.get_status(tid3), states.FAILURE) + self.assertEqual(tb.get_state(tid3), states.FAILURE) self.assertIsInstance(tb.get_result(tid3), KeyError) self.assertEqual(tb.get_traceback(tid3), trace) diff --git a/celery/tests/backends/test_filesystem.py b/celery/tests/backends/test_filesystem.py index b8ff0d5ca..55a3d05dd 100644 --- a/celery/tests/backends/test_filesystem.py +++ b/celery/tests/backends/test_filesystem.py @@ -45,7 +45,7 @@ def test_path_is_incorrect(self): def test_missing_task_is_PENDING(self): tb = FilesystemBackend(app=self.app, url=self.url) - self.assertEqual(tb.get_status('xxx-does-not-exist'), states.PENDING) + self.assertEqual(tb.get_state('xxx-does-not-exist'), states.PENDING) def test_mark_as_done_writes_file(self): tb = FilesystemBackend(app=self.app, url=self.url) @@ -56,7 +56,7 @@ def test_done_task_is_SUCCESS(self): tb = FilesystemBackend(app=self.app, url=self.url) tid = uuid() tb.mark_as_done(tid, 42) - self.assertEqual(tb.get_status(tid), states.SUCCESS) + self.assertEqual(tb.get_state(tid), states.SUCCESS) def test_correct_result(self): data = {'foo': 'bar'} diff --git a/celery/tests/backends/test_redis.py b/celery/tests/backends/test_redis.py index bb2b274cc..a486969c7 100644 --- a/celery/tests/backends/test_redis.py +++ b/celery/tests/backends/test_redis.py @@ -385,10 +385,10 @@ def test_process_cleanup(self): def test_get_set_forget(self): tid = uuid() self.b.store_result(tid, 42, states.SUCCESS) - self.assertEqual(self.b.get_status(tid), states.SUCCESS) + self.assertEqual(self.b.get_state(tid), states.SUCCESS) self.assertEqual(self.b.get_result(tid), 42) self.b.forget(tid) - self.assertEqual(self.b.get_status(tid), states.PENDING) + self.assertEqual(self.b.get_state(tid), states.PENDING) def test_set_expires(self): self.b = self.Backend(expires=512, app=self.app) From 709f51c2d9aca21569100d748e3d74b657f9e4a5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 28 Dec 2015 16:05:39 -0800 Subject: [PATCH 0933/1103] worker_task_log_format is not changed by --loglevel option as documentation says (Issue #2974) --- docs/configuration.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 1a4ebe880..5fe05b352 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -2040,7 +2040,9 @@ worker_log_format The format to use for log messages. -Default is `[%(asctime)s: %(levelname)s/%(processName)s] %(message)s` +Default is:: + + [%(asctime)s: %(levelname)s/%(processName)s] %(message)s See the Python :mod:`logging` module for more information about log formats. @@ -2050,8 +2052,7 @@ formats. worker_task_log_format ~~~~~~~~~~~~~~~~~~~~~~ -The format to use for log messages logged in tasks. Can be overridden using -the :option:`--loglevel` option to :mod:`~celery.bin.worker`. +The format to use for log messages logged in tasks. Default is:: From 6d80d31358b5c3883c8d81dbd8b84dbe0cf935ee Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 29 Dec 2015 15:01:03 -0800 Subject: [PATCH 0934/1103] [Py3][task.http] Query paramters must be bytes. Closes #2967 --- celery/task/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/task/http.py b/celery/task/http.py index 0c1246185..448b47a00 100644 --- a/celery/task/http.py +++ b/celery/task/http.py @@ -17,7 +17,7 @@ from urlparse import urlparse, parse_qsl # noqa from kombu.utils import json -from kombu.utils.encoding import bytes_to_str +from kombu.utils.encoding import bytes_to_str, str_to_bytes from celery import shared_task, __version__ as celery_version from celery.five import items, reraise @@ -109,7 +109,7 @@ def __init__(self, url): def __str__(self): scheme, netloc, path, params, query, fragment = self.parts - query = urlencode(utf8dict(items(self.query))) + query = str_to_bytes(urlencode(utf8dict(items(self.query)))) components = [scheme + '://', netloc, path or '/', ';{0}'.format(params) if params else '', '?{0}'.format(query) if query else '', From 6038ff2aa315384c04652fd6733a61462d3536b0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 29 Dec 2015 17:12:01 -0800 Subject: [PATCH 0935/1103] Adds broker_read_url and broker_write_url settings These enable you to separate the broker URLs used for consuming and producing respectively. In addition to the configuration options two new methods have been added to the app: * ``app.connection_for_read()`` * ``app.connection_for_write()`` these should now be used, instead of `app.connection()`, to specify the intent of the required connection. --- celery/app/amqp.py | 3 +- celery/app/base.py | 48 ++++++++++++++++++++++++--- celery/app/defaults.py | 2 ++ celery/app/utils.py | 22 ++++++++++-- celery/beat.py | 2 +- celery/bin/celery.py | 2 +- celery/bin/graph.py | 3 +- celery/events/__init__.py | 5 +-- celery/events/cursesmon.py | 2 +- celery/events/dumper.py | 2 +- celery/events/snapshot.py | 2 +- celery/task/base.py | 6 ++-- celery/tests/bin/test_celeryevdump.py | 2 +- celery/tests/events/test_events.py | 14 ++++---- celery/tests/worker/test_consumer.py | 27 ++++++++++----- celery/tests/worker/test_control.py | 4 +-- celery/tests/worker/test_worker.py | 6 ++-- celery/worker/__init__.py | 2 +- celery/worker/consumer.py | 8 ++--- docs/configuration.rst | 19 +++++++++++ 20 files changed, 136 insertions(+), 45 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index de1f9b68a..518681d4c 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -585,7 +585,8 @@ def router(self): @property def producer_pool(self): if self._producer_pool is None: - self._producer_pool = pools.producers[self.app.connection()] + self._producer_pool = pools.producers[ + self.app.connection_for_write()] self._producer_pool.limit = self.app.pool.limit return self._producer_pool publisher_pool = producer_pool # compat alias diff --git a/celery/app/base.py b/celery/app/base.py index 1bbc13362..385762282 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -272,7 +272,7 @@ def close(self): use the with statement instead:: with Celery(set_as_current=False) as app: - with app.connection() as conn: + with app.connection_for_write() as conn: pass """ self._pool = None @@ -655,6 +655,22 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, parent.add_trail(result) return result + def connection_for_read(self, url=None, **kwargs): + """Establish connection used for consuming. + + See :meth:`connection` for supported arguments. + + """ + return self._connection(url or self.conf.broker_read_url, **kwargs) + + def connection_for_write(self, url=None, **kwargs): + """Establish connection used for producing. + + See :meth:`connection` for supported arguments. + + """ + return self._connection(url or self.conf.broker_write_url, **kwargs) + def connection(self, hostname=None, userid=None, password=None, virtual_host=None, port=None, ssl=None, connect_timeout=None, transport=None, @@ -662,6 +678,10 @@ def connection(self, hostname=None, userid=None, password=None, login_method=None, failover_strategy=None, **kwargs): """Establish a connection to the message broker. + Please use :meth:`connection_for_read` and + :meth:`connection_for_write` instead, to convey the intent + of use for this connection. + :param url: Either the URL or the hostname of the broker to use. :keyword hostname: URL, Hostname/IP-address of the broker. @@ -674,13 +694,33 @@ def connection(self, hostname=None, userid=None, password=None, :keyword ssl: Defaults to the :setting:`broker_use_ssl` setting. :keyword transport: defaults to the :setting:`broker_transport` setting. + :keyword transport_options: Dictionary of transport specific options. + :keyword heartbeat: AMQP Heartbeat in seconds (pyamqp only). + :keyword login_method: Custom login method to use (amqp only). + :keyword failover_strategy: Custom failover strategy. + :keyword \*\*kwargs: Additional arguments to :class:`kombu.Connection`. :returns :class:`kombu.Connection`: """ + return self.connection_for_write( + hostname or self.conf.broker_write_url, + userid=userid, password=password, + virtual_host=virtual_host, port=port, ssl=ssl, + connect_timeout=connect_timeout, transport=transport, + transport_options=transport_options, heartbeat=heartbeat, + login_method=login_method, failover_strategy=failover_strategy, + **kwargs + ) + + def _connection(self, url, userid=None, password=None, + virtual_host=None, port=None, ssl=None, + connect_timeout=None, transport=None, + transport_options=None, heartbeat=None, + login_method=None, failover_strategy=None, **kwargs): conf = self.conf return self.amqp.Connection( - hostname or conf.broker_url, + url, userid or conf.broker_user, password or conf.broker_password, virtual_host or conf.broker_vhost, @@ -705,7 +745,7 @@ def _acquire_connection(self, pool=True): """Helper for :meth:`connection_or_acquire`.""" if pool: return self.pool.acquire(block=True) - return self.connection() + return self.connection_for_write() def connection_or_acquire(self, connection=None, pool=True, *_, **__): """For use within a with-statement to get a connection from the pool @@ -1002,7 +1042,7 @@ def pool(self): self._ensure_after_fork() limit = self.conf.broker_pool_limit pools.set_limit(limit) - self._pool = pools.connections[self.connection()] + self._pool = pools.connections[self.connection_for_write()] return self._pool @property diff --git a/celery/app/defaults.py b/celery/app/defaults.py index a6f9b8b69..9f8e44cd4 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -98,6 +98,8 @@ def __repr__(self): ), broker=Namespace( url=Option(None, type='string'), + read_url=Option(None, type='string'), + write_url=Option(None, type='string'), transport=Option(type='string'), transport_options=Option({}, type='dict'), connection_timeout=Option(4, type='float'), diff --git a/celery/app/utils.py b/celery/app/utils.py index f3e3f33e2..47254888e 100644 --- a/celery/app/utils.py +++ b/celery/app/utils.py @@ -86,10 +86,28 @@ class Settings(ConfigurationView): """ + @property + def broker_read_url(self): + return ( + os.environ.get('CELERY_BROKER_READ_URL') or + self.get('broker_read_url') or + self.broker_url + ) + + @property + def broker_write_url(self): + return ( + os.environ.get('CELERY_BROKER_WRITE_URL') or + self.get('broker_write_url') or + self.broker_url + ) + @property def broker_url(self): - return (os.environ.get('CELERY_BROKER_URL') or - self.first('broker_url', 'broker_host')) + return ( + os.environ.get('CELERY_BROKER_URL') or + self.first('broker_url', 'broker_host') + ) @property def timezone(self): diff --git a/celery/beat.py b/celery/beat.py index 16871fd10..c4ceca01d 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -381,7 +381,7 @@ def set_schedule(self, schedule): @cached_property def connection(self): - return self.app.connection() + return self.app.connection_for_write() @cached_property def producer(self): diff --git a/celery/bin/celery.py b/celery/bin/celery.py index 91b788480..599875e7d 100644 --- a/celery/bin/celery.py +++ b/celery/bin/celery.py @@ -337,7 +337,7 @@ def do_call_method(self, args, **kwargs): raise self.UsageError( 'Unknown {0.name} method {1}'.format(self, method)) - if self.app.connection().transport.driver_type == 'sql': + if self.app.connection_for_write().transport.driver_type == 'sql': raise self.Error('Broadcast not supported by SQL broker transport') output_json = kwargs.get('json') diff --git a/celery/bin/graph.py b/celery/bin/graph.py index 5216ab0ab..d441a54ca 100644 --- a/celery/bin/graph.py +++ b/celery/bin/graph.py @@ -166,7 +166,8 @@ def maybe_abbr(l, name, max=Wmax): list(range(int(threads))), 'P', Tmax, ) - broker = Broker(args.get('broker', self.app.connection().as_uri())) + broker = Broker(args.get( + 'broker', self.app.connection_for_read().as_uri())) backend = Backend(backend) if backend else None graph = DependencyGraph(formatter=Formatter()) graph.add_arc(broker) diff --git a/celery/events/__init__.py b/celery/events/__init__.py index 44dfd158d..23b3ea0da 100644 --- a/celery/events/__init__.py +++ b/celery/events/__init__.py @@ -140,7 +140,7 @@ def __init__(self, connection=None, hostname=None, enabled=True, if not connection and channel: self.connection = channel.connection.client self.enabled = enabled - conninfo = self.connection or self.app.connection() + conninfo = self.connection or self.app.connection_for_write() self.exchange = get_exchange(conninfo) if conninfo.transport.driver_type in self.DISABLED_TRANSPORTS: self.enabled = False @@ -307,7 +307,8 @@ def __init__(self, channel, handlers=None, routing_key='#', self.routing_key = routing_key self.node_id = node_id or uuid() self.queue_prefix = queue_prefix - self.exchange = get_exchange(self.connection or self.app.connection()) + self.exchange = get_exchange( + self.connection or self.app.connection_for_write()) self.queue = Queue( '.'.join([self.queue_prefix, self.node_id]), exchange=self.exchange, diff --git a/celery/events/cursesmon.py b/celery/events/cursesmon.py index 923ca8a2d..8f49f466e 100644 --- a/celery/events/cursesmon.py +++ b/celery/events/cursesmon.py @@ -508,7 +508,7 @@ def on_connection_error(exc, interval): while 1: print('-> evtop: starting capture...', file=sys.stderr) - with app.connection() as conn: + with app.connection_for_read() as conn: try: conn.ensure_connection(on_connection_error, app.conf.broker_connection_max_retries) diff --git a/celery/events/dumper.py b/celery/events/dumper.py index 672670b97..c793b37e1 100644 --- a/celery/events/dumper.py +++ b/celery/events/dumper.py @@ -88,7 +88,7 @@ def evdump(app=None, out=sys.stdout): app = app_or_default(app) dumper = Dumper(out=out) dumper.say('-> evdump: starting capture...') - conn = app.connection().clone() + conn = app.connection_for_read().clone() def _error_handler(exc, interval): dumper.say(CONNECTION_ERROR % ( diff --git a/celery/events/snapshot.py b/celery/events/snapshot.py index 1888636ef..6ca3a31ad 100644 --- a/celery/events/snapshot.py +++ b/celery/events/snapshot.py @@ -102,7 +102,7 @@ def evcam(camera, freq=1.0, maxrate=None, loglevel=0, cam = instantiate(camera, state, app=app, freq=freq, maxrate=maxrate, timer=timer) cam.install() - conn = app.connection() + conn = app.connection_for_read() recv = app.events.Receiver(conn, handlers={'*': state.event}) try: try: diff --git a/celery/task/base.py b/celery/task/base.py index b248f428a..b7d3b24eb 100644 --- a/celery/task/base.py +++ b/celery/task/base.py @@ -192,10 +192,10 @@ def establish_connection(self): ... # establish fresh connection - with celery.connection() as conn: + with celery.connection_for_write() as conn: ... """ - return self._get_app().connection() + return self._get_app().connection_for_write() def get_publisher(self, connection=None, exchange=None, exchange_type=None, **options): @@ -205,7 +205,7 @@ def get_publisher(self, connection=None, exchange=None, .. code-block:: python - with app.connection() as conn: + with app.connection_for_write() as conn: with app.amqp.Producer(conn) as prod: my_task.apply_async(producer=prod) diff --git a/celery/tests/bin/test_celeryevdump.py b/celery/tests/bin/test_celeryevdump.py index 9eb7d52bc..9fc54b67d 100644 --- a/celery/tests/bin/test_celeryevdump.py +++ b/celery/tests/bin/test_celeryevdump.py @@ -56,7 +56,7 @@ def se(*_a, **_k): raise KeyError() recv.capture.side_effect = se - Conn = app.connection.return_value = Mock(name='conn') + Conn = app.connection_for_read.return_value = Mock(name='conn') conn = Conn.clone.return_value = Mock(name='cloned_conn') conn.connection_errors = (KeyError,) conn.channel_errors = () diff --git a/celery/tests/events/test_events.py b/celery/tests/events/test_events.py index 44ef3c58f..e1810a03d 100644 --- a/celery/tests/events/test_events.py +++ b/celery/tests/events/test_events.py @@ -66,7 +66,7 @@ def test_sql_transports_disabled(self): def test_send(self): producer = MockProducer() - producer.connection = self.app.connection() + producer.connection = self.app.connection_for_write() connection = Mock() connection.transport.driver_type = 'amqp' eventer = self.app.events.Dispatcher(connection, enabled=False, @@ -98,7 +98,7 @@ def test_send(self): def test_send_buffer_group(self): buf_received = [None] producer = MockProducer() - producer.connection = self.app.connection() + producer.connection = self.app.connection_for_write() connection = Mock() connection.transport.driver_type = 'amqp' eventer = self.app.events.Dispatcher( @@ -134,7 +134,7 @@ def test_flush_no_groups_no_errors(self): eventer.flush(errors=False, groups=False) def test_enter_exit(self): - with self.app.connection() as conn: + with self.app.connection_for_write() as conn: d = self.app.events.Dispatcher(conn) d.close = Mock() with d as _d: @@ -144,7 +144,7 @@ def test_enter_exit(self): def test_enable_disable_callbacks(self): on_enable = Mock() on_disable = Mock() - with self.app.connection() as conn: + with self.app.connection_for_write() as conn: with self.app.events.Dispatcher(conn, enabled=False) as d: d.on_enabled.add(on_enable) d.on_disabled.add(on_disable) @@ -154,7 +154,7 @@ def test_enable_disable_callbacks(self): on_disable.assert_called_with() def test_enabled_disable(self): - connection = self.app.connection() + connection = self.app.connection_for_write() channel = connection.channel() try: dispatcher = self.app.events.Dispatcher(connection, @@ -235,7 +235,7 @@ def my_handler(event): self.assertTrue(got_event[0]) def test_itercapture(self): - connection = self.app.connection() + connection = self.app.connection_for_write() try: r = self.app.events.Receiver(connection, node_id='celery.tests') it = r.itercapture(timeout=0.0001, wakeup=False) @@ -284,7 +284,7 @@ def on_efm(*args): r.process.assert_has_calls([call(1), call(2), call(3)]) def test_itercapture_limit(self): - connection = self.app.connection() + connection = self.app.connection_for_write() channel = connection.channel() try: events_received = [0] diff --git a/celery/tests/worker/test_consumer.py b/celery/tests/worker/test_consumer.py index 5880f07ee..d3391dc48 100644 --- a/celery/tests/worker/test_consumer.py +++ b/celery/tests/worker/test_consumer.py @@ -196,8 +196,8 @@ def test_on_close_clears_semaphore_timer_and_reqs(self): c.on_close() def test_connect_error_handler(self): - self.app.connection = _amqp_connection() - conn = self.app.connection.return_value + self.app._connection = _amqp_connection() + conn = self.app._connection.return_value c = self.get_consumer() self.assertTrue(c.connect()) self.assertTrue(conn.ensure_connection.called) @@ -275,7 +275,7 @@ class test_Mingle(AppCase): def test_start_no_replies(self): c = Mock() - c.app.connection = _amqp_connection() + c.app.connection_for_read = _amqp_connection() mingle = Mingle(c) I = c.app.control.inspect.return_value = Mock() I.hello.return_value = {} @@ -284,7 +284,7 @@ def test_start_no_replies(self): def test_start(self): try: c = Mock() - c.app.connection = _amqp_connection() + c.app.connection_for_read = _amqp_connection() mingle = Mingle(c) self.assertTrue(mingle.enabled) @@ -332,14 +332,14 @@ class test_Gossip(AppCase): def test_init(self): c = self.Consumer() - c.app.connection = _amqp_connection() + c.app.connection_for_read = _amqp_connection() g = Gossip(c) self.assertTrue(g.enabled) self.assertIs(c.gossip, g) def test_election(self): c = self.Consumer() - c.app.connection = _amqp_connection() + c.app.connection_for_read = _amqp_connection() g = Gossip(c) g.start(c) g.election('id', 'topic', 'action') @@ -350,7 +350,7 @@ def test_election(self): def test_call_task(self): c = self.Consumer() - c.app.connection = _amqp_connection() + c.app.connection_for_read = _amqp_connection() g = Gossip(c) g.start(c) @@ -381,7 +381,7 @@ def Event(self, id='id', clock=312, def test_on_elect(self): c = self.Consumer() - c.app.connection = _amqp_connection() + c.app.connection_for_read = _amqp_connection() g = Gossip(c) g.start(c) @@ -433,6 +433,7 @@ def setup_election(self, g, c): def test_on_elect_ack_win(self): c = self.Consumer(hostname='foo@x.com') # I will win + c.app.connection_for_read = _amqp_connection() g = Gossip(c) handler = g.election_handlers['topic'] = Mock() self.setup_election(g, c) @@ -440,7 +441,7 @@ def test_on_elect_ack_win(self): def test_on_elect_ack_lose(self): c = self.Consumer(hostname='bar@x.com') # I will lose - c.app.connection = _amqp_connection() + c.app.connection_for_read = _amqp_connection() g = Gossip(c) handler = g.election_handlers['topic'] = Mock() self.setup_election(g, c) @@ -448,6 +449,7 @@ def test_on_elect_ack_lose(self): def test_on_elect_ack_win_but_no_action(self): c = self.Consumer(hostname='foo@x.com') # I will win + c.app.connection_for_read = _amqp_connection() g = Gossip(c) g.election_handlers = {} with patch('celery.worker.consumer.error') as error: @@ -456,6 +458,7 @@ def test_on_elect_ack_win_but_no_action(self): def test_on_node_join(self): c = self.Consumer() + c.app.connection_for_read = _amqp_connection() g = Gossip(c) with patch('celery.worker.consumer.debug') as debug: g.on_node_join(c) @@ -463,6 +466,7 @@ def test_on_node_join(self): def test_on_node_leave(self): c = self.Consumer() + c.app.connection_for_read = _amqp_connection() g = Gossip(c) with patch('celery.worker.consumer.debug') as debug: g.on_node_leave(c) @@ -470,6 +474,7 @@ def test_on_node_leave(self): def test_on_node_lost(self): c = self.Consumer() + c.app.connection_for_read = _amqp_connection() g = Gossip(c) with patch('celery.worker.consumer.info') as info: g.on_node_lost(c) @@ -477,6 +482,7 @@ def test_on_node_lost(self): def test_register_timer(self): c = self.Consumer() + c.app.connection_for_read = _amqp_connection() g = Gossip(c) g.register_timer() c.timer.call_repeatedly.assert_called_with(g.interval, g.periodic) @@ -486,6 +492,7 @@ def test_register_timer(self): def test_periodic(self): c = self.Consumer() + c.app.connection_for_read = _amqp_connection() g = Gossip(c) g.on_node_lost = Mock() state = g.state = Mock() @@ -503,6 +510,7 @@ def test_periodic(self): def test_on_message__task(self): c = self.Consumer() + c.app.connection_for_read = _amqp_connection() g = Gossip(c) self.assertTrue(g.enabled) message = Mock(name='message') @@ -511,6 +519,7 @@ def test_on_message__task(self): def test_on_message(self): c = self.Consumer() + c.app.connection_for_read = _amqp_connection() g = Gossip(c) self.assertTrue(g.enabled) prepare = Mock() diff --git a/celery/tests/worker/test_control.py b/celery/tests/worker/test_control.py index e8356f534..2619cecb8 100644 --- a/celery/tests/worker/test_control.py +++ b/celery/tests/worker/test_control.py @@ -98,7 +98,7 @@ def test_resets(self): def test_loop(self): parent = Mock() - conn = parent.connect.return_value = self.app.connection() + conn = parent.connect.return_value = self.app.connection_for_read() drain = conn.drain_events = Mock() g = gPidbox(parent) parent.connection = Mock() @@ -252,7 +252,7 @@ def test_time_limit(self): def test_active_queues(self): import kombu - x = kombu.Consumer(self.app.connection(), + x = kombu.Consumer(self.app.connection_for_read(), [kombu.Queue('foo', kombu.Exchange('foo'), 'foo'), kombu.Queue('bar', kombu.Exchange('bar'), 'bar')], auto_declare=False) diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index e018d51dc..d2387af54 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -360,7 +360,7 @@ def loop(self, *args, **kwargs): def test_loop_ignores_socket_timeout(self): - class Connection(self.app.connection().__class__): + class Connection(self.app.connection_for_read().__class__): obj = None def drain_events(self, **kwargs): @@ -376,7 +376,7 @@ def drain_events(self, **kwargs): def test_loop_when_socket_error(self): - class Connection(self.app.connection().__class__): + class Connection(self.app.connection_for_read().__class__): obj = None def drain_events(self, **kwargs): @@ -398,7 +398,7 @@ def drain_events(self, **kwargs): def test_loop(self): - class Connection(self.app.connection().__class__): + class Connection(self.app.connection_for_read().__class__): obj = None def drain_events(self, **kwargs): diff --git a/celery/worker/__init__.py b/celery/worker/__init__.py index e85721b95..fe99af132 100644 --- a/celery/worker/__init__.py +++ b/celery/worker/__init__.py @@ -122,7 +122,7 @@ def setup_instance(self, queues=None, ready_callback=None, pidfile=None, self.ready_callback = ready_callback or self.on_consumer_ready # this connection is not established, only used for params - self._conninfo = self.app.connection() + self._conninfo = self.app.connection_for_read() self.use_eventloop = ( self.should_use_eventloop() if use_eventloop is None else use_eventloop diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index dd7d3fc5c..2055f671e 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -178,7 +178,7 @@ def __init__(self, on_task_request, self.pool = pool self.timer = timer self.strategies = self.Strategies() - self.conninfo = self.app.connection() + self.conninfo = self.app.connection_for_read() self.connection_errors = self.conninfo.connection_errors self.channel_errors = self.conninfo.channel_errors self._restart_state = restart_state(maxR=5, maxT=1) @@ -376,7 +376,7 @@ def connect(self): :setting:`broker_connection_retry` setting is enabled """ - conn = self.app.connection(heartbeat=self.amqheartbeat) + conn = self.app.connection_for_read(heartbeat=self.amqheartbeat) # Callback called for each retry while the connection # can't be established. @@ -635,7 +635,7 @@ def __init__(self, c, without_mingle=False, **kwargs): self.enabled = not without_mingle and self.compatible_transport(c.app) def compatible_transport(self, app): - with app.connection() as conn: + with app.connection_for_read() as conn: return conn.transport.driver_type in self.compatible_transports def start(self, c): @@ -776,7 +776,7 @@ def __init__(self, c, without_gossip=False, } def compatible_transport(self, app): - with app.connection() as conn: + with app.connection_for_read() as conn: return conn.transport.driver_type in self.compatible_transports def election(self, id, topic, action=None): diff --git a/docs/configuration.rst b/docs/configuration.rst index 5fe05b352..76401ef35 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1513,6 +1513,25 @@ The brokers will then be used in the :setting:`broker_failover_strategy`. See :ref:`kombu:connection-urls` in the Kombu documentation for more information. +.. setting:: broker_read_url + +.. setting:: broker_write_url + +broker_read_url / broker_write_url +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +These settings can be configured, instead of :setting:`broker_url` to specify +different connection parameters for broker connections used for consuming and +producing. + +Example:: + + broker_read_url = 'amqp://user:pass@broker.example.com:56721' + broker_write_url = 'amqp://user:pass@broker.example.com:56722' + +Both options can also be specified as a list for failover alternates, see +:setting:`broker_url` for more information. + .. setting:: broker_failover_strategy broker_failover_strategy From 362f12086b3e095c194420c2014dda98e8dd49f6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 29 Dec 2015 17:49:34 -0800 Subject: [PATCH 0936/1103] Real fix for #2967 --- celery/task/http.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/celery/task/http.py b/celery/task/http.py index 448b47a00..63eb2c885 100644 --- a/celery/task/http.py +++ b/celery/task/http.py @@ -109,11 +109,12 @@ def __init__(self, url): def __str__(self): scheme, netloc, path, params, query, fragment = self.parts - query = str_to_bytes(urlencode(utf8dict(items(self.query)))) + query = urlencode(utf8dict(items(self.query))) components = [scheme + '://', netloc, path or '/', ';{0}'.format(params) if params else '', '?{0}'.format(query) if query else '', '#{0}'.format(fragment) if fragment else ''] + print('COMP: %r' % (components,)) return ''.join(c for c in components if c) def __repr__(self): @@ -141,7 +142,7 @@ def __init__(self, url, method, task_kwargs, **kwargs): def make_request(self, url, method, params): """Perform HTTP request and return the response.""" - request = Request(url, params) + request = Request(url, str_to_bytes(params)) for key, val in items(self.http_headers): request.add_header(key, val) response = urlopen(request) # user catches errors. From 3364f12e95d5bcced651773cf90f92f93d773c74 Mon Sep 17 00:00:00 2001 From: wyc Date: Tue, 17 Nov 2015 13:42:47 -0500 Subject: [PATCH 0937/1103] Update Django Example and README - Add a result backend - Add requirements.txt - Update README to include requirements and how to run a task --- examples/django/README.rst | 24 ++++++++++++++++++++++++ examples/django/proj/celery.py | 2 ++ examples/django/proj/settings.py | 1 + examples/django/requirements.txt | 2 ++ 4 files changed, 29 insertions(+) create mode 100644 examples/django/requirements.txt diff --git a/examples/django/README.rst b/examples/django/README.rst index 9eebc02ad..e41e9b84e 100644 --- a/examples/django/README.rst +++ b/examples/django/README.rst @@ -27,6 +27,19 @@ Example generic app. This is decoupled from the rest of the project by using the ``@shared_task`` decorator. This decorator returns a proxy that always points to the currently active Celery instance. +Installing requirements +======================= + +The settings file assumes that ``rabbitmq-server`` is running on ``localhost`` +using the default ports. More information here: + +http://docs.celeryproject.org/en/latest/getting-started/brokers/rabbitmq.html + +In addition, some Python requirements must also be satisfied: + +.. code-block:: bash + + $ pip install -r requirements.txt Starting the worker =================== @@ -34,3 +47,14 @@ Starting the worker .. code-block:: bash $ celery -A proj worker -l info + +Running a task +=================== + +.. code-block:: bash + + $ python ./manage.sh shell + >>> from demoapp.tasks import add, mul, xsum + >>> res = add.delay(2,3) + >>> res.get() + 5 diff --git a/examples/django/proj/celery.py b/examples/django/proj/celery.py index d7ea41a48..f35ee8299 100644 --- a/examples/django/proj/celery.py +++ b/examples/django/proj/celery.py @@ -7,6 +7,8 @@ # set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings') +from django.conf import settings # noqa + app = Celery('proj') # Using a string here means the worker will not have to diff --git a/examples/django/proj/settings.py b/examples/django/proj/settings.py index 2b61b564e..8ed566b37 100644 --- a/examples/django/proj/settings.py +++ b/examples/django/proj/settings.py @@ -11,6 +11,7 @@ #: Only add pickle to this list if your broker is secured #: from unwanted access (see userguide/security.html) CELERY_ACCEPT_CONTENT = ['json'] +CELERY_RESULT_BACKEND = 'db+sqlite:///results.sqlite' # Django settings for proj project. diff --git a/examples/django/requirements.txt b/examples/django/requirements.txt new file mode 100644 index 000000000..77a33d8e4 --- /dev/null +++ b/examples/django/requirements.txt @@ -0,0 +1,2 @@ +django==1.8.4 +sqlalchemy==1.0.9 From fc1ce73c09915d8d5de759deb9c223a104113f42 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 30 Dec 2015 00:24:10 -0800 Subject: [PATCH 0938/1103] Removes left over print statements. Closes #2967 --- celery/task/http.py | 1 - celery/tests/bin/test_celeryd_detach.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/celery/task/http.py b/celery/task/http.py index 63eb2c885..609026a14 100644 --- a/celery/task/http.py +++ b/celery/task/http.py @@ -114,7 +114,6 @@ def __str__(self): ';{0}'.format(params) if params else '', '?{0}'.format(query) if query else '', '#{0}'.format(fragment) if fragment else ''] - print('COMP: %r' % (components,)) return ''.join(c for c in components if c) def __repr__(self): diff --git a/celery/tests/bin/test_celeryd_detach.py b/celery/tests/bin/test_celeryd_detach.py index f12e445b2..0e1d0169a 100644 --- a/celery/tests/bin/test_celeryd_detach.py +++ b/celery/tests/bin/test_celeryd_detach.py @@ -61,8 +61,6 @@ def test_parser(self): '--logfile=foo', '--fake', '--enable', 'a', 'b', '-c1', '-d', '2', ]) - print(p.option_list) - print('O: %r V: %r' % (vars(options), values)) self.assertEqual(options.logfile, 'foo') self.assertEqual(values, ['a', 'b']) self.assertEqual(p.leftovers, ['--enable', '-c1', '-d', '2']) From 21f96040de9c364222a98429b5b1129a020a73af Mon Sep 17 00:00:00 2001 From: Fernando Rocha Date: Wed, 30 Dec 2015 18:37:48 -0300 Subject: [PATCH 0939/1103] Fix typo in docs AttributeError: 'TaskProducer' object has no attribute 'send' --- docs/userguide/extending.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index 1080d78a0..c436915ca 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -46,7 +46,7 @@ whenever the connection is established: def send_me_a_message(self, who='world!', producer=None): with app.producer_or_acquire(producer) as producer: - producer.send( + producer.publish( {'hello': who}, serializer='json', exchange=my_queue.exchange, From 509ed75f960a62fe73f6fae2d266a983c5b885fb Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 4 Jan 2016 13:50:39 -0800 Subject: [PATCH 0940/1103] Snapshot example should enable clear_after to match description. Closes #2962 --- docs/userguide/monitoring.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index c3df06960..eb5f42160 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -504,6 +504,7 @@ Here is an example camera, dumping the snapshot to screen: from celery.events.snapshot import Polaroid class DumpCam(Polaroid): + clear_after = True # clear after flush (incl, state.event_count). def on_shutter(self, state): if not state.event_count: From b784c7912afcab34632bf60b935b929f218c9180 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 5 Jan 2016 15:29:35 -0800 Subject: [PATCH 0941/1103] Use the new logging.NullHandler in Python 2.7 --- celery/app/log.py | 7 ++++--- celery/tests/case.py | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/celery/app/log.py b/celery/app/log.py index 4c8fb030e..9b643217f 100644 --- a/celery/app/log.py +++ b/celery/app/log.py @@ -18,7 +18,6 @@ from logging.handlers import WatchedFileHandler -from kombu.log import NullHandler from kombu.utils.encoding import set_default_encoding_file from celery import signals @@ -231,8 +230,10 @@ def _detect_handler(self, logfile=None): return WatchedFileHandler(logfile) def _has_handler(self, logger): - if logger.handlers: - return any(not isinstance(h, NullHandler) for h in logger.handlers) + return any( + not isinstance(h, logging.NullHandler) + for h in logger.handlers or [] + ) def _is_configured(self, logger): return self._has_handler(logger) and not getattr( diff --git a/celery/tests/case.py b/celery/tests/case.py index d342f1dd8..c93e6bbaf 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -33,7 +33,6 @@ import mock # noqa from nose import SkipTest from kombu import Queue -from kombu.log import NullHandler from kombu.utils import symbol_by_name from celery import Celery @@ -561,7 +560,10 @@ def teardown(self): def get_handlers(logger): - return [h for h in logger.handlers if not isinstance(h, NullHandler)] + return [ + h for h in logger.handlers + if not isinstance(h, logging.NullHandler) + ] @contextmanager From 500523a284503d8370280a72de3ce2da522b07a6 Mon Sep 17 00:00:00 2001 From: Marcio Ribeiro Date: Thu, 7 Jan 2016 17:04:44 -0200 Subject: [PATCH 0942/1103] Removed duplicated field `__bound__` --- celery/app/task.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index 5aac03058..12271aa4d 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -272,8 +272,6 @@ class Task(object): _backend = None # set by backend property. - __bound__ = False - # - Tasks are lazily bound, so that configuration is not set # - until the task is actually used From bc373013d84b034ba7b1f2c679062a928d376d63 Mon Sep 17 00:00:00 2001 From: Valentyn Klindukh Date: Fri, 8 Jan 2016 18:13:50 +0200 Subject: [PATCH 0943/1103] do not destroy connection to mongo --- celery/backends/mongodb.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 8935d0d81..e217639c3 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -162,14 +162,6 @@ def _get_connection(self): return self._connection - def process_cleanup(self): - if self._connection is not None: - # MongoDB connection will be closed automatically when object - # goes out of scope - del(self.collection) - del(self.database) - self._connection = None - def encode(self, data): if self.serializer == 'bson': # mongodb handles serialization From 64c596fc9f65065fdfb3e43b6af43162343dbc7e Mon Sep 17 00:00:00 2001 From: Jason Veatch Date: Fri, 8 Jan 2016 16:32:03 -0500 Subject: [PATCH 0944/1103] More detail about prefetch and long-running tasks When reading docs, I thought T3 would be stuck behind T1, but other tasks would be sent to available workers. I've seen that queued tasks aren't sent to available workers unless -Ofair is used, so clarifying this description. --- docs/userguide/optimizing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/userguide/optimizing.rst b/docs/userguide/optimizing.rst index 7d37c9865..934ec7055 100644 --- a/docs/userguide/optimizing.rst +++ b/docs/userguide/optimizing.rst @@ -217,8 +217,8 @@ waiting for long running tasks to complete:: <- T2 complete -> send T3 to Process A - # A still executing T1, T3 stuck in local buffer and - # will not start until T1 returns + # A still executing T1, T3 stuck in local buffer and will not start until + # T1 returns, and other queued tasks will not be sent to idle workers The worker will send tasks to the process as long as the pipe buffer is writable. The pipe buffer size varies based on the operating system: some may From f94dddd50855e275bbee5624dec6397dd0b6ff4d Mon Sep 17 00:00:00 2001 From: Will Thompson Date: Tue, 12 Jan 2016 18:22:01 +0000 Subject: [PATCH 0945/1103] =?UTF-8?q?docs:=20remove=20duplicated=20word=20?= =?UTF-8?q?=E2=80=9Cacknowledgements=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduced in 2e8b4de. --- docs/userguide/optimizing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/optimizing.rst b/docs/userguide/optimizing.rst index 7d37c9865..42cc0465f 100644 --- a/docs/userguide/optimizing.rst +++ b/docs/userguide/optimizing.rst @@ -184,7 +184,7 @@ When users ask if it's possible to disable "prefetching of tasks", often what they really want is to have a worker only reserve as many tasks as there are child processes. -But this is not possible without enabling late acknowledgements +But this is not possible without enabling late acknowledgements; A task that has been started, will be retried if the worker crashes mid execution so the task must be `idempotent`_ (see also notes at :ref:`faq-acks_late-vs-retry`). From 07a9a851c4bae31a2aba4eb7291976de5a215474 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 12 Jan 2016 11:34:28 -0800 Subject: [PATCH 0946/1103] [docs] Rewording portions of the optimizing guide (Issue #2998) --- docs/glossary.rst | 28 ++++++++++++++++++++++++++++ docs/userguide/optimizing.rst | 31 ++++++++++++++++++++----------- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/docs/glossary.rst b/docs/glossary.rst index 32ad2395e..c66daf2ae 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -18,6 +18,32 @@ Glossary ack Short for :term:`acknowledged`. + early acknowledgement + + Task is :term:`acknowledged` just-in-time before being executed, + meaning the task will not be redelivered to another worker if the + machine loses power, or the worker instance is abruptly killed, + mid-execution. + + Configured using :setting:`task_acks_late`. + + late acknowledgment + + Task is :term:`acknowledged` after execution (both if successful, or + if the task is raising an error), which means the task will be + redelivered to another worker in the event of the machine losing + power, or the worker instance being killed mid-execution. + + Configured using :setting:`task_acks_late`. + + early ack + + Short for :term:`early acknowledgement` + + late ack + + Short for :term:`late acknowledgement` + request Task messages are converted to *requests* within the worker. The request information is also available as the task's @@ -54,6 +80,8 @@ Glossary unintended effects, but not necessarily side-effect free in the pure sense (compare to :term:`nullipotent`). + Further reading: http://en.wikipedia.org/wiki/Idempotent + nullipotent describes a function that will have the same effect, and give the same result, even if called zero or multiple times (side-effect free). diff --git a/docs/userguide/optimizing.rst b/docs/userguide/optimizing.rst index 7d37c9865..fc9ce54c0 100644 --- a/docs/userguide/optimizing.rst +++ b/docs/userguide/optimizing.rst @@ -176,20 +176,29 @@ the tasks according to the run-time. (see :ref:`guide-routing`). Reserve one task at a time -------------------------- -When using early acknowledgement (default), a prefetch multiplier of 1 -means the worker will reserve at most one extra task for every active -worker process. +The task message is only deleted from the queue after the task is +:term:`acknowledged`, so if the worker crashes before acknowleding the task, +it can be redelivered to another worker (or the same after recovery). -When users ask if it's possible to disable "prefetching of tasks", often -what they really want is to have a worker only reserve as many tasks as there -are child processes. +When using the default of early acknowledgement, having a prefetch multiplier setting +of 1, means the worker will reserve at most one extra task for every +worker process: or in other words, if the worker is started with `-c 10`, +the worker may reserve at most 20 tasks (10 unacknowledged tasks executing, and 10 +unacknowledged reserved tasks) at any time. -But this is not possible without enabling late acknowledgements -acknowledgements; A task that has been started, will be -retried if the worker crashes mid execution so the task must be `idempotent`_ -(see also notes at :ref:`faq-acks_late-vs-retry`). +Often users ask if disabling "prefetching of tasks" is possible, but what +they really mean by that is to have a worker only reserve as many tasks as +there are worker processes (10 unacknowledged tasks for `-c 10`) -.. _`idempotent`: http://en.wikipedia.org/wiki/Idempotent +That is possible, but not without also enabling +:term:`late acknowledgments`. Using this option over the +default beahvior means a task that has already started executing will be +retried in the event of a power failure or the worker instance being killed +abruptly, so this also means the task must be :term:`idempotent` + +.. seealso:: + + Notes at :ref:`faq-acks_late-vs-retry`. You can enable this behavior by using the following configuration options: From 00b2930325204c47e6b32b6abdfb6734d7445016 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 12 Jan 2016 11:47:24 -0800 Subject: [PATCH 0947/1103] Updates copyright year --- LICENSE | 2 +- celery/__init__.py | 2 +- docs/conf.py | 2 +- docs/copyright.rst | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/LICENSE b/LICENSE index 92a530c9b..06221a278 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2015 Ask Solem & contributors. All rights reserved. +Copyright (c) 2015-2016 Ask Solem & contributors. All rights reserved. Copyright (c) 2012-2014 GoPivotal, Inc. All rights reserved. Copyright (c) 2009, 2010, 2011, 2012 Ask Solem, and individual contributors. All rights reserved. diff --git a/celery/__init__.py b/celery/__init__.py index e6d0b214a..5f3911fce 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """Distributed Task Queue""" -# :copyright: (c) 2015 Ask Solem. All rights reserved. +# :copyright: (c) 2015-2016 Ask Solem. All rights reserved. # :copyright: (c) 2012-2014 GoPivotal, Inc., All rights reserved. # :copyright: (c) 2009 - 2012 Ask Solem and individual contributors, # All rights reserved. diff --git a/docs/conf.py b/docs/conf.py index 694af4ee6..867025d40 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -54,7 +54,7 @@ def linkcode_resolve(domain, info): # General information about the project. project = 'Celery' -copyright = '2009-2015, Ask Solem & Contributors' +copyright = '2009-2016, Ask Solem & Contributors' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/docs/copyright.rst b/docs/copyright.rst index 7a78c9c27..2295029a8 100644 --- a/docs/copyright.rst +++ b/docs/copyright.rst @@ -7,7 +7,7 @@ by Ask Solem .. |copy| unicode:: U+000A9 .. COPYRIGHT SIGN -Copyright |copy| 2009-2015, Ask Solem. +Copyright |copy| 2009-2016, Ask Solem. All rights reserved. This material may be copied or distributed only subject to the terms and conditions set forth in the `Creative Commons From 597a6b1f3359065ff6dbabce7237f86b866313df Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 12 Jan 2016 13:25:11 -0800 Subject: [PATCH 0948/1103] [Django] Ignore InterfaceError when closing database connection. Closes #2996 --- celery/fixups/django.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index e7578004a..6b0ad4462 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -119,6 +119,13 @@ def __init__(self, app): self._cache = import_module('django.core.cache') self._settings = symbol_by_name('django.conf:settings') + try: + self.interface_errors = ( + symbol_by_name('django.db.utils.InterfaceError'), + ) + except (ImportError, AttributeError): + self._interface_errors = () + # Database-related exceptions. DatabaseError = symbol_by_name('django.db:DatabaseError') try: @@ -269,6 +276,8 @@ def _close_database(self): for close in funs: try: close() + except self.interface_errors: + pass except self.database_errors as exc: str_exc = str(exc) if 'closed' not in str_exc and 'not connected' not in str_exc: From baa1282c04c5cc05068dca4f345b9368f3b128a1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 12 Jan 2016 15:36:38 -0800 Subject: [PATCH 0949/1103] [Django] Ignore InterfaceError also after fork (Issue #2996) --- celery/fixups/django.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index 6b0ad4462..5151ff082 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -232,15 +232,21 @@ def on_worker_process_init(self, **kwargs): try: for c in self._db.connections.all(): if c and c.connection: - _maybe_close_fd(c.connection) + self._maybe_close_db_fd(c.connection) except AttributeError: if self._db.connection and self._db.connection.connection: - _maybe_close_fd(self._db.connection.connection) + self._maybe_close_db_fd(self._db.connection.connection) # use the _ version to avoid DB_REUSE preventing the conn.close() call self._close_database() self.close_cache() + def _maybe_close_db_fd(self, fd): + try: + _maybe_close_fd(fd) + except self.interface_errors: + pass + def on_task_prerun(self, sender, **kwargs): """Called before every task.""" if not getattr(sender.request, 'is_eager', False): From e71c5eff0202fd3260e6a67c991897b28a77aced Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 12 Jan 2016 15:36:52 -0800 Subject: [PATCH 0950/1103] flakes --- celery/app/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 385762282..2d662e0ea 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -714,10 +714,10 @@ def connection(self, hostname=None, userid=None, password=None, ) def _connection(self, url, userid=None, password=None, - virtual_host=None, port=None, ssl=None, - connect_timeout=None, transport=None, - transport_options=None, heartbeat=None, - login_method=None, failover_strategy=None, **kwargs): + virtual_host=None, port=None, ssl=None, + connect_timeout=None, transport=None, + transport_options=None, heartbeat=None, + login_method=None, failover_strategy=None, **kwargs): conf = self.conf return self.amqp.Connection( url, From 9b1825c43d5aaa2274581b27b5ef79302b3802ee Mon Sep 17 00:00:00 2001 From: Caleb Mingle Date: Wed, 13 Jan 2016 14:56:12 -0800 Subject: [PATCH 0951/1103] Fixes doc typo in celery.contrib.batches Should say "10 seconds" instead of just "seconds". --- celery/contrib/batches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/contrib/batches.py b/celery/contrib/batches.py index c1b1b4c9d..0ceac4aad 100644 --- a/celery/contrib/batches.py +++ b/celery/contrib/batches.py @@ -17,7 +17,7 @@ **Simple Example** A click counter that flushes the buffer every 100 messages, and every -seconds. Does not do anything with the data, but can easily be modified +10 seconds. Does not do anything with the data, but can easily be modified to store it in a database. .. code-block:: python From 0f854ce519445df229c25ab4cce82dc1549bfbc9 Mon Sep 17 00:00:00 2001 From: Morton Fox Date: Thu, 14 Jan 2016 11:15:57 -0500 Subject: [PATCH 0952/1103] Update RCelery link --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 8622d7141..f7364034d 100644 --- a/README.rst +++ b/README.rst @@ -34,7 +34,7 @@ any language. So far there's RCelery_ for the Ruby programming language, and a `PHP client`, but language interoperability can also be achieved by using webhooks. -.. _RCelery: http://leapfrogdevelopment.github.com/rcelery/ +.. _RCelery: http://leapfrogonline.github.io/rcelery/ .. _`PHP client`: https://github.com/gjedeer/celery-php .. _`using webhooks`: http://docs.celeryproject.org/en/latest/userguide/remote-tasks.html From 2a53f7eb76dd6c05927a072e91ccdabbc18191dd Mon Sep 17 00:00:00 2001 From: Omer Korner Date: Fri, 15 Jan 2016 18:57:46 +0200 Subject: [PATCH 0953/1103] add example for broadcast queue and celerybeat --- docs/userguide/routing.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/userguide/routing.rst b/docs/userguide/routing.rst index 0e72f406b..d883e9a2b 100644 --- a/docs/userguide/routing.rst +++ b/docs/userguide/routing.rst @@ -598,6 +598,24 @@ copies of tasks to all workers connected to it: Now the ``tasks.reload_cache`` task will be sent to every worker consuming from this queue. +Here is another example of broadcast routing, this time with +a celerybeat schedule: + +.. code-block:: python + + from kombu.common import Broadcast + from celery.schedules import crontab + + task_queues = (Broadcast('broadcast_tasks'),) + + task_routes = {'test-task': { + 'task': 'tasks.reload_cache', + 'schedule': crontab(minute=0, hour='*/3'), + 'options': {'exchange': 'broadcast_tasks'} + }, + } + + .. admonition:: Broadcast & Results Note that Celery result does not define what happens if two From c1cd6c6c2e2d4b3b7d70290f7ebeb14ae5ea4dc8 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 15 Jan 2016 11:52:40 -0800 Subject: [PATCH 0954/1103] Fixes build --- celery/tests/fixups/test_django.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/tests/fixups/test_django.py b/celery/tests/fixups/test_django.py index 8da192e03..45ae675df 100644 --- a/celery/tests/fixups/test_django.py +++ b/celery/tests/fixups/test_django.py @@ -261,6 +261,7 @@ def test__close_database(self): conns = [Mock(), Mock(), Mock()] conns[1].close.side_effect = KeyError('already closed') f.database_errors = (KeyError,) + f.interface_errors = () f._db.connections = Mock() # ConnectionHandler f._db.connections.all.side_effect = lambda: conns From 31e8fb24a096fa8a769c1dc090824d1fbe7a0855 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 15 Jan 2016 12:36:53 -0800 Subject: [PATCH 0955/1103] Fixes route with queue name value not working regression. Closes #2987 --- celery/app/routes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/celery/app/routes.py b/celery/app/routes.py index 06ab34abc..c428035b8 100644 --- a/celery/app/routes.py +++ b/celery/app/routes.py @@ -33,6 +33,8 @@ def route_for_task(self, task, *args, **kwargs): return dict(self.map[task]) except KeyError: pass + except ValueError: + return {'queue': self.map[task]} class Router(object): From 53e47ac07ecdaa45473e6a6f455be95424c9e5e3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 15 Jan 2016 13:09:19 -0800 Subject: [PATCH 0956/1103] Fixes build for #2993 --- celery/tests/backends/test_mongodb.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index 6419878e6..f84ee424b 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -206,15 +206,6 @@ def test_get_database_no_existing_no_auth(self, mock_get_connection): self.assertFalse(mock_database.authenticate.called) self.assertTrue(self.backend.__dict__['database'] is mock_database) - def test_process_cleanup(self): - self.backend._connection = None - self.backend.process_cleanup() - self.assertEqual(self.backend._connection, None) - - self.backend._connection = 'not none' - self.backend.process_cleanup() - self.assertEqual(self.backend._connection, None) - @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION From e4b226d8703290d3f2d8d2fbf253f2d9a765588e Mon Sep 17 00:00:00 2001 From: Omer Korner Date: Fri, 15 Jan 2016 23:28:54 +0200 Subject: [PATCH 0957/1103] fixed wrong setting name --- docs/userguide/routing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/routing.rst b/docs/userguide/routing.rst index d883e9a2b..4183a5303 100644 --- a/docs/userguide/routing.rst +++ b/docs/userguide/routing.rst @@ -608,7 +608,7 @@ a celerybeat schedule: task_queues = (Broadcast('broadcast_tasks'),) - task_routes = {'test-task': { + beat_schedule = {'test-task': { 'task': 'tasks.reload_cache', 'schedule': crontab(minute=0, hour='*/3'), 'options': {'exchange': 'broadcast_tasks'} From ad0585140920dd4a3f67d29631ff3a632ab73f4e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 15 Jan 2016 13:36:50 -0800 Subject: [PATCH 0958/1103] Decrease coverage target, ugh. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6e006f4aa..18d35e40a 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@ recreate = False commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} pip install -q -U -r{toxinidir}/requirements/dev.txt nosetests -xsv --with-coverage \ - --cover-inclusive --cover-min-percentage=95 --cover-erase [] + --cover-inclusive --cover-min-percentage=94 --cover-erase [] basepython = 2.7: python2.7 From e6105afe64381d67b703adaeeff15b337a78209a Mon Sep 17 00:00:00 2001 From: Valentyn Klindukh Date: Fri, 15 Jan 2016 23:41:04 +0200 Subject: [PATCH 0959/1103] Update CONTRIBUTORS.txt adding myself, https://github.com/celery/celery/issues/2992 --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index aa2ce705f..486d5882d 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -200,3 +200,4 @@ Sukrit Khera, 2015/10/26 Dave Smith, 2015/10/27 Dennis Brakhane, 2015/10/30 Chris Harris, 2015/11/27 +Valentyn Klindukh, 2016/01/15 From c3f7addc1d4140199a1b34544593338712600ea1 Mon Sep 17 00:00:00 2001 From: Jason Veatch Date: Fri, 15 Jan 2016 16:53:29 -0500 Subject: [PATCH 0960/1103] Improved wording about prefetching --- docs/userguide/optimizing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/optimizing.rst b/docs/userguide/optimizing.rst index 934ec7055..757563a0e 100644 --- a/docs/userguide/optimizing.rst +++ b/docs/userguide/optimizing.rst @@ -218,7 +218,7 @@ waiting for long running tasks to complete:: -> send T3 to Process A # A still executing T1, T3 stuck in local buffer and will not start until - # T1 returns, and other queued tasks will not be sent to idle workers + # T1 returns, and other queued tasks will not be sent to idle processes The worker will send tasks to the process as long as the pipe buffer is writable. The pipe buffer size varies based on the operating system: some may From b6ab2cd74b6f3e44664d08c20d71513ed68c5ca7 Mon Sep 17 00:00:00 2001 From: wyc Date: Fri, 15 Jan 2016 16:59:20 -0500 Subject: [PATCH 0961/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 486d5882d..7119ca993 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -201,3 +201,4 @@ Dave Smith, 2015/10/27 Dennis Brakhane, 2015/10/30 Chris Harris, 2015/11/27 Valentyn Klindukh, 2016/01/15 +Wayne Chang, 2016/01/15 From 7824d0d4ddccdecab5a4b630bd815f77e5eb437f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Jan 2016 12:16:33 -0800 Subject: [PATCH 0962/1103] [Database result backend] Fixes JSON serialization of exceptions (Issue #2441) --- celery/backends/database/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/database/__init__.py b/celery/backends/database/__init__.py index 858092619..3c423960d 100644 --- a/celery/backends/database/__init__.py +++ b/celery/backends/database/__init__.py @@ -134,7 +134,7 @@ def _get_task_meta_for(self, task_id): task = Task(task_id) task.status = states.PENDING task.result = None - return task.to_dict() + return self.meta_from_decoded(task.to_dict()) @retry def _save_group(self, group_id, result): From 92621d5ab483d26fc12727b2c7a7afeda51fdc28 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Jan 2016 12:16:50 -0800 Subject: [PATCH 0963/1103] Wording --- CONTRIBUTORS.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 7119ca993..17040ebeb 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -11,7 +11,7 @@ that everyone must add themselves here, and not be added by others, so it's currently incomplete waiting for everyone to add their names. -The full list of authors can be found in docs/AUTHORS.txt. +The list of authors added before the policy change can be found in docs/AUTHORS.txt. -- From c89c4b112bee637b473c80bf8fb44064cf7eb538 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Jan 2016 12:56:27 -0800 Subject: [PATCH 0964/1103] Fixes problem with chains when using task_protocol 1 in master (Issue #3009) --- celery/canvas.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 299b38e9c..ba75c9409 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -271,14 +271,22 @@ def apply_async(self, args=(), kwargs={}, route_name=None, **options): args, kwargs, options = self.args, self.kwargs, self.options return _apply(args, kwargs, **options) - def append_to_list_option(self, key, value): + def _with_list_option(self, key): items = self.options.setdefault(key, []) if not isinstance(items, MutableSequence): items = self.options[key] = [items] + return items + + def append_to_list_option(self, key, value): + items = self._with_list_option(key) if value not in items: items.append(value) return value + def extend_list_option(self, key, value): + items = self._with_list_option(key) + items.extend(maybe_list(value)) + def link(self, callback): return self.append_to_list_option('link', callback) @@ -418,6 +426,8 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, producer=None, root_id=None, parent_id=None, app=None, **options): app = app or self.app use_link = self._use_link + if use_link is None and app.conf.task_protocol == 1: + use_link = True args = (tuple(args) + tuple(self.args) if args and not self.immutable else self.args) @@ -431,7 +441,7 @@ def run(self, args=(), kwargs={}, group_id=None, chord=None, if results: if link: - tasks[0].set(link=link) + tasks[0].extend_list_option('link', link) first_task = tasks.pop() first_task.apply_async( chain=tasks if not use_link else None, **options) @@ -456,8 +466,8 @@ def prepare_steps(self, args, tasks, # (why is pickle using recursion? or better yet why cannot python # do tail call optimization making recursion actually useful?) use_link = self._use_link - if use_link is None and app.conf.task_protocol > 1: - use_link = False + if use_link is None and app.conf.task_protocol == 1: + use_link = True steps = deque(tasks) steps_pop = steps.pop From 5a718b726f508c3183e7f644aba271bbbe5339cb Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Jan 2016 13:09:07 -0800 Subject: [PATCH 0965/1103] Adds timestamp to worker/beat startup banners. Closes #3010 --- celery/apps/beat.py | 4 ++++ celery/apps/worker.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/celery/apps/beat.py b/celery/apps/beat.py index 24b6828d8..b66756adb 100644 --- a/celery/apps/beat.py +++ b/celery/apps/beat.py @@ -16,6 +16,8 @@ import socket import sys +from datetime import datetime + from celery import VERSION_BANNER, platforms, beat from celery.five import text_t from celery.utils.imports import qualname @@ -25,6 +27,7 @@ __all__ = ['Beat'] STARTUP_INFO_FMT = """ +LocalTime -> {timestamp} Configuration -> . broker -> {conninfo} . loader -> {loader} @@ -124,6 +127,7 @@ def startup_info(self, beat): scheduler = beat.get_scheduler(lazy=True) return STARTUP_INFO_FMT.format( conninfo=self.app.connection().as_uri(), + timestamp=datetime.now().replace(microsecond=0), logfile=self.logfile or '[stderr]', loglevel=LOG_LEVELS[self.loglevel], loader=qualname(self.app.loader), diff --git a/celery/apps/worker.py b/celery/apps/worker.py index a67389bd8..7198172fe 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -17,6 +17,7 @@ import platform as _platform import sys +from datetime import datetime from functools import partial from billiard.process import current_process @@ -69,7 +70,7 @@ def safe_say(msg): BANNER = """\ {hostname} v{version} -{platform} +{platform} {timestamp} [config] .> app: {app} @@ -202,6 +203,7 @@ def startup_info(self): banner = BANNER.format( app=appr, hostname=safe_str(self.hostname), + timestamp=datetime.now().replace(microsecond=0), version=VERSION_BANNER, conninfo=self.app.connection().as_uri(), results=maybe_sanitize_url( From e2cde3448bacfaf1fc3ce54c8658c39aac04b224 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Jan 2016 13:39:19 -0800 Subject: [PATCH 0966/1103] Fixes build --- celery/canvas.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index ba75c9409..e7e18891f 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -525,12 +525,12 @@ def prepare_steps(self, args, tasks, if prev_task: prev_task.set_parent_id(task.id) + if use_link: # link previous task to this task. task.link(prev_task) - if not res.parent and prev_res: - prev_res.parent = res.parent - elif prev_res: + + if prev_res: prev_res.parent = res if is_first_task and parent_id is not None: From 424eb054dce3ad2efa902b8452c7ee1446c8089c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 19 Jan 2016 14:15:46 -0800 Subject: [PATCH 0967/1103] Cryptography crashes Py2.7, so avoid including it in CI --- requirements/test-ci-default.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements/test-ci-default.txt b/requirements/test-ci-default.txt index 3b354d8ad..6d0b42f6b 100644 --- a/requirements/test-ci-default.txt +++ b/requirements/test-ci-default.txt @@ -1,3 +1,4 @@ -r test-ci-base.txt --r extras/auth.txt +#: Disabled for Cryptography crashing on 2.7 after interpreter shutdown. +#-r extras/auth.txt -r extras/riak.txt From 7d545d8906b1a2289ea0357ffffc119544dc0f18 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 20 Jan 2016 12:32:04 -0800 Subject: [PATCH 0968/1103] Cosmetics --- celery/worker/control.py | 53 +++++++++++++++++++++++----------------- celery/worker/request.py | 24 +++++++++--------- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/celery/worker/control.py b/celery/worker/control.py index 1d4b8e711..74ac0c33f 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -31,6 +31,14 @@ logger = get_logger(__name__) +def ok(value): + return {'ok': value} + + +def nok(value): + return {'error': value} + + class Panel(UserDict): data = dict() # Global registry. @@ -90,17 +98,17 @@ def revoke(state, task_id, terminate=False, signal=None, **kwargs): break if not terminated: - return {'ok': 'terminate: tasks unknown'} - return {'ok': 'terminate: {0}'.format(', '.join(terminated))} + return ok('terminate: tasks unknown') + return ok('terminate: {0}'.format(', '.join(terminated))) idstr = ', '.join(task_ids) logger.info('Tasks flagged as revoked: %s', idstr) - return {'ok': 'tasks {0} flagged as revoked'.format(idstr)} + return ok('tasks {0} flagged as revoked'.format(idstr)) @Panel.register def report(state): - return {'ok': state.app.bugreport()} + return ok(state.app.bugreport()) @Panel.register @@ -109,8 +117,8 @@ def enable_events(state): if dispatcher.groups and 'task' not in dispatcher.groups: dispatcher.groups.add('task') logger.info('Events of group {task} enabled by remote.') - return {'ok': 'task events enabled'} - return {'ok': 'task events already enabled'} + return ok('task events enabled') + return ok('task events already enabled') @Panel.register @@ -119,8 +127,8 @@ def disable_events(state): if 'task' in dispatcher.groups: dispatcher.groups.discard('task') logger.info('Events of group {task} disabled by remote.') - return {'ok': 'task events disabled'} - return {'ok': 'task events already disabled'} + return ok('task events disabled') + return ok('task events already disabled') @Panel.register @@ -144,24 +152,24 @@ def rate_limit(state, task_name, rate_limit, **kwargs): try: timeutils.rate(rate_limit) except ValueError as exc: - return {'error': 'Invalid rate limit string: {0!r}'.format(exc)} + return nok('Invalid rate limit string: {0!r}'.format(exc)) try: state.app.tasks[task_name].rate_limit = rate_limit except KeyError: logger.error('Rate limit attempt for unknown task %s', task_name, exc_info=True) - return {'error': 'unknown task'} + return nok('unknown task') state.consumer.reset_rate_limits() if not rate_limit: logger.info('Rate limits disabled for tasks of type %s', task_name) - return {'ok': 'rate limit disabled successfully'} + return ok('rate limit disabled successfully') logger.info('New rate limit for tasks of type %s: %s.', task_name, rate_limit) - return {'ok': 'new rate limit set successfully'} + return ok('new rate limit set successfully') @Panel.register @@ -171,14 +179,14 @@ def time_limit(state, task_name=None, hard=None, soft=None, **kwargs): except KeyError: logger.error('Change time limit attempt for unknown task %s', task_name, exc_info=True) - return {'error': 'unknown task'} + return nok('unknown task') task.soft_time_limit = soft task.time_limit = hard logger.info('New time limits for tasks of type %s: soft=%s hard=%s', task_name, soft, hard) - return {'ok': 'time limits set successfully'} + return ok('time limits set successfully') @Panel.register @@ -295,7 +303,7 @@ def _extract_info(task): @Panel.register def ping(state, **kwargs): - return {'ok': 'pong'} + return ok('pong') @Panel.register @@ -305,7 +313,7 @@ def pool_grow(state, n=1, **kwargs): else: state.consumer.pool.grow(n) state.consumer._update_prefetch_count(n) - return {'ok': 'pool will grow'} + return ok('pool will grow') @Panel.register @@ -315,14 +323,14 @@ def pool_shrink(state, n=1, **kwargs): else: state.consumer.pool.shrink(n) state.consumer._update_prefetch_count(-n) - return {'ok': 'pool will shrink'} + return ok('pool will shrink') @Panel.register def pool_restart(state, modules=None, reload=False, reloader=None, **kwargs): if state.app.conf.worker_pool_restarts: state.consumer.controller.reload(modules, reload, reloader=reloader) - return {'ok': 'reload started'} + return ok('reload started') else: raise ValueError('Pool restarts not enabled') @@ -332,7 +340,7 @@ def autoscale(state, max=None, min=None): autoscaler = state.consumer.controller.autoscaler if autoscaler: max_, min_ = autoscaler.update(max, min) - return {'ok': 'autoscale now min={0} max={1}'.format(max_, min_)} + return ok('autoscale now min={0} max={1}'.format(max_, min_)) raise ValueError('Autoscale not enabled') @@ -349,7 +357,7 @@ def add_consumer(state, queue, exchange=None, exchange_type=None, state.consumer.add_task_queue, queue, exchange, exchange_type, routing_key, **options ) - return {'ok': 'add consumer {0}'.format(queue)} + return ok('add consumer {0}'.format(queue)) @Panel.register @@ -357,7 +365,7 @@ def cancel_consumer(state, queue=None, **_): state.consumer.call_soon( state.consumer.cancel_task_queue, queue, ) - return {'ok': 'no longer consuming from {0}'.format(queue)} + return ok('no longer consuming from {0}'.format(queue)) @Panel.register @@ -370,8 +378,7 @@ def active_queues(state): def _wanted_config_key(key): - return (isinstance(key, string_t) and - not key.startswith('__')) + return isinstance(key, string_t) and not key.startswith('__') @Panel.register diff --git a/celery/worker/request.py b/celery/worker/request.py index 06b210d47..06921efc1 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -394,17 +394,19 @@ def reject(self, requeue=False): self.send_event('task-rejected', requeue=requeue) def info(self, safe=False): - return {'id': self.id, - 'name': self.name, - 'args': self.argsrepr, - 'kwargs': self.kwargsrepr, - 'type': self.type, - 'body': self.body, - 'hostname': self.hostname, - 'time_start': self.time_start, - 'acknowledged': self.acknowledged, - 'delivery_info': self.delivery_info, - 'worker_pid': self.worker_pid} + return { + 'id': self.id, + 'name': self.name, + 'args': self.argsrepr, + 'kwargs': self.kwargsrepr, + 'type': self.type, + 'body': self.body, + 'hostname': self.hostname, + 'time_start': self.time_start, + 'acknowledged': self.acknowledged, + 'delivery_info': self.delivery_info, + 'worker_pid': self.worker_pid, + } def __str__(self): return ' '.join([ From 85fbe12567eeb6e675c4ca0af963c199cdc2793e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 20 Jan 2016 12:35:43 -0800 Subject: [PATCH 0969/1103] time.daylight does not tell us if we are currently in DST. Closes #2983 --- celery/utils/timeutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/utils/timeutils.py b/celery/utils/timeutils.py index 570c34490..e9a52dfac 100644 --- a/celery/utils/timeutils.py +++ b/celery/utils/timeutils.py @@ -355,8 +355,8 @@ def _fields(self, **extra): }, **extra) -def utcoffset(time=_time): - if time.daylight: +def utcoffset(time=_time, localtime=_time.localtime): + if localtime().tm_isdst: return time.altzone // 3600 return time.timezone // 3600 From c5f697829ae7b96b5756313488355342211c28e2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 21 Jan 2016 12:41:28 -0800 Subject: [PATCH 0970/1103] Moves mongodb Bunch to celery.utils.objects --- celery/backends/mongodb.py | 6 ------ celery/utils/objects.py | 7 +++++++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index e217639c3..2f755a24a 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -38,12 +38,6 @@ class InvalidDocument(Exception): # noqa __all__ = ['MongoBackend'] -class Bunch(object): - - def __init__(self, **kw): - self.__dict__.update(kw) - - class MongoBackend(BaseBackend): mongo_host = None diff --git a/celery/utils/objects.py b/celery/utils/objects.py index 1555f9caf..8a2f7f639 100644 --- a/celery/utils/objects.py +++ b/celery/utils/objects.py @@ -11,6 +11,13 @@ __all__ = ['mro_lookup'] +class Bunch(object): + """Object that enables you to modify attributes.""" + + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + def mro_lookup(cls, attr, stop=(), monkey_patched=[]): """Return the first node by MRO order that defines an attribute. From ee27089030bcc3cb3e15bf373491ffe1956620a1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 21 Jan 2016 12:46:18 -0800 Subject: [PATCH 0971/1103] Gossip: Bootsteps can now hook into on_node_join/leave/lost --- celery/tests/worker/test_consumer.py | 22 ++++++++++++ celery/worker/consumer.py | 17 ++++++++++ docs/userguide/extending.rst | 51 ++++++++++++++++++++++++++-- 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/celery/tests/worker/test_consumer.py b/celery/tests/worker/test_consumer.py index d3391dc48..91468b108 100644 --- a/celery/tests/worker/test_consumer.py +++ b/celery/tests/worker/test_consumer.py @@ -337,6 +337,28 @@ def test_init(self): self.assertTrue(g.enabled) self.assertIs(c.gossip, g) + def test_callbacks(self): + c = self.Consumer() + c.app.connection = _amqp_connection() + g = Gossip(c) + on_node_join = Mock(name='on_node_join') + on_node_join2 = Mock(name='on_node_join2') + on_node_leave = Mock(name='on_node_leave') + on_node_lost = Mock(name='on.node_lost') + g.on.node_join.add(on_node_join) + g.on.node_join.add(on_node_join2) + g.on.node_leave.add(on_node_leave) + g.on.node_lost.add(on_node_lost) + + worker = Mock(name='worker') + g.on_node_join(worker) + on_node_join.assert_called_with(worker) + on_node_join2.assert_called_with(worker) + g.on_node_leave(worker) + on_node_leave.assert_called_with(worker) + g.on_node_lost(worker) + on_node_lost.assert_called_with(worker) + def test_election(self): c = self.Consumer() c.app.connection_for_read = _amqp_connection() diff --git a/celery/worker/consumer.py b/celery/worker/consumer.py index 2055f671e..eb8343906 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer.py @@ -39,6 +39,7 @@ from celery.utils import gethostname from celery.utils.functional import noop from celery.utils.log import get_logger +from celery.utils.objects import Bunch from celery.utils.text import truncate from celery.utils.timeutils import humanize_seconds, rate @@ -749,6 +750,11 @@ def __init__(self, c, without_gossip=False, self.Receiver = c.app.events.Receiver self.hostname = c.hostname self.full_hostname = '.'.join([self.hostname, str(c.pid)]) + self.on = Bunch( + node_join=set(), + node_leave=set(), + node_lost=set(), + ) self.timer = c.timer if self.enabled: @@ -836,12 +842,23 @@ def on_elect_ack(self, event): def on_node_join(self, worker): debug('%s joined the party', worker.hostname) + self._call_handlers(self.on.node_join, worker) def on_node_leave(self, worker): debug('%s left', worker.hostname) + self._call_handlers(self.on.node_leave, worker) def on_node_lost(self, worker): info('missed heartbeat from %s', worker.hostname) + self._call_handlers(self.on.node_lost, worker) + + def _call_handlers(self, handlers, *args, **kwargs): + for handler in handlers: + try: + handler(*args, **kwargs) + except Exception as exc: + error('Ignored error from handler %r: %r', + handler, exc, exc_info=1) def register_timer(self): if self._tref is not None: diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index c436915ca..188bdfba6 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -356,8 +356,55 @@ Attributes .. code-block:: python - class Step(bootsteps.StartStopStep): - requires = ('celery.worker.consumer:Events',) + class RatelimitStep(bootsteps.StartStopStep): + """Rate limit tasks based on the number of workers in the + cluster.""" + requires = ('celery.worker.consumer:Gossip',) + + def start(self, c): + self.c = c + self.c.gossip.on.node_join.add(self.on_cluster_size_change) + self.c.gossip.on.node_leave.add(self.on_cluster_size_change) + self.c.gossip.on.node_lost.add(self.on_node_lost) + self.tasks = [ + self.app.tasks['proj.tasks.add'] + self.app.tasks['proj.tasks.mul'] + ] + self.last_size = None + + def on_cluster_size_change(self, worker): + cluster_size = len(self.c.gossip.state.alive_workers()) + if cluster_size != self.last_size: + for task in self.tasks: + task.rate_limit = 1.0 / cluster_size + self.c.reset_rate_limits() + self.last_size = cluster_size + + def on_node_lost(self, worker): + # may have processed heartbeat too late, so wake up in a while + # to see if the worker recovered + self.c.timer.call_after(10.0, self.on_cluster_size_change) + + **Callbacks** + + - ``gossip.on.node_join(worker)`` + + Called whenever a new node joins the cluster, providing a + :class:`~celery.events.state.Worker` instance. + + - ``gossip.on.node_leave(worker)`` + + Called whenever a new node leaves the cluster (shuts down), + providing a :class:`~celery.events.state.Worker` instance. + + - ``gossip.on.node_lost(worker)`` + + Called whenever heartbeat was missed for a worker instance in the + cluster (heartbeat not received or processed in time), + providing a :class:`~celery.events.state.Worker` instance. + + This does not necessarily mean the worker is actually offline, so use a time + out mechanism if the default heartbeat timeout is not sufficient. .. attribute:: pool From 54ded3497819aa1999fabc275b37ab0ca8aec907 Mon Sep 17 00:00:00 2001 From: Mike Attwood Date: Thu, 21 Jan 2016 16:28:46 -0700 Subject: [PATCH 0972/1103] Let celery make tasks from functions with type hints --- celery/tests/utils/test_functional.py | 17 +++++++++++++++++ celery/utils/functional.py | 12 ++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/celery/tests/utils/test_functional.py b/celery/tests/utils/test_functional.py index e2ef575c3..c358351aa 100644 --- a/celery/tests/utils/test_functional.py +++ b/celery/tests/utils/test_functional.py @@ -291,3 +291,20 @@ def f(x, y, kwarg=1): g(1) g(1, 2) g(1, 2, kwarg=3) + + def test_from_fun_with_hints(self): + local = {} + fun = ('def f_hints(x: int, y: int, kwarg: int=1):' + ' pass') + try: + exec(fun, {}, local) + except SyntaxError: + # py2 + return + f_hints = local['f_hints'] + + g = head_from_fun(f_hints) + with self.assertRaises(TypeError): + g(1) + g(1, 2) + g(1, 2, kwarg=3) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 80d0ac9de..c691d45a3 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -13,7 +13,10 @@ from collections import OrderedDict from functools import partial, wraps -from inspect import getargspec, isfunction +try: + from inspect import isfunction, getfullargspec as getargspec +except ImportError: # Py2 + from inspect import isfunction, getargspec # noqa from itertools import chain, islice from amqp import promise @@ -28,6 +31,7 @@ 'regen', 'dictfilter', 'lazy', 'maybe_evaluate', 'head_from_fun'] IS_PY3 = sys.version_info[0] == 3 +IS_PY2 = sys.version_info[0] == 2 KEYWORD_MARK = object() @@ -365,11 +369,15 @@ def _argsfromspec(spec, replace_defaults=True): optional = list(zip(spec.args[-split:], defaults)) else: positional, optional = spec.args, [] + if IS_PY3: # pragma: no cover + keywords = spec.varkw + elif IS_PY2: + keywords = spec.keywords # noqa return ', '.join(filter(None, [ ', '.join(positional), ', '.join('{0}={1}'.format(k, v) for k, v in optional), '*{0}'.format(spec.varargs) if spec.varargs else None, - '**{0}'.format(spec.keywords) if spec.keywords else None, + '**{0}'.format(keywords) if keywords else None, ])) From 78b053c720ba942c9a37a1db81a3f6de0ffcb275 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 22 Jan 2016 12:01:21 -0800 Subject: [PATCH 0973/1103] Error mail: Sets charset to utf-8 by default (Issue #2737) --- celery/app/defaults.py | 2 +- celery/utils/mail.py | 2 +- docs/configuration.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 9f8e44cd4..3690ae751 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -139,7 +139,7 @@ def __repr__(self): backend_settings=Option(None, type='dict'), ), email=Namespace( - charset=Option('us-ascii'), + charset=Option('utf-8'), host=Option('localhost'), host_user=Option(), host_password=Option(), diff --git a/celery/utils/mail.py b/celery/utils/mail.py index 00c5f29a9..585a7abcb 100644 --- a/celery/utils/mail.py +++ b/celery/utils/mail.py @@ -42,7 +42,7 @@ class SendmailWarning(UserWarning): class Message(object): def __init__(self, to=None, sender=None, subject=None, - body=None, charset='us-ascii'): + body=None, charset='utf-8'): self.to = maybe_list(to) self.sender = sender self.subject = subject diff --git a/docs/configuration.rst b/docs/configuration.rst index 76401ef35..3c144a5fc 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1930,7 +1930,7 @@ email_charset ~~~~~~~~~~~~~ .. versionadded:: 4.0 -Charset for outgoing emails. Default is "us-ascii". +Charset for outgoing emails. Default is "utf-8". .. _conf-example-error-mail-config: From b00e3b802d4b9949597fcda5fce83144651924db Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 22 Jan 2016 15:30:12 -0800 Subject: [PATCH 0974/1103] Fixes build --- celery/tests/backends/test_mongodb.py | 7 +------ celery/tests/utils/test_objects.py | 14 ++++++++++++++ celery/tests/worker/test_consumer.py | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 celery/tests/utils/test_objects.py diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index f84ee424b..d2fa023bd 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -10,7 +10,7 @@ from celery import states from celery.backends import mongodb as module from celery.backends.mongodb import ( - InvalidDocument, MongoBackend, Bunch, pymongo, + InvalidDocument, MongoBackend, pymongo, ) from celery.exceptions import ImproperlyConfigured from celery.tests.case import ( @@ -49,11 +49,6 @@ def teardown(self): module.Binary = self._reset['Binary'] datetime.datetime = self._reset['datetime'] - def test_Bunch(self): - x = Bunch(foo='foo', bar=2) - self.assertEqual(x.foo, 'foo') - self.assertEqual(x.bar, 2) - def test_init_no_mongodb(self): prev, module.pymongo = module.pymongo, None try: diff --git a/celery/tests/utils/test_objects.py b/celery/tests/utils/test_objects.py new file mode 100644 index 000000000..88754c1b8 --- /dev/null +++ b/celery/tests/utils/test_objects.py @@ -0,0 +1,14 @@ +from __future__ import absolute_import, unicode_literals + +from celery.utils.objects import Bunch + +from celery.tests.case import Case + + +class test_Bunch(Case): + + def test(self): + x = Bunch(foo='foo', bar=2) + self.assertEqual(x.foo, 'foo') + self.assertEqual(x.bar, 2) + diff --git a/celery/tests/worker/test_consumer.py b/celery/tests/worker/test_consumer.py index 91468b108..bda6599e1 100644 --- a/celery/tests/worker/test_consumer.py +++ b/celery/tests/worker/test_consumer.py @@ -339,7 +339,7 @@ def test_init(self): def test_callbacks(self): c = self.Consumer() - c.app.connection = _amqp_connection() + c.app.connection_for_read = _amqp_connection() g = Gossip(c) on_node_join = Mock(name='on_node_join') on_node_join2 = Mock(name='on_node_join2') From a94e2cbe9cb98e7dc2a69e912d8e7d2fc25c33d3 Mon Sep 17 00:00:00 2001 From: Adaptification Date: Fri, 22 Jan 2016 18:45:52 -0700 Subject: [PATCH 0975/1103] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 17040ebeb..3fc0c6043 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -202,3 +202,4 @@ Dennis Brakhane, 2015/10/30 Chris Harris, 2015/11/27 Valentyn Klindukh, 2016/01/15 Wayne Chang, 2016/01/15 +Mike Attwood, 2016/01/22 From 1abcfa713abda5a5a6e1f52003b51a85ddcdbf25 Mon Sep 17 00:00:00 2001 From: Aleksandr Kuznetsov Date: Wed, 6 Jan 2016 10:33:50 +0300 Subject: [PATCH 0976/1103] Added cleaning of corrupted scheduler files for some storage backend errors --- celery/beat.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/celery/beat.py b/celery/beat.py index c4ceca01d..6fc500ed9 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -419,6 +419,12 @@ def _destroy_open_corrupted_schedule(self, exc): def setup_schedule(self): try: self._store = self._open_schedule() + # In some cases there may be different errors from a storage + # backend for corrupted files. Example - DBPageNotFoundError + # exception from bsddb. In such case the file will be + # successfully opened but the error will be raised on first key + # retrieving. + self._store.keys() except Exception as exc: self._store = self._destroy_open_corrupted_schedule(exc) From b0cfef714a3f692eb12c0a2d63fc51c109dbe384 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 22 Jan 2016 18:00:48 -0800 Subject: [PATCH 0977/1103] Documentation: Adds links to extending attributes --- docs/userguide/extending.rst | 57 +++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index 188bdfba6..792c6b49b 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -106,6 +106,7 @@ and the worker currently defines two blueprints: **Worker**, and **Consumer** ---------------------------------------------------------- +.. _extending-worker_blueprint: Worker ====== @@ -118,21 +119,31 @@ to the Consumer blueprint. The :class:`~celery.worker.WorkController` is the core worker implementation, and contains several methods and attributes that you can use in your bootstep. +.. _extending-worker_blueprint-attributes: + Attributes ---------- +.. _extending-worker-app: + .. attribute:: app The current app instance. +.. _extending-worker-hostname: + .. attribute:: hostname The workers node name (e.g. `worker1@example.com`) +.. _extending-worker-blueprint: + .. attribute:: blueprint This is the worker :class:`~celery.bootsteps.Blueprint`. +.. _extending-worker-hub: + .. attribute:: hub Event loop object (:class:`~kombu.async.Hub`). You can use @@ -148,6 +159,8 @@ Attributes class WorkerStep(bootsteps.StartStopStep): requires = ('celery.worker.components:Hub',) +.. _extending-worker-pool: + .. attribute:: pool The current process/eventlet/gevent/thread pool. @@ -160,6 +173,8 @@ Attributes class WorkerStep(bootsteps.StartStopStep): requires = ('celery.worker.components:Pool',) +.. _extending-worker-timer: + .. attribute:: timer :class:`~kombu.async.timer.Timer` used to schedule functions. @@ -171,6 +186,8 @@ Attributes class WorkerStep(bootsteps.StartStopStep): requires = ('celery.worker.components:Timer',) +.. _extending-worker-statedb: + .. attribute:: statedb :class:`Database `` to persist state between @@ -185,6 +202,8 @@ Attributes class WorkerStep(bootsteps.StartStopStep): requires = ('celery.worker.components:Statedb',) +.. _extending-worker-autoscaler: + .. attribute:: autoscaler :class:`~celery.worker.autoscaler.Autoscaler` used to automatically grow @@ -199,6 +218,8 @@ Attributes class WorkerStep(bootsteps.StartStopStep): requires = ('celery.worker.autoscaler:Autoscaler',) +.. _extending-worker-autoreloader: + .. attribute:: autoreloader :class:`~celery.worker.autoreloder.Autoreloader` used to automatically @@ -212,6 +233,9 @@ Attributes class WorkerStep(bootsteps.StartStopStep): requires = ('celery.worker.autoreloader:Autoreloader',) +Example worker bootstep +----------------------- + An example Worker bootstep could be: .. code-block:: python @@ -243,7 +267,6 @@ An example Worker bootstep could be: Every method is passed the current ``WorkController`` instance as the first argument. - Another example could use the timer to wake up at regular intervals: .. code-block:: python @@ -276,6 +299,8 @@ Another example could use the timer to wake up at regular intervals: if req.time_start and time() - req.time_start > self.timeout: raise SystemExit() +.. _extending-consumer_blueprint: + Consumer ======== @@ -289,25 +314,37 @@ be possible to restart your blueprint. An additional 'shutdown' method is defined for consumer bootsteps, this method is called when the worker is shutdown. +.. _extending-consumer-attributes: + Attributes ---------- +.. _extending-consumer-app: + .. attribute:: app The current app instance. +.. _extending-consumer-controller: + .. attribute:: controller The parent :class:`~@WorkController` object that created this consumer. +.. _extending-consumer-hostname: + .. attribute:: hostname The workers node name (e.g. `worker1@example.com`) +.. _extending-consumer-blueprint: + .. attribute:: blueprint This is the worker :class:`~celery.bootsteps.Blueprint`. +.. _extending-consumer-hub: + .. attribute:: hub Event loop object (:class:`~kombu.async.Hub`). You can use @@ -323,6 +360,7 @@ Attributes class WorkerStep(bootsteps.StartStopStep): requires = ('celery.worker:Hub',) +.. _extending-consumer-connection: .. attribute:: connection @@ -336,6 +374,8 @@ Attributes class Step(bootsteps.StartStopStep): requires = ('celery.worker.consumer:Connection',) +.. _extending-consumer-event_dispatcher: + .. attribute:: event_dispatcher A :class:`@events.Dispatcher` object that can be used to send events. @@ -347,6 +387,8 @@ Attributes class Step(bootsteps.StartStopStep): requires = ('celery.worker.consumer:Events',) +.. _extending-consumer-gossip: + .. attribute:: gossip Worker to worker broadcast communication @@ -406,15 +448,21 @@ Attributes This does not necessarily mean the worker is actually offline, so use a time out mechanism if the default heartbeat timeout is not sufficient. +.. _extending-consumer-pool: + .. attribute:: pool The current process/eventlet/gevent/thread pool. See :class:`celery.concurrency.base.BasePool`. +.. _extending-consumer-timer: + .. attribute:: timer :class:`Timer Date: Fri, 22 Jan 2016 18:51:17 -0800 Subject: [PATCH 0978/1103] [dev] Merge changelog from 3.1 --- docs/history/changelog-3.1.rst | 318 ++++++++++++++++++++++++++++++--- 1 file changed, 293 insertions(+), 25 deletions(-) diff --git a/docs/history/changelog-3.1.rst b/docs/history/changelog-3.1.rst index 1240e3a99..a5f38b92c 100644 --- a/docs/history/changelog-3.1.rst +++ b/docs/history/changelog-3.1.rst @@ -1,43 +1,299 @@ .. _changelog-3.1: -=============================== - Change history for Celery 3.1 -=============================== +================ + Change history +================ This document contains change notes for bugfix releases in the 3.1.x series (Cipater), please see :ref:`whatsnew-3.1` for an overview of what's new in Celery 3.1. -If you're looking for versions prior to 3.1.x you should go to :ref:`history`. +.. _version-3.1.20: + +3.1.20 +====== +:release-date: 2016-01-22 06:50 P.M UTC +:release-by: Ask Solem + +- **Requirements** + + - Now depends on :ref:`Kombu 3.0.33 `. + + - Now depends on :mod:`billiard` 3.3.0.22. + + Includes binary wheels for Microsoft Windows x86 and x86_64! + +- **Task**: Error emails now uses ``utf-8`` charset by default (Issue #2737). + +- **Task**: Retry now forwards original message headers (Issue #3017). + +- **Worker**: Bootsteps can now hook into ``on_node_join``/``leave``/``lost``. + + See :ref:`extending-consumer-gossip` for an example. + +- **Events**: Fixed handling of DST timezones (Issue #2983). + +- **Results**: Redis backend stopped respecting certain settings. + + Contributed by Jeremy Llewellyn. + +- **Results**: Database backend now properly supports JSON exceptions + (Issue #2441). + +- **Results**: Redis ``new_join`` did not properly call task errbacks on chord + error (Issue #2796). + +- **Results**: Restores Redis compatibility with redis-py < 2.10.0 + (Issue #2903). + +- **Results**: Fixed rare issue with chord error handling (Issue #2409). + +- **Tasks**: Using queue-name values in :setting:`CELERY_ROUTES` now works + again (Issue #2987). + +- **General**: Result backend password now sanitized in report output + (Issue #2812, Issue #2004). + +- **Configuration**: Now gives helpful error message when the result backend + configuration points to a module, and not a class (Issue #2945). + +- **Results**: Exceptions sent by JSON serialized workers are now properly + handled by pickle configured workers. + +- **Programs**: ``celery control autoscale`` now works (Issue #2950). + +- **Programs**: ``celery beat --detached`` now runs after fork callbacks. + +- **General**: Fix for LRU cache implementation on Python 3.5 (Issue #2897). + + Contributed by Dennis Brakhane. + + Python 3.5's ``OrderedDict`` does not allow mutation while it is being + iterated over. This breaks "update" if it is called with a dict + larger than the maximum size. + + This commit changes the code to a version that does not iterate over + the dict, and should also be a little bit faster. + +- **Init scripts**: The beat init script now properly reports service as down + when no pid file can be found. + + Eric Zarowny + +- **Beat**: Added cleaning of corrupted scheduler files for some storage + backend errors (Issue #2985). + + Fix contributed by Aleksandr Kuznetsov. + +- **Beat**: Now syncs the schedule even if the schedule is empty. + + Fix contributed by Colin McIntosh. + +- **Supervisord**: Set higher process priority in supervisord example. + + Contributed by George Tantiras. + +- **Documentation**: Includes improvements by: + + Bryson + Caleb Mingle + Christopher Martin + Dieter Adriaenssens + Jason Veatch + Jeremy Cline + Juan Rossi + Kevin Harvey + Kevin McCarthy + Kirill Pavlov + Marco Buttu + Mayflower + Mher Movsisyan + Michael Floering + michael-k + Nathaniel Varona + Rudy Attias + Ryan Luckie + Steven Parker + squfrans + Tadej Janež + TakesxiSximada + Tom S + +.. _version-3.1.19: + +3.1.19 +====== +:release-date: 2015-10-26 01:00 P.M UTC +:release-by: Ask Solem + +- **Requirements** + + - Now depends on :ref:`Kombu 3.0.29 `. + + - Now depends on :mod:`billiard` 3.3.0.21. + +- **Results**: Fixed MongoDB result backend URL parsing problem + (Issue celery/kombu#375). + +- **Worker**: Task request now properly sets ``priority`` in delivery_info. + + Fix contributed by Gerald Manipon. + +- **Beat**: PyPy shelve may raise ``KeyError`` when setting keys + (Issue #2862). + +- **Programs**: :program:`celery beat --deatched` now working on PyPy. + + Fix contributed by Krzysztof Bujniewicz. + +- **Results**: Redis result backend now ensures all pipelines are cleaned up. + + Contributed by Justin Patrin. + +- **Results**: Redis result backend now allows for timeout to be set in the + query portion of the result backend URL. + + E.g. ``CELERY_RESULT_BACKEND = 'redis://?timeout=10'`` + + Contributed by Justin Patrin. + +- **Results**: ``result.get`` now properly handles failures where the + exception value is set to :const:`None` (Issue #2560). + +- **Prefork pool**: Fixed attribute error ``proc.dead``. + +- **Worker**: Fixed worker hanging when gossip/heartbeat disabled + (Issue #1847). + + Fix contributed by Aaron Webber and Bryan Helmig. + +- **Results**: MongoDB result backend now supports pymongo 3.x + (Issue #2744). + + Fix contributed by Sukrit Khera. + +- **Results**: RPC/amqp backends did not deserialize exceptions properly + (Issue #2691). + + Fix contributed by Sukrit Khera. + +- **Programs**: Fixed problem with :program:`celery amqp`'s + ``basic_publish`` (Issue #2013). + +- **Worker**: Embedded beat now properly sets app for thread/process + (Issue #2594). + +- **Documentation**: Many improvements and typos fixed. + + Contributions by: + + Carlos Garcia-Dubus + D. Yu + jerry + Jocelyn Delalande + Josh Kupershmidt + Juan Rossi + kanemra + Paul Pearce + Pavel Savchenko + Sean Wang + Seungha Kim + Zhaorong Ma + +.. _version-3.1.18: + +3.1.18 +====== +:release-date: 2015-04-22 05:30 P.M UTC +:release-by: Ask Solem + +- **Requirements** + + - Now depends on :ref:`Kombu 3.0.25 `. + + - Now depends on :mod:`billiard` 3.3.0.20. + +- **Django**: Now supports Django 1.8 (Issue #2536). + + Fix contributed by Bence Tamas and Mickaël Penhard. + +- **Results**: MongoDB result backend now compatible with pymongo 3.0. + + Fix contributed by Fatih Sucu. + +- **Tasks**: Fixed bug only happening when a task has multiple callbacks + (Issue #2515). + + Fix contributed by NotSqrt. + +- **Commands**: Preload options now support ``--arg value`` syntax. + + Fix contributed by John Anderson. + +- **Compat**: A typo caused ``celery.log.setup_logging_subsystem`` to be + undefined. + + Fix contributed by Gunnlaugur Thor Briem. + +- **init scripts**: The celerybeat generic init script now uses + ``/bin/sh`` instead of bash (Issue #2496). + + Fix contributed by Jelle Verstraaten. + +- **Django**: Fixed a :exc:`TypeError` sometimes occurring in logging + when validating models. + + Fix contributed by Alexander. + +- **Commands**: Worker now supports new ``--executable`` argument that can + be used with ``--detach``. + + Contributed by Bert Vanderbauwhede. + +- **Canvas**: Fixed crash in chord unlock fallback task (Issue #2404). + +- **Worker**: Fixed rare crash occurring with ``--autoscale`` enabled + (Issue #2411). + +- **Django**: Properly recycle worker Django database connections when the + Django ``CONN_MAX_AGE`` setting is enabled (Issue #2453). + + Fix contributed by Luke Burden. .. _version-3.1.17: 3.1.17 ====== +:release-date: 2014-11-19 03:30 P.M UTC +:release-by: Ask Solem -.. admonition:: CELERYD_FORCE_EXECV should not be used. +.. admonition:: Do not enable the :setting:`CELERYD_FORCE_EXECV` setting! - Please disable this option if you're using the RabbitMQ or Redis - transports. + Please review your configuration and disable this option if you're using the + RabbitMQ or Redis transport. - Keeping this option enabled in 3.1 means the async based worker will - be disabled, so using is more likely to lead to trouble than doing - anything good. + Keeping this option enabled after 3.1 means the async based prefork pool will + be disabled, which can easily cause instability. - **Requirements** - Now depends on :ref:`Kombu 3.0.24 `. + Includes the new Qpid transport coming in Celery 3.2, backported to + support those who may still require Python 2.6 compatibility. + - Now depends on :mod:`billiard` 3.3.0.19. -- **Task**: The timing for ETA/countdown tasks were off after the example ``LocalTimezone`` + - ``celery[librabbitmq]`` now depends on librabbitmq 1.6.1. + +- **Task**: The timing of ETA/countdown tasks were off after the example ``LocalTimezone`` implementation in the Python documentation no longer works in Python 3.4. (Issue #2306). - **Task**: Raising :exc:`~celery.exceptions.Ignore` no longer sends ``task-failed`` event (Issue #2365). -- **Redis result backend**: Fixed errors about unbound local ``self``. +- **Redis result backend**: Fixed unbound local errors. Fix contributed by Thomas French. @@ -47,6 +303,13 @@ If you're looking for versions prior to 3.1.x you should go to :ref:`history`. - **Canvas**: chain and group now handles json serialized signatures (Issue #2076). +- **Results**: ``.join_native()`` would accidentally treat the ``STARTED`` + state as being ready (Issue #2326). + + This could lead to the chord callback being called with invalid arguments + when using chords with the :setting:`CELERY_TRACK_STARTED` setting + enabled. + - **Canvas**: The ``chord_size`` attribute is now set for all canvas primitives, making sure more combinations will work with the ``new_join`` optimization for Redis (Issue #2339). @@ -67,11 +330,16 @@ If you're looking for versions prior to 3.1.x you should go to :ref:`history`. Fix contributed by Gino Ledesma. +- **Mongodb Result backend**: Pickling the backend instance will now include + the original url (Issue #2347). + + Fix contributed by Sukrit Khera. + - **Task**: Exception info was not properly set for tasks raising :exc:`~celery.exceptions.Reject` (Issue #2043). -- **Worker**: The set of revokes tasks are now deduplicated when loading from - the worker state database (Issue #2336). +- **Worker**: Duplicates are now removed when loading the set of revoked tasks + from the worker state database (Issue #2336). - **celery.contrib.rdb**: Fixed problems with ``rdb.set_trace`` calling stop from the wrong frame. @@ -132,7 +400,7 @@ If you're looking for versions prior to 3.1.x you should go to :ref:`history`. :release-by: Ask Solem - **Django**: Now makes sure ``django.setup()`` is called - before importing any task modules (Django 1.7 compatibility, Issue #2227) + before importing any task modules (Django 1.7 compatibility, Issue #2227) - **Results**: ``result.get()`` was misbehaving by calling ``backend.get_task_meta`` in a finally call leading to @@ -371,7 +639,7 @@ News and if you use the ``librabbitmq`` module you also have to upgrade to librabbitmq 1.5.0: - .. code-block:: console + .. code-block:: bash $ pip install -U librabbitmq @@ -422,7 +690,7 @@ News exceptions. - **Worker**: No longer sends task error emails for expected errors (in - ``@task(throws=(...,)))``. + ``@task(throws=(..., )))``. - **Canvas**: Fixed problem with exception deserialization when using the JSON serializer (Issue #1987). @@ -467,7 +735,7 @@ News See :ref:`redis-caveats`. - This will be the default in Celery 4.0. + This will be the default in Celery 3.2. - **Results**: The :class:`@AsyncResult` object now keeps a local cache of the final state of the task. @@ -476,7 +744,7 @@ News and you can do so by setting :setting:`CELERY_MAX_CACHED_RESULTS` to :const:`-1`. The lifetime of the cache will then be bound to the lifetime of the result object, which will be the default behavior - in Celery 4.0. + in Celery 3.2. - **Events**: The "Substantial drift" warning message is now logged once per node name only (Issue #1802). @@ -507,9 +775,9 @@ News This means that referring to a number will work when specifying a list of node names and not just for a number range: - .. code-block:: console + .. code-block:: bash - $ celery multi start A B C D -c:1 4 -c:2-4 8 + celery multi start A B C D -c:1 4 -c:2-4 8 In this example ``1`` refers to node A (as it's the first node in the list). @@ -682,7 +950,7 @@ News - **Results:** ``ResultSet.iterate`` is now pending deprecation. - The method will be removed in version 4.0 and removed in version 5.0. + The method will be deprecated in version 3.2 and removed in version 3.3. Use ``result.get(callback=)`` (or ``result.iter_native()`` where available) instead. @@ -735,7 +1003,7 @@ News Example using command-line configuration to set a broker heartbeat from :program:`celery multi`: - .. code-block:: console + .. code-block:: bash $ celery multi start 1 -c3 -- broker.heartbeat=30 @@ -832,7 +1100,7 @@ Synchronous subtasks Tasks waiting for the result of a subtask will now emit a :exc:`RuntimeWarning` warning when using the prefork pool, -and in 4.0 this will result in an exception being raised. +and in 3.2 this will result in an exception being raised. It's not legal for tasks to block by waiting for subtasks as this is likely to lead to resource starvation and eventually @@ -915,7 +1183,7 @@ Fixes Example: - .. code-block:: console + .. code-block:: bash $ celery -A proj worker -n foo@%h --logfile=%n.log --statedb=%n.db From 7164ffbdc098ca210d9a9c0d4080d5ca2ae95f5f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 22 Jan 2016 18:59:21 -0800 Subject: [PATCH 0979/1103] Documentation markup typo --- docs/userguide/extending.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index 792c6b49b..d9889cec8 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -392,7 +392,7 @@ Attributes .. attribute:: gossip Worker to worker broadcast communication - (class:`~celery.worker.consumer.Gossip`). + (:class:`~celery.worker.consumer.Gossip`). A consumer bootstep must require the `Gossip` bootstep to use this. @@ -423,8 +423,8 @@ Attributes self.last_size = cluster_size def on_node_lost(self, worker): - # may have processed heartbeat too late, so wake up in a while - # to see if the worker recovered + # may have processed heartbeat too late, so wake up soon + # in order to see if the worker recovered. self.c.timer.call_after(10.0, self.on_cluster_size_change) **Callbacks** From 39d5db761e4805d6b3ecbe86c2833a02d08b94ea Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 22 Jan 2016 19:00:35 -0800 Subject: [PATCH 0980/1103] Formatting --- docs/userguide/extending.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index d9889cec8..a9c85c981 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -429,17 +429,17 @@ Attributes **Callbacks** - - ``gossip.on.node_join(worker)`` + - `` gossip.on.node_join`` Called whenever a new node joins the cluster, providing a :class:`~celery.events.state.Worker` instance. - - ``gossip.on.node_leave(worker)`` + - `` gossip.on.node_leave`` Called whenever a new node leaves the cluster (shuts down), providing a :class:`~celery.events.state.Worker` instance. - - ``gossip.on.node_lost(worker)`` + - `` gossip.on.node_lost(worker)`` Called whenever heartbeat was missed for a worker instance in the cluster (heartbeat not received or processed in time), From 57c5812cc088f5f611343f3ad0b5ef0d3f9fb254 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 22 Jan 2016 19:00:47 -0800 Subject: [PATCH 0981/1103] Typo --- docs/userguide/extending.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index a9c85c981..51edeebec 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -439,7 +439,7 @@ Attributes Called whenever a new node leaves the cluster (shuts down), providing a :class:`~celery.events.state.Worker` instance. - - `` gossip.on.node_lost(worker)`` + - `` gossip.on.node_lost`` Called whenever heartbeat was missed for a worker instance in the cluster (heartbeat not received or processed in time), From 01c97bf89d875075980ab423b9163bc2caf89d02 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 22 Jan 2016 19:10:23 -0800 Subject: [PATCH 0982/1103] More typos --- docs/userguide/extending.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index 51edeebec..0713a93c1 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -486,7 +486,7 @@ Attributes .. code-block:: python class Step(bootsteps.StartStopStep): - requires = ('celery.worker.consumer:Heart',) + requires = ('celery.worker.consumer:Tasks',) .. _extending-consumer-strategies: @@ -510,7 +510,7 @@ Attributes .. code-block:: python class Step(bootsteps.StartStopStep): - requires = ('celery.worker.consumer:Heart',) + requires = ('celery.worker.consumer:Tasks',) .. _extending-consumer-task_buckets: From 807c13873e9a3d82f53504e7a473e0d983a9e230 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 29 Jan 2016 13:28:58 -0800 Subject: [PATCH 0983/1103] [utils] simple_format now tries to give better error for unmatched keys. Closes #3016 --- celery/utils/__init__.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/celery/utils/__init__.py b/celery/utils/__init__.py index fdbb21ec0..d6053bc65 100644 --- a/celery/utils/__init__.py +++ b/celery/utils/__init__.py @@ -46,6 +46,12 @@ version {removal}. {alternative} """ +UNKNOWN_SIMPLE_FORMAT_KEY = """ +Unknown format %{0} in string {1!r}. +Possible causes: Did you forget to escape the expand sign (use '%%{0!r}'), +or did you escape and the value was expanded twice? (%%N -> %N -> %hostname)? +""".strip() + #: Billiard sets this when execv is enabled. #: We use it to find out the name of the original ``__main__`` #: module, so that we can properly rewrite the name of the @@ -375,7 +381,11 @@ def simple_format(s, keys, pattern=RE_FORMAT, expand=r'\1'): keys.setdefault('%', '%') def resolve(match): - resolver = keys[match.expand(expand)] + key = match.expand(expand) + try: + resolver = keys[key] + except KeyError: + raise ValueError(UNKNOWN_SIMPLE_FORMAT_KEY.format(key, s)) if isinstance(resolver, Callable): return resolver() return resolver From afbd2330ed6f835d0c3774cff15c1c6312a1930d Mon Sep 17 00:00:00 2001 From: David Harrigan Date: Mon, 1 Feb 2016 16:43:04 -0500 Subject: [PATCH 0984/1103] Add auth options in cassandra backend --- celery/backends/cassandra.py | 13 +++++++++++-- celery/tests/backends/test_cassandra.py | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index d406be1df..55bf1e7f3 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -11,6 +11,7 @@ import sys try: # pragma: no cover import cassandra + import cassandra.auth import cassandra.cluster except ImportError: # pragma: no cover cassandra = None # noqa @@ -121,6 +122,13 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, cassandra.ConsistencyLevel.LOCAL_QUORUM, ) + self.auth_provider = None + auth_provider = conf.get('cassandra_auth_provider', None) + auth_kwargs = conf.get('cassandra_auth_kwargs', None) + if auth_provider and auth_kwargs: + auth_provider_class = getattr(cassandra.auth, auth_provider) + self.auth_provider = auth_provider_class(**auth_kwargs) + self._connection = None self._session = None self._write_stmt = None @@ -142,8 +150,9 @@ def _get_connection(self, write=False): """ if self._connection is None: try: - self._connection = cassandra.cluster.Cluster(self.servers, - port=self.port) + self._connection = cassandra.cluster.Cluster( + self.servers, port=self.port, + auth_provider=self.auth_provider) self._session = self._connection.connect(self.keyspace) # We are forced to do concatenation below, as formatting would diff --git a/celery/tests/backends/test_cassandra.py b/celery/tests/backends/test_cassandra.py index 1875b2005..9a798badf 100644 --- a/celery/tests/backends/test_cassandra.py +++ b/celery/tests/backends/test_cassandra.py @@ -9,7 +9,7 @@ AppCase, Mock, mock_module, depends_on_current_app ) -CASSANDRA_MODULES = ['cassandra', 'cassandra.cluster'] +CASSANDRA_MODULES = ['cassandra', 'cassandra.auth', 'cassandra.cluster'] class Object(object): From e8855e0e92b6b94d779651a6fd72a1aa957de24e Mon Sep 17 00:00:00 2001 From: David Harrigan Date: Mon, 1 Feb 2016 17:45:49 -0500 Subject: [PATCH 0985/1103] add cassandra auth option documentation --- docs/configuration.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/configuration.rst b/docs/configuration.rst index 3c144a5fc..2c4be8004 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -970,6 +970,26 @@ cassandra_entry_ttl Time-to-live for status entries. They will expire and be removed after that many seconds after adding. Default (None) means they will never expire. +.. setting:: cassandra_auth_provider + +cassandra_auth_provider +~~~~~~~~~~~~~~~~~~~~~~~ + +AuthProvider class within ``cassandra.auth`` module to use. Values can be +``PlainTextAuthProvider`` or ``SaslAuthProvider``. + +.. setting:: cassandra_auth_kwargs + +cassandra_auth_kwargs +~~~~~~~~~~~~~~~~~~~~~ + +Named arguments to pass into the auth provider. e.g.:: + + cassandra_auth_kwargs = { + username: 'cassandra', + password: 'cassandra' + } + Example configuration ~~~~~~~~~~~~~~~~~~~~~ From 5acf1028c811ec00259ebfe8fdd1ce351db0573b Mon Sep 17 00:00:00 2001 From: David Harrigan Date: Mon, 1 Feb 2016 17:46:07 -0500 Subject: [PATCH 0986/1103] update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 3fc0c6043..ab6ecc0d6 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -203,3 +203,4 @@ Chris Harris, 2015/11/27 Valentyn Klindukh, 2016/01/15 Wayne Chang, 2016/01/15 Mike Attwood, 2016/01/22 +David Harrigan, 2016/2/1 From c31b5cf54dd076930cca7f5e3dbc176f56af7e70 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 1 Feb 2016 17:01:38 -0800 Subject: [PATCH 0987/1103] [Worker] Moves each consumer bootstep into separated module. --- celery/tests/worker/test_consumer.py | 70 ++-- celery/tests/worker/test_worker.py | 10 +- celery/worker/consumer/__init__.py | 17 + celery/worker/consumer/agent.py | 20 ++ celery/worker/consumer/connection.py | 33 ++ celery/worker/{ => consumer}/consumer.py | 412 +---------------------- celery/worker/consumer/control.py | 27 ++ celery/worker/consumer/events.py | 56 +++ celery/worker/consumer/gossip.py | 195 +++++++++++ celery/worker/consumer/heart.py | 30 ++ celery/worker/consumer/mingle.py | 53 +++ celery/worker/consumer/tasks.py | 59 ++++ 12 files changed, 544 insertions(+), 438 deletions(-) create mode 100644 celery/worker/consumer/__init__.py create mode 100644 celery/worker/consumer/agent.py create mode 100644 celery/worker/consumer/connection.py rename celery/worker/{ => consumer}/consumer.py (58%) create mode 100644 celery/worker/consumer/control.py create mode 100644 celery/worker/consumer/events.py create mode 100644 celery/worker/consumer/gossip.py create mode 100644 celery/worker/consumer/heart.py create mode 100644 celery/worker/consumer/mingle.py create mode 100644 celery/worker/consumer/tasks.py diff --git a/celery/tests/worker/test_consumer.py b/celery/tests/worker/test_consumer.py index bda6599e1..67870fbea 100644 --- a/celery/tests/worker/test_consumer.py +++ b/celery/tests/worker/test_consumer.py @@ -7,16 +7,12 @@ from celery.datastructures import LimitedSet from celery.worker import state as worker_state -from celery.worker.consumer import ( - Consumer, - Heart, - Tasks, - Agent, - Mingle, - Gossip, - dump_body, - CLOSE, -) +from celery.worker.consumer.agent import Agent +from celery.worker.consumer.consumer import CLOSE, Consumer, dump_body +from celery.worker.consumer.gossip import Gossip +from celery.worker.consumer.heart import Heart +from celery.worker.consumer.mingle import Mingle +from celery.worker.consumer.tasks import Tasks from celery.tests.case import AppCase, ContextMock, Mock, SkipTest, call, patch @@ -65,19 +61,19 @@ def test_sets_heartbeat(self): self.assertEqual(c.amqheartbeat, 20) def test_gevent_bug_disables_connection_timeout(self): - with patch('celery.worker.consumer._detect_environment') as de: - de.return_value = 'gevent' + with patch('celery.worker.consumer.consumer._detect_environment') as d: + d.return_value = 'gevent' self.app.conf.broker_connection_timeout = 33.33 self.get_consumer() self.assertIsNone(self.app.conf.broker_connection_timeout) def test_limit_moved_to_pool(self): - with patch('celery.worker.consumer.task_reserved') as reserved: + with patch('celery.worker.consumer.consumer.task_reserved') as reserv: c = self.get_consumer() c.on_task_request = Mock(name='on_task_request') request = Mock(name='request') c._limit_move_to_pool(request) - reserved.assert_called_with(request) + reserv.assert_called_with(request) c.on_task_request.assert_called_with(request) def test_update_prefetch_count(self): @@ -112,17 +108,17 @@ def test_on_send_event_buffered(self): def test_limit_task(self): c = self.get_consumer() - with patch('celery.worker.consumer.task_reserved') as reserved: + with patch('celery.worker.consumer.consumer.task_reserved') as reserv: bucket = Mock() request = Mock() bucket.can_consume.return_value = True c._limit_task(request, bucket, 3) bucket.can_consume.assert_called_with(3) - reserved.assert_called_with(request) + reserv.assert_called_with(request) c.on_task_request.assert_called_with(request) - with patch('celery.worker.consumer.task_reserved') as reserved: + with patch('celery.worker.consumer.consumer.task_reserved') as reserv: bucket.can_consume.return_value = False bucket.expected_time.return_value = 3.33 limit_order = c._limit_order @@ -134,7 +130,7 @@ def test_limit_task(self): priority=c._limit_order, ) bucket.expected_time.assert_called_with(4) - self.assertFalse(reserved.called) + self.assertFalse(reserv.called) def test_start_blueprint_raises_EMFILE(self): c = self.get_consumer() @@ -153,7 +149,7 @@ def se(*args, **kwargs): c._restart_state.step.side_effect = se c.blueprint.start.side_effect = socket.error() - with patch('celery.worker.consumer.sleep') as sleep: + with patch('celery.worker.consumer.consumer.sleep') as sleep: c.start() sleep.assert_called_with(1) @@ -182,12 +178,12 @@ def test_register_with_event_loop(self): c.register_with_event_loop(Mock(name='loop')) def test_on_close_clears_semaphore_timer_and_reqs(self): - with patch('celery.worker.consumer.reserved_requests') as reserved: + with patch('celery.worker.consumer.consumer.reserved_requests') as reserv: c = self.get_consumer() c.on_close() c.controller.semaphore.clear.assert_called_with() c.timer.clear.assert_called_with() - reserved.clear.assert_called_with() + reserv.clear.assert_called_with() c.pool.flush.assert_called_with() c.controller = None @@ -375,18 +371,16 @@ def test_call_task(self): c.app.connection_for_read = _amqp_connection() g = Gossip(c) g.start(c) - - with patch('celery.worker.consumer.signature') as signature: - sig = signature.return_value = Mock() - task = Mock() + signature = g.app.signature = Mock(name='app.signature') + task = Mock() + g.call_task(task) + signature.assert_called_with(task) + signature.return_value.apply_async.assert_called_with() + + signature.return_value.apply_async.side_effect = MemoryError() + with patch('celery.worker.consumer.gossip.error') as error: g.call_task(task) - signature.assert_called_with(task, app=c.app) - sig.apply_async.assert_called_with() - - sig.apply_async.side_effect = MemoryError() - with patch('celery.worker.consumer.error') as error: - g.call_task(task) - self.assertTrue(error.called) + self.assertTrue(error.called) def Event(self, id='id', clock=312, hostname='foo@example.com', pid=4312, @@ -414,7 +408,7 @@ def test_on_elect(self): g.dispatcher.send.assert_called_with('worker-elect-ack', id='id1') event.pop('clock') - with patch('celery.worker.consumer.error') as error: + with patch('celery.worker.consumer.gossip.error') as error: g.on_elect(event) self.assertTrue(error.called) @@ -444,7 +438,7 @@ def setup_election(self, g, c): g.on_elect(e3) self.assertEqual(len(g.consensus_requests['id1']), 3) - with patch('celery.worker.consumer.info'): + with patch('celery.worker.consumer.gossip.info'): g.on_elect_ack(e1) self.assertEqual(len(g.consensus_replies['id1']), 1) g.on_elect_ack(e2) @@ -474,7 +468,7 @@ def test_on_elect_ack_win_but_no_action(self): c.app.connection_for_read = _amqp_connection() g = Gossip(c) g.election_handlers = {} - with patch('celery.worker.consumer.error') as error: + with patch('celery.worker.consumer.gossip.error') as error: self.setup_election(g, c) self.assertTrue(error.called) @@ -482,7 +476,7 @@ def test_on_node_join(self): c = self.Consumer() c.app.connection_for_read = _amqp_connection() g = Gossip(c) - with patch('celery.worker.consumer.debug') as debug: + with patch('celery.worker.consumer.gossip.debug') as debug: g.on_node_join(c) debug.assert_called_with('%s joined the party', 'foo@x.com') @@ -490,7 +484,7 @@ def test_on_node_leave(self): c = self.Consumer() c.app.connection_for_read = _amqp_connection() g = Gossip(c) - with patch('celery.worker.consumer.debug') as debug: + with patch('celery.worker.consumer.gossip.debug') as debug: g.on_node_leave(c) debug.assert_called_with('%s left', 'foo@x.com') @@ -498,7 +492,7 @@ def test_on_node_lost(self): c = self.Consumer() c.app.connection_for_read = _amqp_connection() g = Gossip(c) - with patch('celery.worker.consumer.info') as info: + with patch('celery.worker.consumer.gossip.info') as info: g.on_node_lost(c) info.assert_called_with('missed heartbeat from %s', 'foo@x.com') diff --git a/celery/tests/worker/test_worker.py b/celery/tests/worker/test_worker.py index d2387af54..dcfc06336 100644 --- a/celery/tests/worker/test_worker.py +++ b/celery/tests/worker/test_worker.py @@ -214,7 +214,7 @@ def test_close_connection(self): self.assertTrue(eventer.close.call_count) self.assertTrue(heart.closed) - @patch('celery.worker.consumer.warn') + @patch('celery.worker.consumer.consumer.warn') def test_receive_message_unknown(self, warn): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN @@ -250,7 +250,7 @@ def test_receive_message_eta_OverflowError(self, to_timestamp): callback(m) self.assertTrue(m.acknowledged) - @patch('celery.worker.consumer.error') + @patch('celery.worker.consumer.consumer.error') def test_receive_message_InvalidTaskError(self, error): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN @@ -271,7 +271,7 @@ def test_receive_message_InvalidTaskError(self, error): self.assertTrue(error.called) self.assertIn('Received invalid task message', error.call_args[0][0]) - @patch('celery.worker.consumer.crit') + @patch('celery.worker.consumer.consumer.crit') def test_on_decode_error(self, crit): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) @@ -531,8 +531,8 @@ def test_receieve_message_not_registered(self): self.buffer.get_nowait() self.assertTrue(self.timer.empty()) - @patch('celery.worker.consumer.warn') - @patch('celery.worker.consumer.logger') + @patch('celery.worker.consumer.consumer.warn') + @patch('celery.worker.consumer.consumer.logger') def test_receieve_message_ack_raises(self, logger, warn): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.controller = l.app.WorkController() diff --git a/celery/worker/consumer/__init__.py b/celery/worker/consumer/__init__.py new file mode 100644 index 000000000..086ee9a47 --- /dev/null +++ b/celery/worker/consumer/__init__.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import, unicode_literals + +from .consumer import Consumer + +from .agent import Agent +from .connection import Connection +from .control import Control +from .events import Events +from .gossip import Gossip +from .heart import Heart +from .mingle import Mingle +from .tasks import Tasks + +__all__ = [ + 'Consumer', 'Agent', 'Connection', 'Control', + 'Events', 'Gossip', 'Heart', 'Mingle', 'Tasks', +] diff --git a/celery/worker/consumer/agent.py b/celery/worker/consumer/agent.py new file mode 100644 index 000000000..9c1801a13 --- /dev/null +++ b/celery/worker/consumer/agent.py @@ -0,0 +1,20 @@ +from __future__ import absolute_import, unicode_literals + +from celery import bootsteps + +from .connection import Connection + +__all__ = ['Agent'] + + +class Agent(bootsteps.StartStopStep): + + conditional = True + requires = (Connection,) + + def __init__(self, c, **kwargs): + self.agent_cls = self.enabled = c.app.conf.worker_agent + + def create(self, c): + agent = c.agent = self.instantiate(self.agent_cls, c.connection) + return agent diff --git a/celery/worker/consumer/connection.py b/celery/worker/consumer/connection.py new file mode 100644 index 000000000..e54aa248e --- /dev/null +++ b/celery/worker/consumer/connection.py @@ -0,0 +1,33 @@ +from __future__ import absolute_import, unicode_literals + +from kombu.common import ignore_errors + +from celery import bootsteps +from celery.utils.log import get_logger + +__all__ = ['Connection'] +logger = get_logger(__name__) +info = logger.info + + +class Connection(bootsteps.StartStopStep): + + def __init__(self, c, **kwargs): + c.connection = None + + def start(self, c): + c.connection = c.connect() + info('Connected to %s', c.connection.as_uri()) + + def shutdown(self, c): + # We must set self.connection to None here, so + # that the green pidbox thread exits. + connection, c.connection = c.connection, None + if connection: + ignore_errors(connection, connection.close) + + def info(self, c, params='N/A'): + if c.connection: + params = c.connection.info() + params.pop('password', None) # don't send password. + return {'broker': params} diff --git a/celery/worker/consumer.py b/celery/worker/consumer/consumer.py similarity index 58% rename from celery/worker/consumer.py rename to celery/worker/consumer/consumer.py index eb8343906..7a0147490 100644 --- a/celery/worker/consumer.py +++ b/celery/worker/consumer/consumer.py @@ -11,22 +11,17 @@ from __future__ import absolute_import import errno -import kombu import logging import os from collections import defaultdict -from functools import partial -from heapq import heappush -from operator import itemgetter from time import sleep from amqp.promise import ppartial, promise from billiard.common import restart_state from billiard.exceptions import RestartFreqExceeded from kombu.async.semaphore import DummyLock -from kombu.common import QoS, ignore_errors -from kombu.five import buffer_t, items, values +from kombu.five import buffer_t, items from kombu.syn import _detect_environment from kombu.utils.encoding import safe_repr, bytes_t from kombu.utils.limits import TokenBucket @@ -34,22 +29,19 @@ from celery import bootsteps from celery import signals from celery.app.trace import build_tracer -from celery.canvas import signature from celery.exceptions import InvalidTaskError, NotRegistered from celery.utils import gethostname from celery.utils.functional import noop from celery.utils.log import get_logger -from celery.utils.objects import Bunch from celery.utils.text import truncate from celery.utils.timeutils import humanize_seconds, rate -from . import heartbeat, loops, pidbox -from .state import task_reserved, maybe_shutdown, revoked, reserved_requests +from celery.worker import loops +from celery.worker.state import ( + task_reserved, maybe_shutdown, reserved_requests, +) -__all__ = [ - 'Consumer', 'Connection', 'Events', 'Heart', 'Control', - 'Tasks', 'Evloop', 'Agent', 'Mingle', 'Gossip', 'dump_body', -] +__all__ = ['Consumer', 'Evloop', 'dump_body'] CLOSE = bootsteps.CLOSE logger = get_logger(__name__) @@ -117,8 +109,6 @@ delivery_info:{3} headers={4}}} """ -MINGLE_GET_FIELDS = itemgetter('clock', 'revoked') - def dump_body(m, body): # v2 protocol does not deserialize body @@ -130,6 +120,7 @@ def dump_body(m, body): class Consumer(object): + Strategies = dict #: set when consumer is shutting down. @@ -151,15 +142,15 @@ class Consumer(object): class Blueprint(bootsteps.Blueprint): name = 'Consumer' default_steps = [ - 'celery.worker.consumer:Connection', - 'celery.worker.consumer:Mingle', - 'celery.worker.consumer:Events', - 'celery.worker.consumer:Gossip', - 'celery.worker.consumer:Heart', - 'celery.worker.consumer:Control', - 'celery.worker.consumer:Tasks', - 'celery.worker.consumer:Evloop', - 'celery.worker.consumer:Agent', + 'celery.worker.consumer.connection:Connection', + 'celery.worker.consumer.mingle:Mingle', + 'celery.worker.consumer.events:Events', + 'celery.worker.consumer.gossip:Gossip', + 'celery.worker.consumer.heart:Heart', + 'celery.worker.consumer.control:Control', + 'celery.worker.consumer.tasks:Tasks', + 'celery.worker.consumer.consumer:Evloop', + 'celery.worker.consumer.agent:Agent', ] def shutdown(self, parent): @@ -538,377 +529,8 @@ def __repr__(self): ) -class Connection(bootsteps.StartStopStep): - - def __init__(self, c, **kwargs): - c.connection = None - - def start(self, c): - c.connection = c.connect() - info('Connected to %s', c.connection.as_uri()) - - def shutdown(self, c): - # We must set self.connection to None here, so - # that the green pidbox thread exits. - connection, c.connection = c.connection, None - if connection: - ignore_errors(connection, connection.close) - - def info(self, c, params='N/A'): - if c.connection: - params = c.connection.info() - params.pop('password', None) # don't send password. - return {'broker': params} - - -class Events(bootsteps.StartStopStep): - requires = (Connection,) - - def __init__(self, c, send_events=True, - without_heartbeat=False, without_gossip=False, **kwargs): - self.groups = None if send_events else ['worker'] - self.send_events = ( - send_events or - not without_gossip or - not without_heartbeat - ) - c.event_dispatcher = None - - def start(self, c): - # flush events sent while connection was down. - prev = self._close(c) - dis = c.event_dispatcher = c.app.events.Dispatcher( - c.connect(), hostname=c.hostname, - enabled=self.send_events, groups=self.groups, - buffer_group=['task'] if c.hub else None, - on_send_buffered=c.on_send_event_buffered if c.hub else None, - ) - if prev: - dis.extend_buffer(prev) - dis.flush() - - def stop(self, c): - pass - - def _close(self, c): - if c.event_dispatcher: - dispatcher = c.event_dispatcher - # remember changes from remote control commands: - self.groups = dispatcher.groups - - # close custom connection - if dispatcher.connection: - ignore_errors(c, dispatcher.connection.close) - ignore_errors(c, dispatcher.close) - c.event_dispatcher = None - return dispatcher - - def shutdown(self, c): - self._close(c) - - -class Heart(bootsteps.StartStopStep): - requires = (Events,) - - def __init__(self, c, without_heartbeat=False, heartbeat_interval=None, - **kwargs): - self.enabled = not without_heartbeat - self.heartbeat_interval = heartbeat_interval - c.heart = None - - def start(self, c): - c.heart = heartbeat.Heart( - c.timer, c.event_dispatcher, self.heartbeat_interval, - ) - c.heart.start() - - def stop(self, c): - c.heart = c.heart and c.heart.stop() - shutdown = stop - - -class Mingle(bootsteps.StartStopStep): - label = 'Mingle' - requires = (Events,) - compatible_transports = {'amqp', 'redis'} - - def __init__(self, c, without_mingle=False, **kwargs): - self.enabled = not without_mingle and self.compatible_transport(c.app) - - def compatible_transport(self, app): - with app.connection_for_read() as conn: - return conn.transport.driver_type in self.compatible_transports - - def start(self, c): - info('mingle: searching for neighbors') - I = c.app.control.inspect(timeout=1.0, connection=c.connection) - replies = I.hello(c.hostname, revoked._data) or {} - replies.pop(c.hostname, None) - if replies: - info('mingle: sync with %s nodes', - len([reply for reply, value in items(replies) if value])) - for reply in values(replies): - if reply: - try: - other_clock, other_revoked = MINGLE_GET_FIELDS(reply) - except KeyError: # reply from pre-3.1 worker - pass - else: - c.app.clock.adjust(other_clock) - revoked.update(other_revoked) - info('mingle: sync complete') - else: - info('mingle: all alone') - - -class Tasks(bootsteps.StartStopStep): - requires = (Mingle,) - - def __init__(self, c, **kwargs): - c.task_consumer = c.qos = None - - def start(self, c): - c.update_strategies() - - # - RabbitMQ 3.3 completely redefines how basic_qos works.. - # This will detect if the new qos smenatics is in effect, - # and if so make sure the 'apply_global' flag is set on qos updates. - qos_global = not c.connection.qos_semantics_matches_spec - - # set initial prefetch count - c.connection.default_channel.basic_qos( - 0, c.initial_prefetch_count, qos_global, - ) - - c.task_consumer = c.app.amqp.TaskConsumer( - c.connection, on_decode_error=c.on_decode_error, - ) - - def set_prefetch_count(prefetch_count): - return c.task_consumer.qos( - prefetch_count=prefetch_count, - apply_global=qos_global, - ) - c.qos = QoS(set_prefetch_count, c.initial_prefetch_count) - - def stop(self, c): - if c.task_consumer: - debug('Cancelling task consumer...') - ignore_errors(c, c.task_consumer.cancel) - - def shutdown(self, c): - if c.task_consumer: - self.stop(c) - debug('Closing consumer channel...') - ignore_errors(c, c.task_consumer.close) - c.task_consumer = None - - def info(self, c): - return {'prefetch_count': c.qos.value if c.qos else 'N/A'} - - -class Agent(bootsteps.StartStopStep): - conditional = True - requires = (Connection,) - - def __init__(self, c, **kwargs): - self.agent_cls = self.enabled = c.app.conf.worker_agent - - def create(self, c): - agent = c.agent = self.instantiate(self.agent_cls, c.connection) - return agent - - -class Control(bootsteps.StartStopStep): - requires = (Tasks,) - - def __init__(self, c, **kwargs): - self.is_green = c.pool is not None and c.pool.is_green - self.box = (pidbox.gPidbox if self.is_green else pidbox.Pidbox)(c) - self.start = self.box.start - self.stop = self.box.stop - self.shutdown = self.box.shutdown - - def include_if(self, c): - return (c.app.conf.worker_enable_remote_control and - c.conninfo.supports_exchange_type('fanout')) - - -class Gossip(bootsteps.ConsumerStep): - label = 'Gossip' - requires = (Mingle,) - _cons_stamp_fields = itemgetter( - 'id', 'clock', 'hostname', 'pid', 'topic', 'action', 'cver', - ) - compatible_transports = {'amqp', 'redis'} - - def __init__(self, c, without_gossip=False, - interval=5.0, heartbeat_interval=2.0, **kwargs): - self.enabled = not without_gossip and self.compatible_transport(c.app) - self.app = c.app - c.gossip = self - self.Receiver = c.app.events.Receiver - self.hostname = c.hostname - self.full_hostname = '.'.join([self.hostname, str(c.pid)]) - self.on = Bunch( - node_join=set(), - node_leave=set(), - node_lost=set(), - ) - - self.timer = c.timer - if self.enabled: - self.state = c.app.events.State( - on_node_join=self.on_node_join, - on_node_leave=self.on_node_leave, - max_tasks_in_memory=1, - ) - if c.hub: - c._mutex = DummyLock() - self.update_state = self.state.event - self.interval = interval - self.heartbeat_interval = heartbeat_interval - self._tref = None - self.consensus_requests = defaultdict(list) - self.consensus_replies = {} - self.event_handlers = { - 'worker.elect': self.on_elect, - 'worker.elect.ack': self.on_elect_ack, - } - self.clock = c.app.clock - - self.election_handlers = { - 'task': self.call_task - } - - def compatible_transport(self, app): - with app.connection_for_read() as conn: - return conn.transport.driver_type in self.compatible_transports - - def election(self, id, topic, action=None): - self.consensus_replies[id] = [] - self.dispatcher.send( - 'worker-elect', - id=id, topic=topic, action=action, cver=1, - ) - - def call_task(self, task): - try: - signature(task, app=self.app).apply_async() - except Exception as exc: - error('Could not call task: %r', exc, exc_info=1) - - def on_elect(self, event): - try: - (id_, clock, hostname, pid, - topic, action, _) = self._cons_stamp_fields(event) - except KeyError as exc: - return error('election request missing field %s', exc, exc_info=1) - heappush( - self.consensus_requests[id_], - (clock, '%s.%s' % (hostname, pid), topic, action), - ) - self.dispatcher.send('worker-elect-ack', id=id_) - - def start(self, c): - super(Gossip, self).start(c) - self.dispatcher = c.event_dispatcher - - def on_elect_ack(self, event): - id = event['id'] - try: - replies = self.consensus_replies[id] - except KeyError: - return # not for us - alive_workers = self.state.alive_workers() - replies.append(event['hostname']) - - if len(replies) >= len(alive_workers): - _, leader, topic, action = self.clock.sort_heap( - self.consensus_requests[id], - ) - if leader == self.full_hostname: - info('I won the election %r', id) - try: - handler = self.election_handlers[topic] - except KeyError: - error('Unknown election topic %r', topic, exc_info=1) - else: - handler(action) - else: - info('node %s elected for %r', leader, id) - self.consensus_requests.pop(id, None) - self.consensus_replies.pop(id, None) - - def on_node_join(self, worker): - debug('%s joined the party', worker.hostname) - self._call_handlers(self.on.node_join, worker) - - def on_node_leave(self, worker): - debug('%s left', worker.hostname) - self._call_handlers(self.on.node_leave, worker) - - def on_node_lost(self, worker): - info('missed heartbeat from %s', worker.hostname) - self._call_handlers(self.on.node_lost, worker) - - def _call_handlers(self, handlers, *args, **kwargs): - for handler in handlers: - try: - handler(*args, **kwargs) - except Exception as exc: - error('Ignored error from handler %r: %r', - handler, exc, exc_info=1) - - def register_timer(self): - if self._tref is not None: - self._tref.cancel() - self._tref = self.timer.call_repeatedly(self.interval, self.periodic) - - def periodic(self): - workers = self.state.workers - dirty = set() - for worker in values(workers): - if not worker.alive: - dirty.add(worker) - self.on_node_lost(worker) - for worker in dirty: - workers.pop(worker.hostname, None) - - def get_consumers(self, channel): - self.register_timer() - ev = self.Receiver(channel, routing_key='worker.#', - queue_ttl=self.heartbeat_interval) - return [kombu.Consumer( - channel, - queues=[ev.queue], - on_message=partial(self.on_message, ev.event_from_message), - no_ack=True - )] - - def on_message(self, prepare, message): - _type = message.delivery_info['routing_key'] - - # For redis when `fanout_patterns=False` (See Issue #1882) - if _type.split('.', 1)[0] == 'task': - return - try: - handler = self.event_handlers[_type] - except KeyError: - pass - else: - return handler(message.payload) - - hostname = (message.headers.get('hostname') or - message.payload['hostname']) - if hostname != self.hostname: - type, event = prepare(message.payload) - self.update_state(event) - else: - self.clock.forward() - - class Evloop(bootsteps.StartStopStep): + label = 'event loop' last = True diff --git a/celery/worker/consumer/control.py b/celery/worker/consumer/control.py new file mode 100644 index 000000000..f99b2fc7e --- /dev/null +++ b/celery/worker/consumer/control.py @@ -0,0 +1,27 @@ +from __future__ import absolute_import, unicode_literals + +from celery import bootsteps +from celery.utils.log import get_logger + +from celery.worker import pidbox + +from .tasks import Tasks + +__all__ = ['Control'] +logger = get_logger(__name__) + + +class Control(bootsteps.StartStopStep): + + requires = (Tasks,) + + def __init__(self, c, **kwargs): + self.is_green = c.pool is not None and c.pool.is_green + self.box = (pidbox.gPidbox if self.is_green else pidbox.Pidbox)(c) + self.start = self.box.start + self.stop = self.box.stop + self.shutdown = self.box.shutdown + + def include_if(self, c): + return (c.app.conf.worker_enable_remote_control and + c.conninfo.supports_exchange_type('fanout')) diff --git a/celery/worker/consumer/events.py b/celery/worker/consumer/events.py new file mode 100644 index 000000000..0f32f203d --- /dev/null +++ b/celery/worker/consumer/events.py @@ -0,0 +1,56 @@ +from __future__ import absolute_import, unicode_literals + +from kombu.common import ignore_errors + +from celery import bootsteps + +from .connection import Connection + +__all__ = ['Events'] + + +class Events(bootsteps.StartStopStep): + + requires = (Connection,) + + def __init__(self, c, send_events=True, + without_heartbeat=False, without_gossip=False, **kwargs): + self.groups = None if send_events else ['worker'] + self.send_events = ( + send_events or + not without_gossip or + not without_heartbeat + ) + c.event_dispatcher = None + + def start(self, c): + # flush events sent while connection was down. + prev = self._close(c) + dis = c.event_dispatcher = c.app.events.Dispatcher( + c.connect(), hostname=c.hostname, + enabled=self.send_events, groups=self.groups, + buffer_group=['task'] if c.hub else None, + on_send_buffered=c.on_send_event_buffered if c.hub else None, + ) + if prev: + dis.extend_buffer(prev) + dis.flush() + + def stop(self, c): + pass + + def _close(self, c): + if c.event_dispatcher: + dispatcher = c.event_dispatcher + # remember changes from remote control commands: + self.groups = dispatcher.groups + + # close custom connection + if dispatcher.connection: + ignore_errors(c, dispatcher.connection.close) + ignore_errors(c, dispatcher.close) + c.event_dispatcher = None + return dispatcher + + def shutdown(self, c): + self._close(c) diff --git a/celery/worker/consumer/gossip.py b/celery/worker/consumer/gossip.py new file mode 100644 index 000000000..8289ad89c --- /dev/null +++ b/celery/worker/consumer/gossip.py @@ -0,0 +1,195 @@ +from __future__ import absolute_import, unicode_literals + +from collections import defaultdict +from functools import partial +from heapq import heappush +from operator import itemgetter + +from kombu import Consumer +from kombu.async.semaphore import DummyLock + +from celery import bootsteps +from celery.five import values +from celery.utils.log import get_logger +from celery.utils.objects import Bunch + +from .mingle import Mingle + +__all__ = ['Gossip'] +logger = get_logger(__name__) +debug, info, error = logger.debug, logger.info, logger.error + + +class Gossip(bootsteps.ConsumerStep): + + label = 'Gossip' + requires = (Mingle,) + _cons_stamp_fields = itemgetter( + 'id', 'clock', 'hostname', 'pid', 'topic', 'action', 'cver', + ) + compatible_transports = {'amqp', 'redis'} + + def __init__(self, c, without_gossip=False, + interval=5.0, heartbeat_interval=2.0, **kwargs): + self.enabled = not without_gossip and self.compatible_transport(c.app) + self.app = c.app + c.gossip = self + self.Receiver = c.app.events.Receiver + self.hostname = c.hostname + self.full_hostname = '.'.join([self.hostname, str(c.pid)]) + self.on = Bunch( + node_join=set(), + node_leave=set(), + node_lost=set(), + ) + + self.timer = c.timer + if self.enabled: + self.state = c.app.events.State( + on_node_join=self.on_node_join, + on_node_leave=self.on_node_leave, + max_tasks_in_memory=1, + ) + if c.hub: + c._mutex = DummyLock() + self.update_state = self.state.event + self.interval = interval + self.heartbeat_interval = heartbeat_interval + self._tref = None + self.consensus_requests = defaultdict(list) + self.consensus_replies = {} + self.event_handlers = { + 'worker.elect': self.on_elect, + 'worker.elect.ack': self.on_elect_ack, + } + self.clock = c.app.clock + + self.election_handlers = { + 'task': self.call_task + } + + def compatible_transport(self, app): + with app.connection_for_read() as conn: + return conn.transport.driver_type in self.compatible_transports + + def election(self, id, topic, action=None): + self.consensus_replies[id] = [] + self.dispatcher.send( + 'worker-elect', + id=id, topic=topic, action=action, cver=1, + ) + + def call_task(self, task): + try: + self.app.signature(task).apply_async() + except Exception as exc: + error('Could not call task: %r', exc, exc_info=1) + + def on_elect(self, event): + try: + (id_, clock, hostname, pid, + topic, action, _) = self._cons_stamp_fields(event) + except KeyError as exc: + return error('election request missing field %s', exc, exc_info=1) + heappush( + self.consensus_requests[id_], + (clock, '%s.%s' % (hostname, pid), topic, action), + ) + self.dispatcher.send('worker-elect-ack', id=id_) + + def start(self, c): + super(Gossip, self).start(c) + self.dispatcher = c.event_dispatcher + + def on_elect_ack(self, event): + id = event['id'] + try: + replies = self.consensus_replies[id] + except KeyError: + return # not for us + alive_workers = self.state.alive_workers() + replies.append(event['hostname']) + + if len(replies) >= len(alive_workers): + _, leader, topic, action = self.clock.sort_heap( + self.consensus_requests[id], + ) + if leader == self.full_hostname: + info('I won the election %r', id) + try: + handler = self.election_handlers[topic] + except KeyError: + error('Unknown election topic %r', topic, exc_info=1) + else: + handler(action) + else: + info('node %s elected for %r', leader, id) + self.consensus_requests.pop(id, None) + self.consensus_replies.pop(id, None) + + def on_node_join(self, worker): + debug('%s joined the party', worker.hostname) + self._call_handlers(self.on.node_join, worker) + + def on_node_leave(self, worker): + debug('%s left', worker.hostname) + self._call_handlers(self.on.node_leave, worker) + + def on_node_lost(self, worker): + info('missed heartbeat from %s', worker.hostname) + self._call_handlers(self.on.node_lost, worker) + + def _call_handlers(self, handlers, *args, **kwargs): + for handler in handlers: + try: + handler(*args, **kwargs) + except Exception as exc: + error('Ignored error from handler %r: %r', + handler, exc, exc_info=1) + + def register_timer(self): + if self._tref is not None: + self._tref.cancel() + self._tref = self.timer.call_repeatedly(self.interval, self.periodic) + + def periodic(self): + workers = self.state.workers + dirty = set() + for worker in values(workers): + if not worker.alive: + dirty.add(worker) + self.on_node_lost(worker) + for worker in dirty: + workers.pop(worker.hostname, None) + + def get_consumers(self, channel): + self.register_timer() + ev = self.Receiver(channel, routing_key='worker.#', + queue_ttl=self.heartbeat_interval) + return [Consumer( + channel, + queues=[ev.queue], + on_message=partial(self.on_message, ev.event_from_message), + no_ack=True + )] + + def on_message(self, prepare, message): + _type = message.delivery_info['routing_key'] + + # For redis when `fanout_patterns=False` (See Issue #1882) + if _type.split('.', 1)[0] == 'task': + return + try: + handler = self.event_handlers[_type] + except KeyError: + pass + else: + return handler(message.payload) + + hostname = (message.headers.get('hostname') or + message.payload['hostname']) + if hostname != self.hostname: + type, event = prepare(message.payload) + self.update_state(event) + else: + self.clock.forward() diff --git a/celery/worker/consumer/heart.py b/celery/worker/consumer/heart.py new file mode 100644 index 000000000..0f0173c63 --- /dev/null +++ b/celery/worker/consumer/heart.py @@ -0,0 +1,30 @@ +from __future__ import absolute_import, unicode_literals + +from celery import bootsteps + +from celery.worker import heartbeat + +from .events import Events + +__all__ = ['Heart'] + + +class Heart(bootsteps.StartStopStep): + + requires = (Events,) + + def __init__(self, c, + without_heartbeat=False, heartbeat_interval=None, **kwargs): + self.enabled = not without_heartbeat + self.heartbeat_interval = heartbeat_interval + c.heart = None + + def start(self, c): + c.heart = heartbeat.Heart( + c.timer, c.event_dispatcher, self.heartbeat_interval, + ) + c.heart.start() + + def stop(self, c): + c.heart = c.heart and c.heart.stop() + shutdown = stop diff --git a/celery/worker/consumer/mingle.py b/celery/worker/consumer/mingle.py new file mode 100644 index 000000000..70f07f6b3 --- /dev/null +++ b/celery/worker/consumer/mingle.py @@ -0,0 +1,53 @@ +from __future__ import absolute_import, unicode_literals + +from operator import itemgetter + +from celery import bootsteps +from celery.five import items, values +from celery.utils.log import get_logger + +from celery.worker.state import revoked + +from .events import Events + +__all__ = ['Mingle'] + +MINGLE_GET_FIELDS = itemgetter('clock', 'revoked') + +logger = get_logger(__name__) +info = logger.info + + +class Mingle(bootsteps.StartStopStep): + + label = 'Mingle' + requires = (Events,) + compatible_transports = {'amqp', 'redis'} + + def __init__(self, c, without_mingle=False, **kwargs): + self.enabled = not without_mingle and self.compatible_transport(c.app) + + def compatible_transport(self, app): + with app.connection_for_read() as conn: + return conn.transport.driver_type in self.compatible_transports + + def start(self, c): + info('mingle: searching for neighbors') + I = c.app.control.inspect(timeout=1.0, connection=c.connection) + replies = I.hello(c.hostname, revoked._data) or {} + replies.pop(c.hostname, None) + if replies: + info('mingle: sync with %s nodes', + len([reply for reply, value in items(replies) if value])) + for reply in values(replies): + if reply: + try: + other_clock, other_revoked = MINGLE_GET_FIELDS(reply) + except KeyError: # reply from pre-3.1 worker + pass + else: + c.app.clock.adjust(other_clock) + revoked.update(other_revoked) + info('mingle: sync complete') + else: + info('mingle: all alone') diff --git a/celery/worker/consumer/tasks.py b/celery/worker/consumer/tasks.py new file mode 100644 index 000000000..56467455f --- /dev/null +++ b/celery/worker/consumer/tasks.py @@ -0,0 +1,59 @@ +from __future__ import absolute_import, unicode_literals + +from kombu.common import QoS, ignore_errors + +from celery import bootsteps +from celery.utils.log import get_logger + +from .mingle import Mingle + +__all__ = ['Tasks'] +logger = get_logger(__name__) +debug = logger.debug + + +class Tasks(bootsteps.StartStopStep): + + requires = (Mingle,) + + def __init__(self, c, **kwargs): + c.task_consumer = c.qos = None + + def start(self, c): + c.update_strategies() + + # - RabbitMQ 3.3 completely redefines how basic_qos works.. + # This will detect if the new qos smenatics is in effect, + # and if so make sure the 'apply_global' flag is set on qos updates. + qos_global = not c.connection.qos_semantics_matches_spec + + # set initial prefetch count + c.connection.default_channel.basic_qos( + 0, c.initial_prefetch_count, qos_global, + ) + + c.task_consumer = c.app.amqp.TaskConsumer( + c.connection, on_decode_error=c.on_decode_error, + ) + + def set_prefetch_count(prefetch_count): + return c.task_consumer.qos( + prefetch_count=prefetch_count, + apply_global=qos_global, + ) + c.qos = QoS(set_prefetch_count, c.initial_prefetch_count) + + def stop(self, c): + if c.task_consumer: + debug('Cancelling task consumer...') + ignore_errors(c, c.task_consumer.cancel) + + def shutdown(self, c): + if c.task_consumer: + self.stop(c) + debug('Closing consumer channel...') + ignore_errors(c, c.task_consumer.close) + c.task_consumer = None + + def info(self, c): + return {'prefetch_count': c.qos.value if c.qos else 'N/A'} From dd711fac7a67aea5d6ab01d7ca09b1802cd6b3ff Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 1 Feb 2016 17:05:32 -0800 Subject: [PATCH 0988/1103] Use US spelling of cancelled -> canceled --- celery/concurrency/asynpool.py | 2 +- celery/concurrency/eventlet.py | 4 ++-- celery/concurrency/gevent.py | 2 +- celery/contrib/batches.py | 2 +- celery/tests/utils/test_timer2.py | 4 ++-- celery/tests/worker/test_control.py | 6 +++--- celery/tests/worker/test_heartbeat.py | 4 ++-- celery/worker/consumer/consumer.py | 2 +- celery/worker/consumer/tasks.py | 2 +- celery/worker/pidbox.py | 2 +- docs/userguide/workers.rst | 4 ++-- 11 files changed, 17 insertions(+), 17 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 781370a16..4b9aeff67 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -805,7 +805,7 @@ def _write_job(proc, fd, job): # writes job to the worker process. # Operation must complete if more than one byte of data # was written. If the broker connection is lost - # and no data was written the operation shall be cancelled. + # and no data was written the operation shall be canceled. header, body, body_size = job._payload errors = 0 try: diff --git a/celery/concurrency/eventlet.py b/celery/concurrency/eventlet.py index 7a8c9ae1b..6991e0608 100644 --- a/celery/concurrency/eventlet.py +++ b/celery/concurrency/eventlet.py @@ -62,7 +62,7 @@ def _enter(self, eta, priority, entry): g.entry = entry g.eta = eta g.priority = priority - g.cancelled = False + g.canceled = False return g def _entry_exit(self, g, entry): @@ -71,7 +71,7 @@ def _entry_exit(self, g, entry): g.wait() except self.GreenletExit: entry.cancel() - g.cancelled = True + g.canceled = True finally: self._queue.discard(g) diff --git a/celery/concurrency/gevent.py b/celery/concurrency/gevent.py index ba39c8f8b..dc0f13203 100644 --- a/celery/concurrency/gevent.py +++ b/celery/concurrency/gevent.py @@ -56,7 +56,7 @@ def _enter(self, eta, priority, entry): g.entry = entry g.eta = eta g.priority = priority - g.cancelled = False + g.canceled = False return g def _entry_exit(self, g): diff --git a/celery/contrib/batches.py b/celery/contrib/batches.py index 0ceac4aad..c2ca0c41b 100644 --- a/celery/contrib/batches.py +++ b/celery/contrib/batches.py @@ -241,7 +241,7 @@ def _do_flush(self): logger.debug('Batches: Buffer complete: %s', len(requests)) self.flush(requests) if not requests: - logger.debug('Batches: Cancelling timer: Nothing in buffer.') + logger.debug('Batches: Canceling timer: Nothing in buffer.') if self._tref: self._tref.cancel() # cancel timer. self._tref = None diff --git a/celery/tests/utils/test_timer2.py b/celery/tests/utils/test_timer2.py index 5bcd1ba37..e159b209f 100644 --- a/celery/tests/utils/test_timer2.py +++ b/celery/tests/utils/test_timer2.py @@ -25,7 +25,7 @@ def timed(x, y, moo='foo'): def test_cancel(self): tref = timer2.Entry(lambda x: x, (1,), {}) tref.cancel() - self.assertTrue(tref.cancelled) + self.assertTrue(tref.canceled) def test_repr(self): tref = timer2.Entry(lambda x: x(1,), {}) @@ -123,7 +123,7 @@ def test_call_repeatedly(self): args2, _ = t.schedule.enter_after.call_args_list[1] sec2, tref2, _ = args2 self.assertEqual(sec2, 0.03) - tref2.cancelled = True + tref2.canceled = True tref2() self.assertEqual(t.schedule.enter_after.call_count, 2) diff --git a/celery/tests/worker/test_control.py b/celery/tests/worker/test_control.py index 2619cecb8..cb016215a 100644 --- a/celery/tests/worker/test_control.py +++ b/celery/tests/worker/test_control.py @@ -347,7 +347,7 @@ def test_add__cancel_consumer(self): class MockConsumer(object): queues = [] - cancelled = [] + canceled = [] consuming = False hub = Mock(name='hub') @@ -358,7 +358,7 @@ def consume(self): self.consuming = True def cancel_by_queue(self, queue): - self.cancelled.append(queue) + self.canceled.append(queue) def consuming_from(self, queue): return queue in self.queues @@ -372,7 +372,7 @@ def consuming_from(self, queue): self.assertTrue(consumer.task_consumer.consuming) panel.handle('add_consumer', {'queue': 'MyQueue'}) panel.handle('cancel_consumer', {'queue': 'MyQueue'}) - self.assertIn('MyQueue', consumer.task_consumer.cancelled) + self.assertIn('MyQueue', consumer.task_consumer.canceled) def test_revoked(self): worker_state.revoked.clear() diff --git a/celery/tests/worker/test_heartbeat.py b/celery/tests/worker/test_heartbeat.py index 5568e4ec4..50559ca11 100644 --- a/celery/tests/worker/test_heartbeat.py +++ b/celery/tests/worker/test_heartbeat.py @@ -34,10 +34,10 @@ class MockTimer(object): def call_repeatedly(self, secs, fun, args=(), kwargs={}): class entry(tuple): - cancelled = False + canceled = False def cancel(self): - self.cancelled = True + self.canceled = True return entry((secs, fun, args, kwargs)) diff --git a/celery/worker/consumer/consumer.py b/celery/worker/consumer/consumer.py index 7a0147490..41ae346c1 100644 --- a/celery/worker/consumer/consumer.py +++ b/celery/worker/consumer/consumer.py @@ -424,7 +424,7 @@ def add_task_queue(self, queue, exchange=None, exchange_type=None, info('Started consuming from %s', queue) def cancel_task_queue(self, queue): - info('Cancelling queue %s', queue) + info('Canceling queue %s', queue) self.app.amqp.queues.deselect(queue) self.task_consumer.cancel_by_queue(queue) diff --git a/celery/worker/consumer/tasks.py b/celery/worker/consumer/tasks.py index 56467455f..2a4f9b785 100644 --- a/celery/worker/consumer/tasks.py +++ b/celery/worker/consumer/tasks.py @@ -45,7 +45,7 @@ def set_prefetch_count(prefetch_count): def stop(self, c): if c.task_consumer: - debug('Cancelling task consumer...') + debug('Canceling task consumer...') ignore_errors(c, c.task_consumer.cancel) def shutdown(self, c): diff --git a/celery/worker/pidbox.py b/celery/worker/pidbox.py index 72bdd3714..374aaca1f 100644 --- a/celery/worker/pidbox.py +++ b/celery/worker/pidbox.py @@ -71,7 +71,7 @@ def _close_channel(self, c): def shutdown(self, c): self.on_stop() if self.consumer: - debug('Cancelling broadcast consumer...') + debug('Canceling broadcast consumer...') ignore_errors(c, self.consumer.cancel) self.stop(self.c) diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index 6a78c8438..7a2294a30 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -656,8 +656,8 @@ even other options: .. control:: cancel_consumer -Queues: Cancelling consumers ----------------------------- +Queues: Canceling consumers +--------------------------- You can cancel a consumer by queue name using the :control:`cancel_consumer` control command. From 915dcc9ac32c6e7311654c1531ae06fcc855f726 Mon Sep 17 00:00:00 2001 From: Gao Jiangmiao Date: Wed, 3 Feb 2016 15:32:46 +0800 Subject: [PATCH 0989/1103] Fix typo in docs: after_task_publush -> after_task_publish --- docs/internals/deprecation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internals/deprecation.rst b/docs/internals/deprecation.rst index 23df5be0b..4d0900ea6 100644 --- a/docs/internals/deprecation.rst +++ b/docs/internals/deprecation.rst @@ -156,7 +156,7 @@ Task_sent signal ---------------- The :signal:`task_sent` signal will be removed in version 4.0. -Please use the :signal:`before_task_publish` and :signal:`after_task_publush` +Please use the :signal:`before_task_publish` and :signal:`after_task_publish` signals instead. Result From 54049ea21c36771fdadc19c020d353524f52cef6 Mon Sep 17 00:00:00 2001 From: Evgeniy Date: Wed, 3 Feb 2016 13:42:56 +0300 Subject: [PATCH 0990/1103] bug in __init__ --- examples/eventlet/bulk_task_producer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/eventlet/bulk_task_producer.py b/examples/eventlet/bulk_task_producer.py index 4bc75a215..891a900fc 100644 --- a/examples/eventlet/bulk_task_producer.py +++ b/examples/eventlet/bulk_task_producer.py @@ -10,7 +10,7 @@ class Receipt(object): result = None def __init__(self, callback=None): - self.callback = None + self.callback = callback self.ready = Event() def finished(self, result): From 8394816deb646c7cd277e6be6879f8faf0e47123 Mon Sep 17 00:00:00 2001 From: David Harrigan Date: Fri, 5 Feb 2016 13:42:39 -0500 Subject: [PATCH 0991/1103] raise ImproperlyConfigured exception with invalid auth_provider --- celery/backends/cassandra.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index 55bf1e7f3..e6a3f02e7 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -30,6 +30,11 @@ use the Cassandra backend. See https://github.com/datastax/python-driver """ +E_NO_SUCH_CASSANDRA_AUTH_PROVIDER = """ +CASSANDRA_AUTH_PROVIDER you provided is not a valid auth_provider class. +See https://datastax.github.io/python-driver/api/cassandra/auth.html. +""" + Q_INSERT_RESULT = """ INSERT INTO {table} ( task_id, status, result, date_done, traceback, children) VALUES ( @@ -126,7 +131,9 @@ def __init__(self, servers=None, keyspace=None, table=None, entry_ttl=None, auth_provider = conf.get('cassandra_auth_provider', None) auth_kwargs = conf.get('cassandra_auth_kwargs', None) if auth_provider and auth_kwargs: - auth_provider_class = getattr(cassandra.auth, auth_provider) + auth_provider_class = getattr(cassandra.auth, auth_provider, None) + if not auth_provider_class: + raise ImproperlyConfigured(E_NO_SUCH_CASSANDRA_AUTH_PROVIDER) self.auth_provider = auth_provider_class(**auth_kwargs) self._connection = None From a184da39ebfe9d4972e26c5be30457d2a7ff97f4 Mon Sep 17 00:00:00 2001 From: David Harrigan Date: Fri, 5 Feb 2016 13:43:14 -0500 Subject: [PATCH 0992/1103] update CONTRIBUTORS --- CONTRIBUTORS.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index ab6ecc0d6..54c8b06d5 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -203,4 +203,4 @@ Chris Harris, 2015/11/27 Valentyn Klindukh, 2016/01/15 Wayne Chang, 2016/01/15 Mike Attwood, 2016/01/22 -David Harrigan, 2016/2/1 +David Harrigan, 2016/02/01 From 8003449a688ed3aad8df787418b8b4182f29c245 Mon Sep 17 00:00:00 2001 From: Jason Veatch Date: Tue, 9 Feb 2016 13:53:09 -0500 Subject: [PATCH 0993/1103] Deleted docs on removed CentOS init script For additional info, see https://github.com/celery/celery/issues/1895 --- docs/tutorials/daemonizing.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/tutorials/daemonizing.rst b/docs/tutorials/daemonizing.rst index edb7e80b3..be8a5b8a8 100644 --- a/docs/tutorials/daemonizing.rst +++ b/docs/tutorials/daemonizing.rst @@ -424,9 +424,3 @@ Windows See this excellent external tutorial: http://www.calazan.com/windows-tip-run-applications-in-the-background-using-task-scheduler/ - -CentOS -====== -In CentOS we can take advantage of built-in service helpers, such as the -pid-based status checker function in ``/etc/init.d/functions``. -See the sample script in http://github.com/celery/celery/tree/3.1/extra/centos/. From 391eb97f69de5743ca269aab217e53fd2cb73449 Mon Sep 17 00:00:00 2001 From: David Harrigan Date: Mon, 15 Feb 2016 01:08:49 -0500 Subject: [PATCH 0994/1103] add test case for cassandra auth_provider option --- celery/tests/backends/test_cassandra.py | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/celery/tests/backends/test_cassandra.py b/celery/tests/backends/test_cassandra.py index 9a798badf..d97e584f4 100644 --- a/celery/tests/backends/test_cassandra.py +++ b/celery/tests/backends/test_cassandra.py @@ -168,3 +168,30 @@ def shutdown(self): x.process_cleanup() self.assertEquals(RAMHoggingCluster.objects_alive, 0) + + def test_auth_provider(self): + """Ensure valid auth_provider works properly, and invalid one raises + ImproperlyConfigured exception.""" + class DummyAuth(object): + ValidAuthProvider = Mock() + + with mock_module(*CASSANDRA_MODULES): + from celery.backends import cassandra as mod + + mod.cassandra = Mock() + mod.cassandra.auth = DummyAuth + + # Valid auth_provider + self.app.conf.cassandra_auth_provider = 'ValidAuthProvider' + self.app.conf.cassandra_auth_kwargs = { + 'username': 'stuff' + } + mod.CassandraBackend(app=self.app) + + # Invalid auth_provider + self.app.conf.cassandra_auth_provider = 'SpiderManAuth' + self.app.conf.cassandra_auth_kwargs = { + 'username': 'Jack' + } + with self.assertRaises(ImproperlyConfigured): + mod.CassandraBackend(app=self.app) From c9e6d154664c74429986f3ea23ec4b4a63a57a11 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 15 Feb 2016 10:20:21 +0200 Subject: [PATCH 0995/1103] Enable coverage for cassandra. --- .coveragerc | 1 - 1 file changed, 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 3c2098230..c2d1c7c9a 100644 --- a/.coveragerc +++ b/.coveragerc @@ -16,7 +16,6 @@ omit = *celery/contrib/sphinx.py *celery/backends/couchdb.py *celery/backends/couchbase.py - *celery/backends/cassandra.py *celery/backends/riak.py *celery/concurrency/asynpool.py *celery/utils/debug.py From 15edb778de1e9cf76ca565fd964936bf16585b6a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 19 Feb 2016 11:56:30 -0800 Subject: [PATCH 0996/1103] [Stresstests] Adds vagrant thing --- funtests/stress/run/Vagrantfile | 125 ++++++++++++ .../stress/run/provision/celeryd-init.config | 12 ++ funtests/stress/run/provision/provision.sh | 187 ++++++++++++++++++ 3 files changed, 324 insertions(+) create mode 100644 funtests/stress/run/Vagrantfile create mode 100644 funtests/stress/run/provision/celeryd-init.config create mode 100644 funtests/stress/run/provision/provision.sh diff --git a/funtests/stress/run/Vagrantfile b/funtests/stress/run/Vagrantfile new file mode 100644 index 000000000..65b4e1443 --- /dev/null +++ b/funtests/stress/run/Vagrantfile @@ -0,0 +1,125 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + # All Vagrant configuration is done here. The most common configuration + # options are documented and commented below. For a complete reference, + # please see the online documentation at vagrantup.com. + + # Every Vagrant virtual environment requires a box to build off of. + config.vm.box = "ubuntu/trusty64" + + config.vm.provision :shell, path: "provision/provision.sh", + privileged: true + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # config.vm.network "forwarded_port", guest: 80, host: 8080 + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + config.vm.network "private_network", ip: "192.168.33.123" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # If true, then any SSH connections made will enable agent forwarding. + # Default value: false + # config.ssh.forward_agent = true + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # # Don't boot with headless mode + # vb.gui = true + # + # # Use VBoxManage to customize the VM. For example to change memory: + vb.customize ["modifyvm", :id, "--memory", "1024"] + end + # + # View the documentation for the provider you're using for more + # information on available options. + + # Enable provisioning with CFEngine. CFEngine Community packages are + # automatically installed. For example, configure the host as a + # policy server and optionally a policy file to run: + # + # config.vm.provision "cfengine" do |cf| + # cf.am_policy_hub = true + # # cf.run_file = "motd.cf" + # end + # + # You can also configure and bootstrap a client to an existing + # policy server: + # + # config.vm.provision "cfengine" do |cf| + # cf.policy_server_address = "10.0.2.15" + # end + + # Enable provisioning with Puppet stand alone. Puppet manifests + # are contained in a directory path relative to this Vagrantfile. + # You will need to create the manifests directory and a manifest in + # the file default.pp in the manifests_path directory. + # + # config.vm.provision "puppet" do |puppet| + # puppet.manifests_path = "manifests" + # puppet.manifest_file = "site.pp" + # end + + # Enable provisioning with chef solo, specifying a cookbooks path, roles + # path, and data_bags path (all relative to this Vagrantfile), and adding + # some recipes and/or roles. + # + # config.vm.provision "chef_solo" do |chef| + # chef.cookbooks_path = "../my-recipes/cookbooks" + # chef.roles_path = "../my-recipes/roles" + # chef.data_bags_path = "../my-recipes/data_bags" + # chef.add_recipe "mysql" + # chef.add_role "web" + # + # # You may also specify custom JSON attributes: + # chef.json = { :mysql_password => "foo" } + # end + + # Enable provisioning with chef server, specifying the chef server URL, + # and the path to the validation key (relative to this Vagrantfile). + # + # The Opscode Platform uses HTTPS. Substitute your organization for + # ORGNAME in the URL and validation key. + # + # If you have your own Chef Server, use the appropriate URL, which may be + # HTTP instead of HTTPS depending on your configuration. Also change the + # validation key to validation.pem. + # + # config.vm.provision "chef_client" do |chef| + # chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME" + # chef.validation_key_path = "ORGNAME-validator.pem" + # end + # + # If you're using the Opscode platform, your validator client is + # ORGNAME-validator, replacing ORGNAME with your organization name. + # + # If you have your own Chef Server, the default validation client name is + # chef-validator, unless you changed the configuration. + # + # chef.validation_client_name = "ORGNAME-validator" +end diff --git a/funtests/stress/run/provision/celeryd-init.config b/funtests/stress/run/provision/celeryd-init.config new file mode 100644 index 000000000..5659c3f8c --- /dev/null +++ b/funtests/stress/run/provision/celeryd-init.config @@ -0,0 +1,12 @@ +CELERYD_NODES="worker1" +CELERY_BIN="/usr/local/bin/celery" +CELERY_APP="stress" +CELERYD_CHDIR="/opt/devel/celery/funtests/stress" +#CELERYD_OPTS="" +CELERYD_LOG_FILE="/var/log/celery/%n%I.log" +CELERYD_PID_FILE="/var/run/celery/%n.pid" + +CELERYD_USER="celery" +CELERYD_GROUP="celery" + +CELERY_CREATE_DIRS=1 diff --git a/funtests/stress/run/provision/provision.sh b/funtests/stress/run/provision/provision.sh new file mode 100644 index 000000000..00ef85421 --- /dev/null +++ b/funtests/stress/run/provision/provision.sh @@ -0,0 +1,187 @@ +#!/bin/bash + +echo "------------ HELLO ---------------" + +APT_SOURCES_LST="/etc/apt/sources.list.d/" + +DEVEL_DIR="/opt/devel" + +WGET="wget" +RABBITMQCTL="rabbitmqctl" + +RABBITMQ_APT_URL="http://www.rabbitmq.com/debian/" +RABBITMQ_APT_VER="testing main" +RABBITMQ_APT_KEY="https://www.rabbitmq.com/rabbitmq-signing-key-public.asc" +RABBITMQ_DEB="rabbitmq-server" + +RABBITMQ_USERNAME="testing" +RABBITMQ_PASSWORD="t3s71ng" +RABBITMQ_VHOST="/testing" + +REDIS_DEB="redis-server" +REDIS_CONF="/etc/redis/redis.conf" + +GIT_ROOT="${DEVEL_DIR}" + +GITHUB_ROOT="https://github.com/" +CELERY_GITHUB_USER="celery" +CELERY_USER="celery" +CELERY_GROUP="celery" +CELERY_DIR="${GIT_ROOT}/celery" +CELERY_FUNTESTS="${CELERY_DIR}/funtests/stress" +CELERY_CONFIG_SRC="${CELERY_FUNTESTS}/run/provision/celeryd-init.config" +CELERY_CONFIG_DST="/etc/default/celeryd" + + +die () { + echo $* + exit 1 +} + +# --- grent + +add_real_user () { + user_shell=${3:-/bin/bash} + addgroup $2 + echo creating user "$1 group='$2' shell='${user_shell}'" + echo | adduser -q "$1" --shell="${user_shell}" \ + --ingroup="$2" \ + --disabled-password 1>/dev/null 2>&1 + id "$1" || die "Not able to create user" +} + +for_user_makedir () { + mkdir "$2" + chown "$1" "$2" + chmod 0755 "$2" +} + +# --- directories + +make_directories () { + mkdir -p "${DEVEL_DIR}" +} + + +# --- apt + +apt_update() { + apt-get update +} + +add_apt_source () { + echo "deb $1" >> "${APT_SOURCES_LST}/rabbitmq.list" +} + +add_apt_key() { + "$WGET" --quiet -O - "$1" | apt-key add - +} + +apt_install () { + apt-get install -y "$1" +} + +# --- rabbitmq + +rabbitmq_add_user () { + "$RABBITMQCTL" add_user "$1" "$2" +} + +rabbitmq_add_vhost () { + "$RABBITMQCTL" add_vhost "$1" +} + +rabbitmq_set_perm () { + "$RABBITMQCTL" set_permissions -p $1 $2 '.*' '.*' '.*' +} + +install_rabbitmq() { + add_apt_source "${RABBITMQ_APT_URL} ${RABBITMQ_APT_VER}" + add_apt_key "${RABBITMQ_APT_KEY}" + apt_update + apt_install "${RABBITMQ_DEB}" + + rabbitmq_add_user "${RABBITMQ_USERNAME}" "${RABBITMQ_PASSWORD}" + rabbitmq_add_vhost "${RABBITMQ_VHOST}" + rabbitmq_set_perm "${RABBITMQ_VHOST}" "${RABBITMQ_USERNAME}" +} + +# --- redis + +restart_redis () { + service redis-server restart +} + + +install_redis () { + apt_install "${REDIS_DEB}" + sed -i 's/^bind .*$/#bind 127.0.0.1/' "${REDIS_CONF}" + restart_redis +} + +# --- git + +install_git () { + apt_install git +} + + +github_clone () { + (cd "${GIT_ROOT}"; git clone "${GITHUB_ROOT}/${1}/${2}") + chown "${CELERY_USER}" "${CELERY_DIR}" + ls -l /opt/devel/celery +} + + +# --- pip + +pip_install () { + pip install -U "$1" +} + +install_pip () { + apt_install python-setuptools + easy_install pip + pip_install virtualenv +} + +# --- celery + +restart_celery () { + service celeryd restart +} + + +install_celery_service () { + cp "${CELERY_DIR}/extra/generic-init.d/celeryd" /etc/init.d/ + chmod +x "/etc/init.d/celeryd" + update-rc.d celeryd defaults + cp "${CELERY_CONFIG_SRC}" "${CELERY_CONFIG_DEST}" + update-rc.d celeryd enable + restart_celery +} + +install_celery () { + pip_install celery + add_real_user "${CELERY_USER}" "${CELERY_GROUP}" + echo github_clone "'${CELERY_GITHUB_USER}'" "'celery'" + github_clone "${CELERY_GITHUB_USER}" celery + (cd ${CELERY_DIR}; pip install -r requirements/dev.txt); + (cd ${CELERY_DIR}; python setup.py develop); + install_celery_service +} + + +# --- MAIN + +provision () { + make_directories + apt_update + install_git + install_rabbitmq + install_redis + install_pip + install_celery +} + +provision From bf263955324e30254e115c76523df804acda8d8f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 19 Feb 2016 12:32:58 -0800 Subject: [PATCH 0997/1103] Some vagrant fixes --- .../stress/run/provision/celeryd-init.config | 2 +- funtests/stress/run/provision/provision.sh | 20 ++++++++++++++----- funtests/stress/stress/templates.py | 5 +++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/funtests/stress/run/provision/celeryd-init.config b/funtests/stress/run/provision/celeryd-init.config index 5659c3f8c..162ed0844 100644 --- a/funtests/stress/run/provision/celeryd-init.config +++ b/funtests/stress/run/provision/celeryd-init.config @@ -2,7 +2,7 @@ CELERYD_NODES="worker1" CELERY_BIN="/usr/local/bin/celery" CELERY_APP="stress" CELERYD_CHDIR="/opt/devel/celery/funtests/stress" -#CELERYD_OPTS="" +CELERYD_OPTS="-c10 --maxtasksperchild=256 -Z vagrant1" CELERYD_LOG_FILE="/var/log/celery/%n%I.log" CELERYD_PID_FILE="/var/run/celery/%n.pid" diff --git a/funtests/stress/run/provision/provision.sh b/funtests/stress/run/provision/provision.sh index 00ef85421..78e557abb 100644 --- a/funtests/stress/run/provision/provision.sh +++ b/funtests/stress/run/provision/provision.sh @@ -1,7 +1,5 @@ #!/bin/bash -echo "------------ HELLO ---------------" - APT_SOURCES_LST="/etc/apt/sources.list.d/" DEVEL_DIR="/opt/devel" @@ -56,12 +54,21 @@ for_user_makedir () { chmod 0755 "$2" } -# --- directories +# --- system make_directories () { mkdir -p "${DEVEL_DIR}" } +enable_bash_vi_mode () { + echo "set -o vi" >> /etc/bash.bashrc +} + +configure_system () { + make_directories + enable_bash_vi_mode +} + # --- apt @@ -143,6 +150,7 @@ install_pip () { apt_install python-setuptools easy_install pip pip_install virtualenv + pip_install setproctitle } # --- celery @@ -156,7 +164,8 @@ install_celery_service () { cp "${CELERY_DIR}/extra/generic-init.d/celeryd" /etc/init.d/ chmod +x "/etc/init.d/celeryd" update-rc.d celeryd defaults - cp "${CELERY_CONFIG_SRC}" "${CELERY_CONFIG_DEST}" + echo "cp \'${CELERY_CONFIG_SRC}\' \'${CELERY_CONFIG_DST}'" + cp "${CELERY_CONFIG_SRC}" "${CELERY_CONFIG_DST}" update-rc.d celeryd enable restart_celery } @@ -175,8 +184,9 @@ install_celery () { # --- MAIN provision () { - make_directories apt_update + configure_system + apt_install powertop install_git install_rabbitmq install_redis diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index 75118d06f..bc5cb7ff9 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -131,3 +131,8 @@ class sqs(default): @template() class proto1(default): task_protocol = 1 + + +@template() +class vagrant1(default): + broker_url = 'pyamqp://testing:t3s71ng@192.168.33.123//testing' From 86a07c7b001414a8545d35f9dd24f581a5c7ac37 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 19 Feb 2016 13:10:08 -0800 Subject: [PATCH 0998/1103] [Stress] setproctitle requires python headers --- funtests/stress/run/provision/provision.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/funtests/stress/run/provision/provision.sh b/funtests/stress/run/provision/provision.sh index 78e557abb..764db52f9 100644 --- a/funtests/stress/run/provision/provision.sh +++ b/funtests/stress/run/provision/provision.sh @@ -150,6 +150,7 @@ install_pip () { apt_install python-setuptools easy_install pip pip_install virtualenv + apt_install python-dev pip_install setproctitle } From 5e40cfbd773eb5e6e89c1c2d67ca3d5d979bba48 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 19 Feb 2016 15:00:08 -0800 Subject: [PATCH 0999/1103] [Stress] Copy stresstests to test independent of version --- .../stress/run/provision/celeryd-init.config | 2 +- funtests/stress/run/provision/provision.sh | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/funtests/stress/run/provision/celeryd-init.config b/funtests/stress/run/provision/celeryd-init.config index 162ed0844..8669040bc 100644 --- a/funtests/stress/run/provision/celeryd-init.config +++ b/funtests/stress/run/provision/celeryd-init.config @@ -1,7 +1,7 @@ CELERYD_NODES="worker1" CELERY_BIN="/usr/local/bin/celery" CELERY_APP="stress" -CELERYD_CHDIR="/opt/devel/celery/funtests/stress" +CELERYD_CHDIR="/opt/devel/stress" CELERYD_OPTS="-c10 --maxtasksperchild=256 -Z vagrant1" CELERYD_LOG_FILE="/var/log/celery/%n%I.log" CELERYD_PID_FILE="/var/run/celery/%n.pid" diff --git a/funtests/stress/run/provision/provision.sh b/funtests/stress/run/provision/provision.sh index 764db52f9..b9fc14b71 100644 --- a/funtests/stress/run/provision/provision.sh +++ b/funtests/stress/run/provision/provision.sh @@ -29,6 +29,7 @@ CELERY_DIR="${GIT_ROOT}/celery" CELERY_FUNTESTS="${CELERY_DIR}/funtests/stress" CELERY_CONFIG_SRC="${CELERY_FUNTESTS}/run/provision/celeryd-init.config" CELERY_CONFIG_DST="/etc/default/celeryd" +STRESS_DIR="${GIT_ROOT}/stress" die () { @@ -48,12 +49,6 @@ add_real_user () { id "$1" || die "Not able to create user" } -for_user_makedir () { - mkdir "$2" - chown "$1" "$2" - chmod 0755 "$2" -} - # --- system make_directories () { @@ -134,12 +129,11 @@ install_git () { github_clone () { - (cd "${GIT_ROOT}"; git clone "${GITHUB_ROOT}/${1}/${2}") + mkdir "${CELERY_DIR}" chown "${CELERY_USER}" "${CELERY_DIR}" - ls -l /opt/devel/celery + (cd "${GIT_ROOT}"; sudo -u celery git clone "${GITHUB_ROOT}/${1}/${2}") } - # --- pip pip_install () { @@ -181,6 +175,11 @@ install_celery () { install_celery_service } +install_stress () { + mkdir "${STRESS_DIR}" + chown "${CELERY_USER}" "${STRESS_DIR}" + cp -r "${CELERY_DIR}/funtests/stress/*" "${STRESS_DIR}" +} # --- MAIN @@ -193,6 +192,7 @@ provision () { install_redis install_pip install_celery + install_stress } provision From 3626b4c9b82a44fa78d4fbea3e3abc7349ffdbf3 Mon Sep 17 00:00:00 2001 From: Sebastian Kalinowski Date: Wed, 3 Feb 2016 15:28:43 +0100 Subject: [PATCH 1000/1103] Add nodes names to DuplicateNodenameWarning DuplicateNodenameWarning didn't have a list of duplicates included into the warning text. --- celery/app/control.py | 2 +- celery/tests/app/test_control.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/celery/app/control.py b/celery/app/control.py index 4444e0551..4b68f4b99 100644 --- a/celery/app/control.py +++ b/celery/app/control.py @@ -23,7 +23,7 @@ __all__ = ['Inspect', 'Control', 'flatten_reply'] W_DUPNODE = """\ -Received multiple replies from node name: {0!r}. +Received multiple replies from node {0}: {1}. Please make sure you give each node a unique nodename using the `-n` option.\ """ diff --git a/celery/tests/app/test_control.py b/celery/tests/app/test_control.py index ad4bc823a..125bc7682 100644 --- a/celery/tests/app/test_control.py +++ b/celery/tests/app/test_control.py @@ -7,6 +7,7 @@ from kombu.pidbox import Mailbox from celery.app import control +from celery.exceptions import DuplicateNodenameWarning from celery.utils import uuid from celery.tests.case import AppCase @@ -48,14 +49,15 @@ def test_flatten_reply(self): {'foo@example.com': {'hello': 20}}, {'bar@example.com': {'hello': 30}} ] - with warnings.catch_warnings(record=True) as w: + with self.assertWarns(DuplicateNodenameWarning) as w: nodes = control.flatten_reply(reply) - self.assertIn( - 'multiple replies', - str(w[-1].message), - ) - self.assertIn('foo@example.com', nodes) - self.assertIn('bar@example.com', nodes) + + self.assertIn( + 'Received multiple replies from node name: foo@example.com.', + str(w.warning) + ) + self.assertIn('foo@example.com', nodes) + self.assertIn('bar@example.com', nodes) class test_inspect(AppCase): From 5e2fe4b7364f05ee710766ccbe69469cbe064728 Mon Sep 17 00:00:00 2001 From: Jeff Widman Date: Tue, 23 Feb 2016 16:01:38 -0800 Subject: [PATCH 1001/1103] Fix typos in Changelog --- docs/whatsnew-3.1.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/whatsnew-3.1.rst b/docs/whatsnew-3.1.rst index 5a77ef926..a411e61da 100644 --- a/docs/whatsnew-3.1.rst +++ b/docs/whatsnew-3.1.rst @@ -218,7 +218,7 @@ implementation. - Rare race conditions fixed - Most of these bugs were never reported to us, but was discovered while + Most of these bugs were never reported to us, but were discovered while running the new stress test suite. Caveats @@ -314,7 +314,7 @@ but if you would like to experiment with it you should know that: app.config_from_object('django.conf:settings') Neither will it automatically traverse your installed apps to find task - modules, but this still available as an option you must enable: + modules. If you want this behavior, you must explictly pass a list of Django instances to the Celery app: .. code-block:: python @@ -334,7 +334,7 @@ but if you would like to experiment with it you should know that: guide `. To get started with the new API you should first read the :ref:`first-steps` -tutorial, and then you should read the Django specific instructions in +tutorial, and then you should read the Django-specific instructions in :ref:`django-first-steps`. The fixes and improvements applied by the django-celery library are now @@ -375,7 +375,7 @@ but starting with this version that field is also used to order them. Also, events now record timezone information by including a new ``utcoffset`` field in the event message. This is a signed integer telling the difference from UTC time in hours, -so e.g. an even sent from the Europe/London timezone in daylight savings +so e.g. an event sent from the Europe/London timezone in daylight savings time will have an offset of 1. :class:`@events.Receiver` will automatically convert the timestamps @@ -389,8 +389,8 @@ to the local timezone. starts. If all of the workers are shutdown the clock value will be lost - and reset to 0, to protect against this you should specify - a :option:`--statedb` so that the worker can persist the clock + and reset to 0. To protect against this, you should specify + :option:`--statedb` so that the worker can persist the clock value at shutdown. You may notice that the logical clock is an integer value and @@ -499,8 +499,8 @@ and you can write extensions that take advantage of this already. Some ideas include consensus protocols, reroute task to best worker (based on resource usage or data locality) or restarting workers when they crash. -We believe that this is a small addition but one that really opens -up for amazing possibilities. +We believe that although this is a small addition, it opens +amazing possibilities. You can disable this bootstep using the ``--without-gossip`` argument. From 566ea49d5635c97807235ee8e2201c64576e2a06 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 25 Feb 2016 12:54:22 -0800 Subject: [PATCH 1002/1103] [Stress] Small fix for provision script --- funtests/stress/run/provision/provision.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/funtests/stress/run/provision/provision.sh b/funtests/stress/run/provision/provision.sh index b9fc14b71..d4de824d2 100644 --- a/funtests/stress/run/provision/provision.sh +++ b/funtests/stress/run/provision/provision.sh @@ -172,13 +172,12 @@ install_celery () { github_clone "${CELERY_GITHUB_USER}" celery (cd ${CELERY_DIR}; pip install -r requirements/dev.txt); (cd ${CELERY_DIR}; python setup.py develop); - install_celery_service } install_stress () { mkdir "${STRESS_DIR}" chown "${CELERY_USER}" "${STRESS_DIR}" - cp -r "${CELERY_DIR}/funtests/stress/*" "${STRESS_DIR}" + cp -r ${CELERY_DIR}/funtests/stress/* "${STRESS_DIR}/" } # --- MAIN @@ -193,6 +192,7 @@ provision () { install_pip install_celery install_stress + install_celery_service } provision From 9e31b2790c1fb3cc148591c91b85d9e201bffc0b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 17 Dec 2015 11:59:47 -0800 Subject: [PATCH 1003/1103] [async result] Callback based result backends (related to Issue #2529) --- celery/backends/amqp.py | 189 +++++++++++++++++++++++++++++----------- celery/backends/base.py | 14 +++ celery/result.py | 94 ++++++++++++++------ funtests/stress/t.py | 30 +++++++ 4 files changed, 248 insertions(+), 79 deletions(-) create mode 100644 funtests/stress/t.py diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index 853200bc3..65fddaf1f 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -46,12 +46,122 @@ class NoCacheQueue(Queue): can_cache_declaration = False +class ResultConsumer(object): + Consumer = Consumer + + def __init__(self, backend, app, accept, pending_results): + self.backend = backend + self.app = app + self.accept = accept + self._pending_results = pending_results + self._consumer = None + self._conn = None + self.on_message = None + self.bucket = None + + def consume(self, task_id, timeout=None, no_ack=True, on_interval=None): + wait = self.drain_events + with self.app.pool.acquire_channel(block=True) as (conn, channel): + binding = self.backend._create_binding(task_id) + with self.Consumer(channel, binding, + no_ack=no_ack, accept=self.accept) as consumer: + while 1: + try: + return wait( + conn, consumer, timeout, on_interval)[task_id] + except KeyError: + continue + + def wait_for_pending(self, result, + callback=None, propagate=True, **kwargs): + for _ in self._wait_for_pending(result, **kwargs): + pass + return result.maybe_throw(callback=callback, propagate=propagate) + + def _wait_for_pending(self, result, timeout=None, interval=0.5, + no_ack=True, on_interval=None, callback=None, + on_message=None, propagate=True): + prev_on_m, self.on_message = self.on_message, on_message + try: + for _ in self.drain_events_until( + result.on_ready, timeout=timeout, + on_interval=on_interval): + yield + except socket.timeout: + raise TimeoutError('The operation timed out.') + finally: + self.on_message = prev_on_m + + def collect_for_pending(self, result, bucket=None, **kwargs): + prev_bucket, self.bucket = self.bucket, bucket + try: + for _ in self._wait_for_pending(result, **kwargs): + yield + finally: + self.bucket = prev_bucket + + def start(self, initial_queue, no_ack=True): + self._conn = self.app.connection() + self._consumer = self.Consumer( + self._conn.default_channel, [initial_queue], + callbacks=[self.on_state_change], no_ack=no_ack, + accept=self.accept) + self._consumer.consume() + + def stop(self): + try: + self._consumer.cancel() + finally: + self._connection.close() + + def consume_from(self, queue): + if self._consumer is None: + return self.start(queue) + if not self._consumer.consuming_from(queue): + self._consumer.add_queue(queue) + self._consumer.consume() + + def cancel_for(self, queue): + self._consumer.cancel_by_queue(queue) + + def on_state_change(self, meta, message): + if self.on_message: + self.on_message(meta) + if meta['status'] in states.READY_STATES: + try: + result = self._pending_results[meta['task_id']] + except KeyError: + return + result._maybe_set_cache(meta) + if self.bucket is not None: + self.bucket.append(result) + + def drain_events_until(self, p, timeout=None, on_interval=None, + monotonic=monotonic, wait=None): + wait = wait or self._conn.drain_events + time_start = monotonic() + + while 1: + # Total time spent may exceed a single call to wait() + if timeout and monotonic() - time_start >= timeout: + raise socket.timeout() + try: + yield wait(timeout=1) + except socket.timeout: + pass + if on_interval: + on_interval() + if p.ready: # got event on the wanted channel. + break + + class AMQPBackend(BaseBackend): """Publishes results by sending messages.""" Exchange = Exchange Queue = NoCacheQueue Consumer = Consumer Producer = Producer + ResultConsumer = ResultConsumer BacklogLimitExceeded = BacklogLimitExceeded @@ -83,6 +193,8 @@ def __init__(self, app, connection=None, exchange=None, exchange_type=None, self.queue_arguments = dictfilter({ 'x-expires': maybe_s_to_ms(self.expires), }) + self.result_consumer = self.ResultConsumer( + self, self.app, self.accept, self._pending_results) def _create_exchange(self, name, type='direct', delivery_mode=2): return self.Exchange(name=name, @@ -136,22 +248,6 @@ def store_result(self, task_id, result, state, def on_reply_declare(self, task_id): return [self._create_binding(task_id)] - def wait_for(self, task_id, timeout=None, cache=True, - no_ack=True, on_interval=None, - READY_STATES=states.READY_STATES, - PROPAGATE_STATES=states.PROPAGATE_STATES, - **kwargs): - cached_meta = self._cache.get(task_id) - if cache and cached_meta and \ - cached_meta['status'] in READY_STATES: - return cached_meta - else: - try: - return self.consume(task_id, timeout=timeout, no_ack=no_ack, - on_interval=on_interval) - except socket.timeout: - raise TimeoutError('The operation timed out.') - def get_task_meta(self, task_id, backlog_limit=1000): # Polling and using basic_get with self.app.pool.acquire_channel(block=True) as (_, channel): @@ -189,50 +285,37 @@ def get_task_meta(self, task_id, backlog_limit=1000): return {'status': states.PENDING, 'result': None} poll = get_task_meta # XXX compat - def drain_events(self, connection, consumer, - timeout=None, on_interval=None, now=monotonic, wait=None): - wait = wait or connection.drain_events - results = {} + def wait_for_pending(self, result, timeout=None, interval=0.5, + no_ack=True, on_interval=None, on_message=None, + callback=None, propagate=True): + return self.result_consumer.wait_for_pending( + result, timeout=timeout, interval=interval, + no_ack=no_ack, on_interval=on_interval, + callback=callback, on_message=on_message, propagate=propagate, + ) - def callback(meta, message): - if meta['status'] in states.READY_STATES: - results[meta['task_id']] = self.meta_from_decoded(meta) + def collect_for_pending(self, result, bucket=None, timeout=None, + interval=0.5, no_ack=True, on_interval=None, + on_message=None, callback=None, propagate=True): + return self.result_consumer.collect_for_pending( + result, bucket=bucket, timeout=timeout, interval=interval, + no_ack=no_ack, on_interval=on_interval, + callback=callback, on_message=on_message, propagate=propagate, + ) - consumer.callbacks[:] = [callback] - time_start = now() + def add_pending_result(self, result): + if result.id not in self._pending_results: + self._pending_results[result.id] = result + self.result_consumer.consume_from(self._create_binding(result.id)) - while 1: - # Total time spent may exceed a single call to wait() - if timeout and now() - time_start >= timeout: - raise socket.timeout() - try: - wait(timeout=1) - except socket.timeout: - pass - if on_interval: - on_interval() - if results: # got event on the wanted channel. - break - self._cache.update(results) - return results - - def consume(self, task_id, timeout=None, no_ack=True, on_interval=None): - wait = self.drain_events - with self.app.pool.acquire_channel(block=True) as (conn, channel): - binding = self._create_binding(task_id) - with self.Consumer(channel, binding, - no_ack=no_ack, accept=self.accept) as consumer: - while 1: - try: - return wait( - conn, consumer, timeout, on_interval)[task_id] - except KeyError: - continue + def remove_pending_result(self, result): + self._pending_results.pop(result.id, None) + # XXX cancel queue after result consumed def _many_bindings(self, ids): return [self._create_binding(task_id) for task_id in ids] - def get_many(self, task_ids, timeout=None, no_ack=True, + def xxx_get_many(self, task_ids, timeout=None, no_ack=True, on_message=None, on_interval=None, now=monotonic, getfields=itemgetter('status', 'task_id'), READY_STATES=states.READY_STATES, diff --git a/celery/backends/base.py b/celery/backends/base.py index 8a30ec044..feeeea375 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -107,6 +107,7 @@ def __init__(self, app, self.accept = prepare_accept_content( conf.accept_content if accept is None else accept, ) + self._pending_results = {} def mark_as_started(self, task_id, **meta): """Mark a task as started""" @@ -221,6 +222,19 @@ def decode(self, payload): content_encoding=self.content_encoding, accept=self.accept) + def wait_for_pending(self, result, timeout=None, interval=0.5, + no_ack=True, on_interval=None, callback=None, + propagate=True): + meta = self.wait_for( + result.id, timeout=timeout, + interval=interval, + on_interval=on_interval, + no_ack=no_ack, + ) + if meta: + result._maybe_set_cache(meta) + return result.maybe_throw(propagate=propagate, callback=callback) + def wait_for(self, task_id, timeout=None, interval=0.5, no_ack=True, on_interval=None): """Wait for task and return its result. diff --git a/celery/result.py b/celery/result.py index 1dfbb69df..5e5ce6f16 100644 --- a/celery/result.py +++ b/celery/result.py @@ -14,7 +14,7 @@ from contextlib import contextmanager from copy import copy -from amqp import promise +from amqp.promise import Thenable, barrier, promise from kombu.utils import cached_property from . import current_app @@ -86,8 +86,17 @@ def __init__(self, id, backend=None, self.id = id self.backend = backend or self.app.backend self.parent = parent + self.on_ready = promise(self._on_fulfilled) self._cache = None + def then(self, callback, on_error=None): + self.backend.add_pending_result(self) + return self.on_ready.then(callback, on_error) + + def _on_fulfilled(self, result): + self.backend.remove_pending_result(self) + return result + def as_tuple(self): parent = self.parent return (self.id, parent and parent.as_tuple()), None @@ -159,28 +168,22 @@ def get(self, timeout=None, propagate=True, interval=0.5, if self._cache: if propagate: - self.maybe_reraise() + self.maybe_throw() return self.result - meta = self.backend.wait_for( - self.id, timeout=timeout, + self.backend.add_pending_result(self) + return self.backend.wait_for_pending( + self, timeout=timeout, interval=interval, on_interval=_on_interval, no_ack=no_ack, + propagate=propagate, ) - if meta: - self._maybe_set_cache(meta) - state = meta['status'] - if state in PROPAGATE_STATES and propagate: - raise meta['result'] - if callback is not None: - callback(self.id, meta['result']) - return meta['result'] wait = get # deprecated alias to :meth:`get`. def _maybe_reraise_parent_error(self): for node in reversed(list(self._parents())): - node.maybe_reraise() + node.maybe_throw() def _parents(self): node = self.parent @@ -268,9 +271,17 @@ def failed(self): """Returns :const:`True` if the task failed.""" return self.state == states.FAILURE - def maybe_reraise(self): - if self.state in states.PROPAGATE_STATES: - raise self.result + def throw(self, *args, **kwargs): + self.on_ready.throw(*args, **kwargs) + + def maybe_throw(self, propagate=True, callback=None): + cache = self._get_task_meta() if self._cache is None else self._cache + state, value = cache['status'], cache['result'] + if state in states.PROPAGATE_STATES and propagate: + self.throw(value) + if callback is not None: + callback(self.id, value) + return value def build_graph(self, intermediate=False, formatter=None): graph = DependencyGraph( @@ -333,8 +344,10 @@ def children(self): def _maybe_set_cache(self, meta): if meta: state = meta['status'] - if state == states.SUCCESS or state in states.PROPAGATE_STATES: - return self._set_cache(meta) + if state in states.READY_STATES: + d = self._set_cache(self.backend.meta_from_decoded(meta)) + self.on_ready(self) + return d return meta def _get_task_meta(self): @@ -405,6 +418,7 @@ def task_id(self): @task_id.setter # noqa def task_id(self, id): self.id = id +Thenable.register(AsyncResult) class ResultSet(ResultBase): @@ -421,6 +435,7 @@ class ResultSet(ResultBase): def __init__(self, results, app=None, **kwargs): self._app = app self.results = results + self.on_ready = barrier(self.results, (self,), callback=self._on_ready) def add(self, result): """Add :class:`AsyncResult` as a new member of the set. @@ -430,6 +445,10 @@ def add(self, result): """ if result not in self.results: self.results.append(result) + self.ready.add(result) + + def _on_ready(self, result): + self.backend.remove_pending_result(result) def remove(self, result): """Remove result from the set; it must be a member. @@ -482,9 +501,9 @@ def failed(self): """ return any(result.failed() for result in self.results) - def maybe_reraise(self): + def maybe_throw(self, callback=None, propagate=True): for result in self.results: - result.maybe_reraise() + result.maybe_throw(callback=callback, propagate=propagate) def waiting(self): """Are any of the tasks incomplete? @@ -655,6 +674,12 @@ def join(self, timeout=None, propagate=True, interval=0.5, results.append(value) return results + def then(self, callback, on_error=None): + for result in self.results: + self.backend.add_pending_result(result) + result.on_ready.then(self.on_ready) + return self.on_ready.then(callback, on_error) + def iter_native(self, timeout=None, interval=0.5, no_ack=True, on_message=None, on_interval=None): """Backend optimized version of :meth:`iterate`. @@ -670,12 +695,21 @@ def iter_native(self, timeout=None, interval=0.5, no_ack=True, """ results = self.results if not results: - return iter([]) - return self.backend.get_many( - {r.id for r in results}, - timeout=timeout, interval=interval, no_ack=no_ack, - on_message=on_message, on_interval=on_interval, - ) + raise StopIteration() + ids = set() + for result in self.results: + self.backend.add_pending_result(result) + ids.add(result.id) + bucket = deque() + for _ in self.backend.collect_for_pending( + self, + bucket=bucket, + timeout=timeout, interval=interval, no_ack=no_ack, + on_message=on_message, on_interval=on_interval): + while bucket: + result = bucket.popleft() + if result.id in ids: + yield result.id, result._cache def join_native(self, timeout=None, propagate=True, interval=0.5, callback=None, no_ack=True, @@ -749,6 +783,7 @@ def app(self, app): # noqa @property def backend(self): return self.app.backend if self.app else self.results[0].backend +Thenable.register(ResultSet) class GroupResult(ResultSet): @@ -822,6 +857,7 @@ def restore(self, id, backend=None): return ( backend or (self.app.backend if self.app else current_app.backend) ).restore_group(id) +Thenable.register(ResultSet) class EagerResult(AsyncResult): @@ -832,6 +868,11 @@ def __init__(self, id, ret_value, state, traceback=None): self._result = ret_value self._state = state self._traceback = traceback + self.on_ready = promise() + self.on_ready() + + def then(self, callback, on_error=None): + return self.on_ready.then(callback, on_error) def _get_task_meta(self): return {'task_id': self.id, 'result': self._result, 'status': @@ -887,6 +928,7 @@ def traceback(self): @property def supports_native_join(self): return False +Thenable.register(EagerResult) def result_from_tuple(r, app=None): diff --git a/funtests/stress/t.py b/funtests/stress/t.py new file mode 100644 index 000000000..37688936b --- /dev/null +++ b/funtests/stress/t.py @@ -0,0 +1,30 @@ +from celery import group +import socket +from stress.app import add, raising + +def on_ready(result): + print('RESULT: %r' % (result,)) + +def test(): + group(add.s(i, i) for i in range(10)).delay().then(on_ready) + + p = group(add.s(i, i) for i in range(10)).delay() + print(p.get(timeout=5)) + + p = add.delay(2, 2) + print(p.get(timeout=5)) + p = add.delay(2, 2) + print(p.get(timeout=5)) + p = add.delay(2, 2) + print(p.get(timeout=5)) + p = add.delay(2, 2) + print(p.get(timeout=5)) + p = raising.delay() + try: + print(p.get(timeout=5)) + except Exception as exc: + print('raised: %r' % (exc),) + + +for i in range(100): + test() From 3ef4f0cdb98de17cc91ccb10e014d483f575b3e2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 23 Dec 2015 14:16:02 -0800 Subject: [PATCH 1004/1103] [async result] Work in progress on Async result backend --- celery/backends/amqp.py | 208 ++++------------------- celery/backends/base.py | 288 +++++++++++++++++++++++++++----- celery/backends/rpc.py | 3 + celery/canvas.py | 13 +- celery/result.py | 40 ++--- funtests/stress/stress/suite.py | 2 +- funtests/stress/t.py | 62 ++++--- 7 files changed, 352 insertions(+), 264 deletions(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index 65fddaf1f..44d4806d6 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -10,21 +10,16 @@ """ from __future__ import absolute_import -import socket - -from collections import deque -from operator import itemgetter - from kombu import Exchange, Queue, Producer, Consumer +from kombu.utils import register_after_fork from celery import states -from celery.exceptions import TimeoutError -from celery.five import range, monotonic +from celery.five import range from celery.utils.functional import dictfilter from celery.utils.log import get_logger from celery.utils.timeutils import maybe_s_to_ms -from .base import BaseBackend +from .base import AsyncBackendMixin, Backend, BaseResultConsumer __all__ = ['BacklogLimitExceeded', 'AMQPBackend'] @@ -42,78 +37,45 @@ def repair_uuid(s): return '%s-%s-%s-%s-%s' % (s[:8], s[8:12], s[12:16], s[16:20], s[20:]) +def _on_after_fork_cleanup_backend(backend): + backend._after_fork() + + class NoCacheQueue(Queue): can_cache_declaration = False -class ResultConsumer(object): +class ResultConsumer(BaseResultConsumer): Consumer = Consumer - def __init__(self, backend, app, accept, pending_results): - self.backend = backend - self.app = app - self.accept = accept - self._pending_results = pending_results + def __init__(self, *args, **kwargs): + super(ResultConsumer, self).__init__(*args, **kwargs) + self._connection = None self._consumer = None - self._conn = None - self.on_message = None - self.bucket = None - - def consume(self, task_id, timeout=None, no_ack=True, on_interval=None): - wait = self.drain_events - with self.app.pool.acquire_channel(block=True) as (conn, channel): - binding = self.backend._create_binding(task_id) - with self.Consumer(channel, binding, - no_ack=no_ack, accept=self.accept) as consumer: - while 1: - try: - return wait( - conn, consumer, timeout, on_interval)[task_id] - except KeyError: - continue - - def wait_for_pending(self, result, - callback=None, propagate=True, **kwargs): - for _ in self._wait_for_pending(result, **kwargs): - pass - return result.maybe_throw(callback=callback, propagate=propagate) - - def _wait_for_pending(self, result, timeout=None, interval=0.5, - no_ack=True, on_interval=None, callback=None, - on_message=None, propagate=True): - prev_on_m, self.on_message = self.on_message, on_message - try: - for _ in self.drain_events_until( - result.on_ready, timeout=timeout, - on_interval=on_interval): - yield - except socket.timeout: - raise TimeoutError('The operation timed out.') - finally: - self.on_message = prev_on_m - - def collect_for_pending(self, result, bucket=None, **kwargs): - prev_bucket, self.bucket = self.bucket, bucket - try: - for _ in self._wait_for_pending(result, **kwargs): - yield - finally: - self.bucket = prev_bucket def start(self, initial_queue, no_ack=True): - self._conn = self.app.connection() + self._connection = self.app.connection() self._consumer = self.Consumer( - self._conn.default_channel, [initial_queue], + self._connection.default_channel, [initial_queue], callbacks=[self.on_state_change], no_ack=no_ack, accept=self.accept) self._consumer.consume() + def drain_events(self, timeout=None): + return self._connection.drain_events(timeout=timeout) + def stop(self): try: self._consumer.cancel() finally: self._connection.close() + def on_after_fork(self): + self._consumer = None + if self._connection is not None: + self._connection.collect() + self._connection = None + def consume_from(self, queue): if self._consumer is None: return self.start(queue) @@ -122,40 +84,10 @@ def consume_from(self, queue): self._consumer.consume() def cancel_for(self, queue): - self._consumer.cancel_by_queue(queue) - - def on_state_change(self, meta, message): - if self.on_message: - self.on_message(meta) - if meta['status'] in states.READY_STATES: - try: - result = self._pending_results[meta['task_id']] - except KeyError: - return - result._maybe_set_cache(meta) - if self.bucket is not None: - self.bucket.append(result) - - def drain_events_until(self, p, timeout=None, on_interval=None, - monotonic=monotonic, wait=None): - wait = wait or self._conn.drain_events - time_start = monotonic() - - while 1: - # Total time spent may exceed a single call to wait() - if timeout and monotonic() - time_start >= timeout: - raise socket.timeout() - try: - yield wait(timeout=1) - except socket.timeout: - pass - if on_interval: - on_interval() - if p.ready: # got event on the wanted channel. - break - - -class AMQPBackend(BaseBackend): + self._consumer.cancel_by_queue(queue.name) + + +class AMQPBackend(Backend, AsyncBackendMixin): """Publishes results by sending messages.""" Exchange = Exchange Queue = NoCacheQueue @@ -195,6 +127,15 @@ def __init__(self, app, connection=None, exchange=None, exchange_type=None, }) self.result_consumer = self.ResultConsumer( self, self.app, self.accept, self._pending_results) + if register_after_fork is not None: + register_after_fork(self, _on_after_fork_cleanup_backend) + + def _after_fork(self): + self._pending_results.clear() + self.result_consumer._after_fork() + + def on_result_fulfilled(self, result): + self.result_consumer.cancel_for(self._create_binding(result.id)) def _create_exchange(self, name, type='direct', delivery_mode=2): return self.Exchange(name=name, @@ -285,85 +226,6 @@ def get_task_meta(self, task_id, backlog_limit=1000): return {'status': states.PENDING, 'result': None} poll = get_task_meta # XXX compat - def wait_for_pending(self, result, timeout=None, interval=0.5, - no_ack=True, on_interval=None, on_message=None, - callback=None, propagate=True): - return self.result_consumer.wait_for_pending( - result, timeout=timeout, interval=interval, - no_ack=no_ack, on_interval=on_interval, - callback=callback, on_message=on_message, propagate=propagate, - ) - - def collect_for_pending(self, result, bucket=None, timeout=None, - interval=0.5, no_ack=True, on_interval=None, - on_message=None, callback=None, propagate=True): - return self.result_consumer.collect_for_pending( - result, bucket=bucket, timeout=timeout, interval=interval, - no_ack=no_ack, on_interval=on_interval, - callback=callback, on_message=on_message, propagate=propagate, - ) - - def add_pending_result(self, result): - if result.id not in self._pending_results: - self._pending_results[result.id] = result - self.result_consumer.consume_from(self._create_binding(result.id)) - - def remove_pending_result(self, result): - self._pending_results.pop(result.id, None) - # XXX cancel queue after result consumed - - def _many_bindings(self, ids): - return [self._create_binding(task_id) for task_id in ids] - - def xxx_get_many(self, task_ids, timeout=None, no_ack=True, - on_message=None, on_interval=None, - now=monotonic, getfields=itemgetter('status', 'task_id'), - READY_STATES=states.READY_STATES, - PROPAGATE_STATES=states.PROPAGATE_STATES, **kwargs): - with self.app.pool.acquire_channel(block=True) as (conn, channel): - ids = set(task_ids) - cached_ids = set() - mark_cached = cached_ids.add - for task_id in ids: - try: - cached = self._cache[task_id] - except KeyError: - pass - else: - if cached['status'] in READY_STATES: - yield task_id, cached - mark_cached(task_id) - ids.difference_update(cached_ids) - results = deque() - push_result = results.append - push_cache = self._cache.__setitem__ - decode_result = self.meta_from_decoded - - def _on_message(message): - body = decode_result(message.decode()) - if on_message is not None: - on_message(body) - state, uid = getfields(body) - if state in READY_STATES: - push_result(body) \ - if uid in task_ids else push_cache(uid, body) - - bindings = self._many_bindings(task_ids) - with self.Consumer(channel, bindings, on_message=_on_message, - accept=self.accept, no_ack=no_ack): - wait = conn.drain_events - popleft = results.popleft - while ids: - wait(timeout=timeout) - while results: - state = popleft() - task_id = state['task_id'] - ids.discard(task_id) - push_cache(task_id, state) - yield task_id, state - if on_interval: - on_interval() - def reload_task_result(self, task_id): raise NotImplementedError( 'reload_task_result is not supported by this backend.') diff --git a/celery/backends/base.py b/celery/backends/base.py index feeeea375..14ef7a247 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -13,23 +13,27 @@ """ from __future__ import absolute_import -import time +import socket import sys +import time +from collections import deque from datetime import timedelta +from weakref import WeakKeyDictionary from billiard.einfo import ExceptionInfo from kombu.serialization import ( dumps, loads, prepare_accept_content, registry as serializer_registry, ) +from kombu.syn import detect_environment from kombu.utils.encoding import bytes_to_str, ensure_bytes, from_utf8 from celery import states from celery import current_app, group, maybe_signature from celery.app import current_task from celery.exceptions import ChordError, TimeoutError, TaskRevokedError -from celery.five import items +from celery.five import items, monotonic from celery.result import ( GroupResult, ResultBase, allow_join_result, result_from_tuple, ) @@ -61,7 +65,7 @@ def ignore(self, *a, **kw): __setitem__ = update = setdefault = ignore -class BaseBackend(object): +class Backend(object): READY_STATES = states.READY_STATES UNREADY_STATES = states.UNREADY_STATES EXCEPTION_STATES = states.EXCEPTION_STATES @@ -222,46 +226,6 @@ def decode(self, payload): content_encoding=self.content_encoding, accept=self.accept) - def wait_for_pending(self, result, timeout=None, interval=0.5, - no_ack=True, on_interval=None, callback=None, - propagate=True): - meta = self.wait_for( - result.id, timeout=timeout, - interval=interval, - on_interval=on_interval, - no_ack=no_ack, - ) - if meta: - result._maybe_set_cache(meta) - return result.maybe_throw(propagate=propagate, callback=callback) - - def wait_for(self, task_id, - timeout=None, interval=0.5, no_ack=True, on_interval=None): - """Wait for task and return its result. - - If the task raises an exception, this exception - will be re-raised by :func:`wait_for`. - - If `timeout` is not :const:`None`, this raises the - :class:`celery.exceptions.TimeoutError` exception if the operation - takes longer than `timeout` seconds. - - """ - - time_elapsed = 0.0 - - while 1: - meta = self.get_task_meta(task_id) - if meta['status'] in states.READY_STATES: - return meta - if on_interval: - on_interval() - # avoid hammering the CPU checking status. - time.sleep(interval) - time_elapsed += interval - if timeout and time_elapsed >= timeout: - raise TimeoutError('The operation timed out.') - def prepare_expires(self, value, type=None): if value is None: value = self.app.conf.result_expires @@ -406,9 +370,247 @@ def current_task_children(self, request=None): def __reduce__(self, args=(), kwargs={}): return (unpickle_backend, (self.__class__, args, kwargs)) + + +class SyncBackendMixin(object): + + def iter_native(self, result, timeout=None, interval=0.5, no_ack=True, + on_message=None, on_interval=None): + results = result.results + if not results: + return iter([]) + return self.get_many( + {r.id for r in results}, + timeout=timeout, interval=interval, no_ack=no_ack, + on_message=on_message, on_interval=on_interval, + ) + + def wait_for_pending(self, result, timeout=None, interval=0.5, + no_ack=True, on_interval=None, callback=None, + propagate=True): + meta = self.wait_for( + result.id, timeout=timeout, + interval=interval, + on_interval=on_interval, + no_ack=no_ack, + ) + if meta: + result._maybe_set_cache(meta) + return result.maybe_throw(propagate=propagate, callback=callback) + + def wait_for(self, task_id, + timeout=None, interval=0.5, no_ack=True, on_interval=None): + """Wait for task and return its result. + + If the task raises an exception, this exception + will be re-raised by :func:`wait_for`. + + If `timeout` is not :const:`None`, this raises the + :class:`celery.exceptions.TimeoutError` exception if the operation + takes longer than `timeout` seconds. + + """ + + time_elapsed = 0.0 + + while 1: + meta = self.get_task_meta(task_id) + if meta['status'] in states.READY_STATES: + return meta + if on_interval: + on_interval() + # avoid hammering the CPU checking status. + time.sleep(interval) + time_elapsed += interval + if timeout and time_elapsed >= timeout: + raise TimeoutError('The operation timed out.') + + def add_pending_result(self, result): + return result + + def remove_pending_result(self, result): + return result + + +class AsyncBackendMixin(object): + + def _collect_into(self, result, bucket): + self.result_consumer.buckets[result] = bucket + + def iter_native(self, result, timeout=None, interval=0.5, no_ack=True, + on_message=None, on_interval=None): + results = result.results + if not results: + raise StopIteration() + + bucket = deque() + for result in results: + self._collect_into(result, bucket) + + for _ in self._wait_for_pending( + result, + timeout=timeout, interval=interval, no_ack=no_ack, + on_message=on_message, on_interval=on_interval): + while bucket: + result = bucket.popleft() + yield result.id, result._cache + while bucket: + result = bucket.popleft() + yield result.id, result._cache + + def add_pending_result(self, result): + if result.id not in self._pending_results: + self._pending_results[result.id] = result + self.result_consumer.consume_from(self._create_binding(result.id)) + return result + + def remove_pending_result(self, result): + self._pending_results.pop(result.id, None) + self.on_result_fulfilled(result) + return result + + def on_result_fulfilled(self, result): + pass + + def wait_for_pending(self, result, + callback=None, propagate=True, **kwargs): + for _ in self._wait_for_pending(result, **kwargs): + pass + return result.maybe_throw(callback=callback, propagate=propagate) + + def _wait_for_pending(self, result, timeout=None, interval=0.5, + no_ack=True, on_interval=None, on_message=None, + callback=None, propagate=True): + return self.result_consumer._wait_for_pending( + result, timeout=timeout, interval=interval, + no_ack=no_ack, on_interval=on_interval, + callback=callback, on_message=on_message, propagate=propagate, + ) + + +class BaseBackend(Backend, SyncBackendMixin): + pass BaseDictBackend = BaseBackend # XXX compat + +class Drainer(object): + + def __init__(self, result_consumer): + self.result_consumer = result_consumer + + def drain_events_until(self, p, timeout=None, on_interval=None, + monotonic=monotonic, wait=None): + wait = wait or self.result_consumer.drain_events + time_start = monotonic() + + while 1: + # Total time spent may exceed a single call to wait() + if timeout and monotonic() - time_start >= timeout: + raise socket.timeout() + try: + yield self.wait_for(p, wait, timeout=1) + except socket.timeout: + pass + if on_interval: + on_interval() + if p.ready: # got event on the wanted channel. + break + + def wait_for(self, p, wait, timeout=None): + wait(timeout=timeout) + + +class EventletDrainer(Drainer): + _g = None + _stopped = False + + def run(self): + while not self._stopped: + try: + print("DRAINING!!!!!!!!!!!!!!!!") + self.result_consumer.drain_events(timeout=10) + except socket.timeout: + pass + + def start(self): + from eventlet import spawn + if self._g is None: + self._g = spawn(self.run) + + def stop(self): + self._stopped = True + + def wait_for(self, p, wait, timeout=None): + if self._g is None: + self.start() + if not p.ready: + time.sleep(0) + + +drainers = {'default': Drainer, 'eventlet': EventletDrainer} + +class BaseResultConsumer(object): + + def __init__(self, backend, app, accept, pending_results): + self.backend = backend + self.app = app + self.accept = accept + self._pending_results = pending_results + self.on_message = None + self.buckets = WeakKeyDictionary() + self.drainer = drainers[detect_environment()](self) + + def drain_events(self, timeout=None): + raise NotImplementedError('subclass responsibility') + + def _after_fork(self): + self.bucket.clear() + self.buckets = WeakKeyDictionary() + self.on_message = None + self.on_after_fork() + + def on_after_fork(self): + pass + + def drain_events_until(self, p, timeout=None, on_interval=None): + return self.drainer.drain_events_until( + p, timeout=timeout, on_interval=on_interval) + + def _wait_for_pending(self, result, timeout=None, interval=0.5, + no_ack=True, on_interval=None, callback=None, + on_message=None, propagate=True): + prev_on_m, self.on_message = self.on_message, on_message + try: + for _ in self.drain_events_until( + result.on_ready, timeout=timeout, + on_interval=on_interval): + yield + time.sleep(0) + except socket.timeout: + raise TimeoutError('The operation timed out.') + finally: + self.on_message = prev_on_m + + def on_state_change(self, meta, message): + if self.on_message: + self.on_message(meta) + if meta['status'] in states.READY_STATES: + try: + result = self._pending_results[meta['task_id']] + except KeyError: + return + result._maybe_set_cache(meta) + buckets = self.buckets + try: + buckets[result].append(result) + buckets.pop(result) + except KeyError: + pass + time.sleep(0) + + + class KeyValueStoreBackend(BaseBackend): key_t = ensure_bytes task_keyprefix = 'celery-task-meta-' diff --git a/celery/backends/rpc.py b/celery/backends/rpc.py index c78153622..ee282eed1 100644 --- a/celery/backends/rpc.py +++ b/celery/backends/rpc.py @@ -54,6 +54,9 @@ def destination_for(self, task_id, request): def on_reply_declare(self, task_id): pass + def on_result_fulfilled(self, result): + pass + @property def binding(self): return self.Queue(self.oid, self.exchange, self.oid, diff --git a/celery/canvas.py b/celery/canvas.py index e7e18891f..e44ea497d 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -20,6 +20,7 @@ from operator import itemgetter from itertools import chain as _chain +from amqp.promise import barrier from kombu.utils import cached_property, fxrange, reprcall, uuid from celery._state import current_app @@ -730,7 +731,7 @@ def _prepared(self, tasks, partial_args, group_id, root_id, app, dict=dict, task.args = tuple(partial_args) + tuple(task.args) yield task, task.freeze(group_id=group_id, root_id=root_id) - def _apply_tasks(self, tasks, producer=None, app=None, + def _apply_tasks(self, tasks, producer=None, app=None, p=None, add_to_parent=None, chord=None, **options): app = app or self.app with app.producer_or_acquire(producer) as producer: @@ -738,6 +739,9 @@ def _apply_tasks(self, tasks, producer=None, app=None, sig.apply_async(producer=producer, add_to_parent=False, chord=sig.options.get('chord') or chord, **options) + if p: + p.add_noincr(res) + res.backend.add_pending_result(res) yield res # <-- r.parent, etc set in the frozen result. def _freeze_gid(self, options): @@ -762,9 +766,10 @@ def apply_async(self, args=(), kwargs=None, add_to_parent=True, options, group_id, root_id = self._freeze_gid(options) tasks = self._prepared(self.tasks, args, group_id, root_id, app) - result = self.app.GroupResult( - group_id, list(self._apply_tasks(tasks, producer, app, **options)), - ) + p = barrier() + results = list(self._apply_tasks(tasks, producer, app, p, **options)) + result = self.app.GroupResult(group_id, results, ready_barrier=p) + p.finalize() # - Special case of group(A.s() | group(B.s(), C.s())) # That is, group with single item that is a chain but the diff --git a/celery/result.py b/celery/result.py index 5e5ce6f16..c4e7f0034 100644 --- a/celery/result.py +++ b/celery/result.py @@ -432,10 +432,13 @@ class ResultSet(ResultBase): #: List of results in in the set. results = None - def __init__(self, results, app=None, **kwargs): + def __init__(self, results, app=None, ready_barrier=None, **kwargs): self._app = app + self._cache = None self.results = results - self.on_ready = barrier(self.results, (self,), callback=self._on_ready) + self._on_full = ready_barrier or barrier(self.results) + self._on_full.then(promise(self._on_ready)) + self.on_ready = promise() def add(self, result): """Add :class:`AsyncResult` as a new member of the set. @@ -447,8 +450,10 @@ def add(self, result): self.results.append(result) self.ready.add(result) - def _on_ready(self, result): - self.backend.remove_pending_result(result) + def _on_ready(self): + self.backend.remove_pending_result(self) + self._cache = [r.get() for r in self.results] + self.on_ready(self) def remove(self, result): """Remove result from the set; it must be a member. @@ -594,6 +599,8 @@ def get(self, timeout=None, propagate=True, interval=0.5, current result backend. """ + if self._cache is not None: + return self._cache return (self.join_native if self.supports_native_join else self.join)( timeout=timeout, propagate=propagate, interval=interval, callback=callback, no_ack=no_ack, @@ -675,9 +682,6 @@ def join(self, timeout=None, propagate=True, interval=0.5, return results def then(self, callback, on_error=None): - for result in self.results: - self.backend.add_pending_result(result) - result.on_ready.then(self.on_ready) return self.on_ready.then(callback, on_error) def iter_native(self, timeout=None, interval=0.5, no_ack=True, @@ -693,23 +697,11 @@ def iter_native(self, timeout=None, interval=0.5, no_ack=True, result backends. """ - results = self.results - if not results: - raise StopIteration() - ids = set() - for result in self.results: - self.backend.add_pending_result(result) - ids.add(result.id) - bucket = deque() - for _ in self.backend.collect_for_pending( - self, - bucket=bucket, - timeout=timeout, interval=interval, no_ack=no_ack, - on_message=on_message, on_interval=on_interval): - while bucket: - result = bucket.popleft() - if result.id in ids: - yield result.id, result._cache + return self.backend.iter_native( + self, + timeout=timeout, interval=interval, no_ack=no_ack, + on_message=on_message, on_interval=on_interval, + ) def join_native(self, timeout=None, propagate=True, interval=0.5, callback=None, no_ack=True, diff --git a/funtests/stress/stress/suite.py b/funtests/stress/stress/suite.py index 763c41727..0f4298aba 100755 --- a/funtests/stress/stress/suite.py +++ b/funtests/stress/stress/suite.py @@ -201,7 +201,7 @@ def runtest(self, fun, n=50, index=0, repeats=1): self.speaker.beep() raise finally: - print('{0} {1} iterations in {2}s'.format( + print('{0} {1} iterations in {2}'.format( 'failed after' if failed else 'completed', i + 1, humanize_seconds(monotonic() - elapsed), )) diff --git a/funtests/stress/t.py b/funtests/stress/t.py index 37688936b..ac6ef9b1f 100644 --- a/funtests/stress/t.py +++ b/funtests/stress/t.py @@ -1,30 +1,54 @@ +#import eventlet +#eventlet.monkey_patch() + from celery import group import socket from stress.app import add, raising def on_ready(result): - print('RESULT: %r' % (result,)) + print('RESULT: %r' % (result.get(),)) + +finished = [0] def test(): - group(add.s(i, i) for i in range(10)).delay().then(on_ready) - - p = group(add.s(i, i) for i in range(10)).delay() - print(p.get(timeout=5)) - - p = add.delay(2, 2) - print(p.get(timeout=5)) - p = add.delay(2, 2) - print(p.get(timeout=5)) - p = add.delay(2, 2) - print(p.get(timeout=5)) - p = add.delay(2, 2) - print(p.get(timeout=5)) - p = raising.delay() + #group(add.s(i, i) for i in range(1000)).delay().then(on_ready) + + p = group(add.s(i, i) for i in range(1000)).delay() + x = p.get(timeout=5) + y = p.get(timeout=5) try: - print(p.get(timeout=5)) - except Exception as exc: - print('raised: %r' % (exc),) + assert x == y + except AssertionError: + print('-' * 64) + print('X: %r' % (x,)) + print('Y: %r' % (y,)) + raise + assert not any(m is None for m in x) + assert not any(m is None for m in y) + + #p = add.delay(2, 2) + #print(p.get(timeout=5)) + #p = add.delay(2, 2) + #print(p.get(timeout=5)) + #p = add.delay(2, 2) + #print(p.get(timeout=5)) + #p = add.delay(2, 2) + #print(p.get(timeout=5)) + #p = raising.delay() + #try: + # print(p.get(timeout=5)) + #except Exception as exc: + # print('raised: %r' % (exc),) + finished[0] += 1 -for i in range(100): +for i in range(10): test() + + +#for i in range(2): +# eventlet.spawn(test) + +#while finished[0] < 100: +# import time +# time.sleep(0) From 7a47ddb1c787289d70a592c0fb02e8a3343deb19 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 25 Feb 2016 15:34:02 -0800 Subject: [PATCH 1005/1103] [async result] Cleanup branch for merge --- celery/backends/amqp.py | 6 +- celery/backends/async.py | 201 +++++++++++++++++++++++++++++++++++++++ celery/backends/base.py | 180 +---------------------------------- funtests/stress/t.py | 54 ----------- 4 files changed, 206 insertions(+), 235 deletions(-) create mode 100644 celery/backends/async.py delete mode 100644 funtests/stress/t.py diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index 44d4806d6..89ee6a423 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -19,7 +19,8 @@ from celery.utils.log import get_logger from celery.utils.timeutils import maybe_s_to_ms -from .base import AsyncBackendMixin, Backend, BaseResultConsumer +from . import base +from .async import AsyncBackendMixin, BaseResultConsumer __all__ = ['BacklogLimitExceeded', 'AMQPBackend'] @@ -87,8 +88,9 @@ def cancel_for(self, queue): self._consumer.cancel_by_queue(queue.name) -class AMQPBackend(Backend, AsyncBackendMixin): +class AMQPBackend(base.Backend, AsyncBackendMixin): """Publishes results by sending messages.""" + Exchange = Exchange Queue = NoCacheQueue Consumer = Consumer diff --git a/celery/backends/async.py b/celery/backends/async.py new file mode 100644 index 000000000..4f2acd825 --- /dev/null +++ b/celery/backends/async.py @@ -0,0 +1,201 @@ +""" + celery.backends.async + ~~~~~~~~~~~~~~~~~~~~~ + + Async backend support utilitites. + +""" +from __future__ import absolute_import, unicode_literals + +import socket +import time + +from collections import deque +from weakref import WeakKeyDictionary + +from kombu.syn import detect_environment + +from celery import states +from celery.exceptions import TimeoutError +from celery.five import monotonic + +drainers = {} + + +def register_drainer(name): + + def _inner(cls): + drainers[name] = cls + return cls + return _inner + + +@register_drainer('default') +class Drainer(object): + + def __init__(self, result_consumer): + self.result_consumer = result_consumer + + def drain_events_until(self, p, timeout=None, on_interval=None, + monotonic=monotonic, wait=None): + wait = wait or self.result_consumer.drain_events + time_start = monotonic() + + while 1: + # Total time spent may exceed a single call to wait() + if timeout and monotonic() - time_start >= timeout: + raise socket.timeout() + try: + yield self.wait_for(p, wait, timeout=1) + except socket.timeout: + pass + if on_interval: + on_interval() + if p.ready: # got event on the wanted channel. + break + + def wait_for(self, p, wait, timeout=None): + wait(timeout=timeout) + + +@register_drainer('eventlet') +class EventletDrainer(Drainer): + _g = None + _stopped = False + + def run(self): + while not self._stopped: + try: + self.result_consumer.drain_events(timeout=10) + except socket.timeout: + pass + + def start(self): + from eventlet import spawn + if self._g is None: + self._g = spawn(self.run) + + def stop(self): + self._stopped = True + + def wait_for(self, p, wait, timeout=None): + if self._g is None: + self.start() + if not p.ready: + time.sleep(0) + + +class AsyncBackendMixin(object): + + def _collect_into(self, result, bucket): + self.result_consumer.buckets[result] = bucket + + def iter_native(self, result, timeout=None, interval=0.5, no_ack=True, + on_message=None, on_interval=None): + results = result.results + if not results: + raise StopIteration() + + bucket = deque() + for result in results: + self._collect_into(result, bucket) + + for _ in self._wait_for_pending( + result, + timeout=timeout, interval=interval, no_ack=no_ack, + on_message=on_message, on_interval=on_interval): + while bucket: + result = bucket.popleft() + yield result.id, result._cache + while bucket: + result = bucket.popleft() + yield result.id, result._cache + + def add_pending_result(self, result): + if result.id not in self._pending_results: + self._pending_results[result.id] = result + self.result_consumer.consume_from(self._create_binding(result.id)) + return result + + def remove_pending_result(self, result): + self._pending_results.pop(result.id, None) + self.on_result_fulfilled(result) + return result + + def on_result_fulfilled(self, result): + pass + + def wait_for_pending(self, result, + callback=None, propagate=True, **kwargs): + for _ in self._wait_for_pending(result, **kwargs): + pass + return result.maybe_throw(callback=callback, propagate=propagate) + + def _wait_for_pending(self, result, timeout=None, interval=0.5, + no_ack=True, on_interval=None, on_message=None, + callback=None, propagate=True): + return self.result_consumer._wait_for_pending( + result, timeout=timeout, interval=interval, + no_ack=no_ack, on_interval=on_interval, + callback=callback, on_message=on_message, propagate=propagate, + ) + + +class BaseResultConsumer(object): + + def __init__(self, backend, app, accept, pending_results): + self.backend = backend + self.app = app + self.accept = accept + self._pending_results = pending_results + self.on_message = None + self.buckets = WeakKeyDictionary() + self.drainer = drainers[detect_environment()](self) + + def drain_events(self, timeout=None): + raise NotImplementedError('subclass responsibility') + + def _after_fork(self): + self.bucket.clear() + self.buckets = WeakKeyDictionary() + self.on_message = None + self.on_after_fork() + + def on_after_fork(self): + pass + + def drain_events_until(self, p, timeout=None, on_interval=None): + return self.drainer.drain_events_until( + p, timeout=timeout, on_interval=on_interval) + + def _wait_for_pending(self, result, timeout=None, interval=0.5, + no_ack=True, on_interval=None, callback=None, + on_message=None, propagate=True): + prev_on_m, self.on_message = self.on_message, on_message + try: + for _ in self.drain_events_until( + result.on_ready, timeout=timeout, + on_interval=on_interval): + yield + time.sleep(0) + except socket.timeout: + raise TimeoutError('The operation timed out.') + finally: + self.on_message = prev_on_m + + def on_state_change(self, meta, message): + if self.on_message: + self.on_message(meta) + if meta['status'] in states.READY_STATES: + try: + result = self._pending_results[meta['task_id']] + except KeyError: + return + result._maybe_set_cache(meta) + buckets = self.buckets + try: + buckets[result].append(result) + buckets.pop(result) + except KeyError: + pass + time.sleep(0) diff --git a/celery/backends/base.py b/celery/backends/base.py index 14ef7a247..705c18fec 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -13,27 +13,23 @@ """ from __future__ import absolute_import -import socket import sys import time -from collections import deque from datetime import timedelta -from weakref import WeakKeyDictionary from billiard.einfo import ExceptionInfo from kombu.serialization import ( dumps, loads, prepare_accept_content, registry as serializer_registry, ) -from kombu.syn import detect_environment from kombu.utils.encoding import bytes_to_str, ensure_bytes, from_utf8 from celery import states from celery import current_app, group, maybe_signature from celery.app import current_task from celery.exceptions import ChordError, TimeoutError, TaskRevokedError -from celery.five import items, monotonic +from celery.five import items from celery.result import ( GroupResult, ResultBase, allow_join_result, result_from_tuple, ) @@ -432,185 +428,11 @@ def remove_pending_result(self, result): return result -class AsyncBackendMixin(object): - - def _collect_into(self, result, bucket): - self.result_consumer.buckets[result] = bucket - - def iter_native(self, result, timeout=None, interval=0.5, no_ack=True, - on_message=None, on_interval=None): - results = result.results - if not results: - raise StopIteration() - - bucket = deque() - for result in results: - self._collect_into(result, bucket) - - for _ in self._wait_for_pending( - result, - timeout=timeout, interval=interval, no_ack=no_ack, - on_message=on_message, on_interval=on_interval): - while bucket: - result = bucket.popleft() - yield result.id, result._cache - while bucket: - result = bucket.popleft() - yield result.id, result._cache - - def add_pending_result(self, result): - if result.id not in self._pending_results: - self._pending_results[result.id] = result - self.result_consumer.consume_from(self._create_binding(result.id)) - return result - - def remove_pending_result(self, result): - self._pending_results.pop(result.id, None) - self.on_result_fulfilled(result) - return result - - def on_result_fulfilled(self, result): - pass - - def wait_for_pending(self, result, - callback=None, propagate=True, **kwargs): - for _ in self._wait_for_pending(result, **kwargs): - pass - return result.maybe_throw(callback=callback, propagate=propagate) - - def _wait_for_pending(self, result, timeout=None, interval=0.5, - no_ack=True, on_interval=None, on_message=None, - callback=None, propagate=True): - return self.result_consumer._wait_for_pending( - result, timeout=timeout, interval=interval, - no_ack=no_ack, on_interval=on_interval, - callback=callback, on_message=on_message, propagate=propagate, - ) - - class BaseBackend(Backend, SyncBackendMixin): pass BaseDictBackend = BaseBackend # XXX compat - -class Drainer(object): - - def __init__(self, result_consumer): - self.result_consumer = result_consumer - - def drain_events_until(self, p, timeout=None, on_interval=None, - monotonic=monotonic, wait=None): - wait = wait or self.result_consumer.drain_events - time_start = monotonic() - - while 1: - # Total time spent may exceed a single call to wait() - if timeout and monotonic() - time_start >= timeout: - raise socket.timeout() - try: - yield self.wait_for(p, wait, timeout=1) - except socket.timeout: - pass - if on_interval: - on_interval() - if p.ready: # got event on the wanted channel. - break - - def wait_for(self, p, wait, timeout=None): - wait(timeout=timeout) - - -class EventletDrainer(Drainer): - _g = None - _stopped = False - - def run(self): - while not self._stopped: - try: - print("DRAINING!!!!!!!!!!!!!!!!") - self.result_consumer.drain_events(timeout=10) - except socket.timeout: - pass - - def start(self): - from eventlet import spawn - if self._g is None: - self._g = spawn(self.run) - - def stop(self): - self._stopped = True - - def wait_for(self, p, wait, timeout=None): - if self._g is None: - self.start() - if not p.ready: - time.sleep(0) - - -drainers = {'default': Drainer, 'eventlet': EventletDrainer} - -class BaseResultConsumer(object): - - def __init__(self, backend, app, accept, pending_results): - self.backend = backend - self.app = app - self.accept = accept - self._pending_results = pending_results - self.on_message = None - self.buckets = WeakKeyDictionary() - self.drainer = drainers[detect_environment()](self) - - def drain_events(self, timeout=None): - raise NotImplementedError('subclass responsibility') - - def _after_fork(self): - self.bucket.clear() - self.buckets = WeakKeyDictionary() - self.on_message = None - self.on_after_fork() - - def on_after_fork(self): - pass - - def drain_events_until(self, p, timeout=None, on_interval=None): - return self.drainer.drain_events_until( - p, timeout=timeout, on_interval=on_interval) - - def _wait_for_pending(self, result, timeout=None, interval=0.5, - no_ack=True, on_interval=None, callback=None, - on_message=None, propagate=True): - prev_on_m, self.on_message = self.on_message, on_message - try: - for _ in self.drain_events_until( - result.on_ready, timeout=timeout, - on_interval=on_interval): - yield - time.sleep(0) - except socket.timeout: - raise TimeoutError('The operation timed out.') - finally: - self.on_message = prev_on_m - - def on_state_change(self, meta, message): - if self.on_message: - self.on_message(meta) - if meta['status'] in states.READY_STATES: - try: - result = self._pending_results[meta['task_id']] - except KeyError: - return - result._maybe_set_cache(meta) - buckets = self.buckets - try: - buckets[result].append(result) - buckets.pop(result) - except KeyError: - pass - time.sleep(0) - - - class KeyValueStoreBackend(BaseBackend): key_t = ensure_bytes task_keyprefix = 'celery-task-meta-' diff --git a/funtests/stress/t.py b/funtests/stress/t.py deleted file mode 100644 index ac6ef9b1f..000000000 --- a/funtests/stress/t.py +++ /dev/null @@ -1,54 +0,0 @@ -#import eventlet -#eventlet.monkey_patch() - -from celery import group -import socket -from stress.app import add, raising - -def on_ready(result): - print('RESULT: %r' % (result.get(),)) - -finished = [0] - -def test(): - #group(add.s(i, i) for i in range(1000)).delay().then(on_ready) - - p = group(add.s(i, i) for i in range(1000)).delay() - x = p.get(timeout=5) - y = p.get(timeout=5) - try: - assert x == y - except AssertionError: - print('-' * 64) - print('X: %r' % (x,)) - print('Y: %r' % (y,)) - raise - assert not any(m is None for m in x) - assert not any(m is None for m in y) - - #p = add.delay(2, 2) - #print(p.get(timeout=5)) - #p = add.delay(2, 2) - #print(p.get(timeout=5)) - #p = add.delay(2, 2) - #print(p.get(timeout=5)) - #p = add.delay(2, 2) - #print(p.get(timeout=5)) - #p = raising.delay() - #try: - # print(p.get(timeout=5)) - #except Exception as exc: - # print('raised: %r' % (exc),) - finished[0] += 1 - - -for i in range(10): - test() - - -#for i in range(2): -# eventlet.spawn(test) - -#while finished[0] < 100: -# import time -# time.sleep(0) From 072ad1937f7d445a496369f0370033a0ba558ddf Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 25 Feb 2016 16:13:27 -0800 Subject: [PATCH 1006/1103] Tests passing --- celery/backends/async.py | 4 + celery/backends/base.py | 4 + celery/result.py | 23 +++-- celery/tests/backends/test_amqp.py | 141 ----------------------------- celery/tests/tasks/test_result.py | 19 ++-- 5 files changed, 35 insertions(+), 156 deletions(-) diff --git a/celery/backends/async.py b/celery/backends/async.py index 4f2acd825..d751ab6e6 100644 --- a/celery/backends/async.py +++ b/celery/backends/async.py @@ -140,6 +140,10 @@ def _wait_for_pending(self, result, timeout=None, interval=0.5, callback=callback, on_message=on_message, propagate=propagate, ) + @property + def is_async(self): + return True + class BaseResultConsumer(object): diff --git a/celery/backends/base.py b/celery/backends/base.py index 705c18fec..9030d4225 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -427,6 +427,10 @@ def add_pending_result(self, result): def remove_pending_result(self, result): return result + @property + def is_async(self): + return False + class BaseBackend(Backend, SyncBackendMixin): pass diff --git a/celery/result.py b/celery/result.py index c4e7f0034..a37e9e7d0 100644 --- a/celery/result.py +++ b/celery/result.py @@ -168,7 +168,7 @@ def get(self, timeout=None, propagate=True, interval=0.5, if self._cache: if propagate: - self.maybe_throw() + self.maybe_throw(callback=callback) return self.result self.backend.add_pending_result(self) @@ -178,6 +178,7 @@ def get(self, timeout=None, propagate=True, interval=0.5, on_interval=_on_interval, no_ack=no_ack, propagate=propagate, + callback=callback, ) wait = get # deprecated alias to :meth:`get`. @@ -436,9 +437,10 @@ def __init__(self, results, app=None, ready_barrier=None, **kwargs): self._app = app self._cache = None self.results = results - self._on_full = ready_barrier or barrier(self.results) - self._on_full.then(promise(self._on_ready)) self.on_ready = promise() + self._on_full = ready_barrier + if self._on_full: + self._on_full.then(promise(self.on_ready)) def add(self, result): """Add :class:`AsyncResult` as a new member of the set. @@ -448,12 +450,14 @@ def add(self, result): """ if result not in self.results: self.results.append(result) - self.ready.add(result) + if self._on_full: + self._on_full.add(result) def _on_ready(self): self.backend.remove_pending_result(self) - self._cache = [r.get() for r in self.results] - self.on_ready(self) + if self.backend.is_async: + self._cache = [r.get() for r in self.results] + self.on_ready(self) def remove(self, result): """Remove result from the set; it must be a member. @@ -867,9 +871,16 @@ def then(self, callback, on_error=None): return self.on_ready.then(callback, on_error) def _get_task_meta(self): + return self._cache + + @property + def _cache(self): return {'task_id': self.id, 'result': self._result, 'status': self._state, 'traceback': self._traceback} + def __del__(self): + pass + def __reduce__(self): return self.__class__, self.__reduce_args__() diff --git a/celery/tests/backends/test_amqp.py b/celery/tests/backends/test_amqp.py index 64c4fa721..91d3e6d11 100644 --- a/celery/tests/backends/test_amqp.py +++ b/celery/tests/backends/test_amqp.py @@ -239,31 +239,6 @@ def test_poll_result(self): 'Returns cache if no new states', ) - def test_wait_for(self): - b = self.create_backend() - - tid = uuid() - with self.assertRaises(TimeoutError): - b.wait_for(tid, timeout=0.1) - b.store_result(tid, None, states.STARTED) - with self.assertRaises(TimeoutError): - b.wait_for(tid, timeout=0.1) - b.store_result(tid, None, states.RETRY) - with self.assertRaises(TimeoutError): - b.wait_for(tid, timeout=0.1) - b.store_result(tid, 42, states.SUCCESS) - self.assertEqual(b.wait_for(tid, timeout=1)['result'], 42) - b.store_result(tid, 56, states.SUCCESS) - self.assertEqual(b.wait_for(tid, timeout=1)['result'], 42, - 'result is cached') - self.assertEqual(b.wait_for(tid, timeout=1, cache=False)['result'], 56) - b.store_result(tid, KeyError('foo'), states.FAILURE) - res = b.wait_for(tid, timeout=1, cache=False) - self.assertEqual(res['status'], states.FAILURE) - b.store_result(tid, KeyError('foo'), states.PENDING) - with self.assertRaises(TimeoutError): - b.wait_for(tid, timeout=0.01, cache=False) - def test_drain_events_decodes_exceptions_in_meta(self): tid = uuid() b = self.create_backend(serializer="json") @@ -276,122 +251,6 @@ def test_drain_events_decodes_exceptions_in_meta(self): self.assertEqual(cm.exception.__class__.__name__, "RuntimeError") self.assertEqual(str(cm.exception), "aap") - def test_drain_events_remaining_timeouts(self): - class Connection(object): - def drain_events(self, timeout=None): - pass - - b = self.create_backend() - with self.app.pool.acquire_channel(block=False) as (_, channel): - binding = b._create_binding(uuid()) - consumer = b.Consumer(channel, binding, no_ack=True) - callback = Mock() - with self.assertRaises(socket.timeout): - b.drain_events(Connection(), consumer, timeout=0.1, - on_interval=callback) - callback.assert_called_with() - - def test_get_many(self): - b = self.create_backend(max_cached_results=10) - - tids = [] - for i in range(10): - tid = uuid() - b.store_result(tid, i, states.SUCCESS) - tids.append(tid) - - res = list(b.get_many(tids, timeout=1)) - expected_results = [ - (_tid, {'status': states.SUCCESS, - 'result': i, - 'traceback': None, - 'task_id': _tid, - 'children': None}) - for i, _tid in enumerate(tids) - ] - self.assertEqual(sorted(res), sorted(expected_results)) - self.assertDictEqual(b._cache[res[0][0]], res[0][1]) - cached_res = list(b.get_many(tids, timeout=1)) - self.assertEqual(sorted(cached_res), sorted(expected_results)) - - # times out when not ready in cache (this shouldn't happen) - b._cache[res[0][0]]['status'] = states.RETRY - with self.assertRaises(socket.timeout): - list(b.get_many(tids, timeout=0.01)) - - # times out when result not yet ready - with self.assertRaises(socket.timeout): - tids = [uuid()] - b.store_result(tids[0], i, states.PENDING) - list(b.get_many(tids, timeout=0.01)) - - def test_get_many_on_message(self): - b = self.create_backend(max_cached_results=10) - - tids = [] - for i in range(10): - tid = uuid() - b.store_result(tid, '', states.PENDING) - b.store_result(tid, 'comment_%i_1' % i, states.STARTED) - b.store_result(tid, 'comment_%i_2' % i, states.STARTED) - b.store_result(tid, 'final result %i' % i, states.SUCCESS) - tids.append(tid) - - expected_messages = {} - for i, _tid in enumerate(tids): - expected_messages[_tid] = [] - expected_messages[_tid].append((states.PENDING, '')) - expected_messages[_tid].append( - (states.STARTED, 'comment_%i_1' % i), - ) - expected_messages[_tid].append( - (states.STARTED, 'comment_%i_2' % i), - ) - expected_messages[_tid].append( - (states.SUCCESS, 'final result %i' % i), - ) - - on_message_results = {} - - def on_message(body): - if not body['task_id'] in on_message_results: - on_message_results[body['task_id']] = [] - on_message_results[body['task_id']].append( - (body['status'], body['result']), - ) - - list(b.get_many(tids, timeout=1, on_message=on_message)) - self.assertEqual(sorted(on_message_results), sorted(expected_messages)) - - def test_get_many_raises_outer_block(self): - - class Backend(AMQPBackend): - - def Consumer(*args, **kwargs): - raise KeyError('foo') - - b = Backend(self.app) - with self.assertRaises(KeyError): - next(b.get_many(['id1'])) - - def test_get_many_raises_inner_block(self): - with patch('kombu.connection.Connection.drain_events') as drain: - drain.side_effect = KeyError('foo') - b = AMQPBackend(self.app) - with self.assertRaises(KeyError): - next(b.get_many(['id1'])) - - def test_consume_raises_inner_block(self): - with patch('kombu.connection.Connection.drain_events') as drain: - - def se(*args, **kwargs): - drain.side_effect = ValueError() - raise KeyError('foo') - drain.side_effect = se - b = AMQPBackend(self.app) - with self.assertRaises(ValueError): - next(b.consume('id1')) - def test_no_expires(self): b = self.create_backend(expires=None) app = self.app diff --git a/celery/tests/tasks/test_result.py b/celery/tests/tasks/test_result.py index bf39668c5..b1b6c100e 100644 --- a/celery/tests/tasks/test_result.py +++ b/celery/tests/tasks/test_result.py @@ -3,6 +3,7 @@ from contextlib import contextmanager from celery import states +from celery.backends.base import SyncBackendMixin from celery.exceptions import ( ImproperlyConfigured, IncompleteStream, TimeoutError, ) @@ -100,17 +101,15 @@ def test_propagates_for_parent(self): x = self.app.AsyncResult(uuid()) x.backend = Mock(name='backend') x.backend.get_task_meta.return_value = {} - x.backend.wait_for.return_value = { - 'status': states.SUCCESS, 'result': 84, - } + x.backend.wait_for_pending.return_value = 84 x.parent = EagerResult(uuid(), KeyError('foo'), states.FAILURE) with self.assertRaises(KeyError): x.get(propagate=True) - self.assertFalse(x.backend.wait_for.called) + self.assertFalse(x.backend.wait_for_pending.called) x.parent = EagerResult(uuid(), 42, states.SUCCESS) self.assertEqual(x.get(propagate=True), 84) - self.assertTrue(x.backend.wait_for.called) + self.assertTrue(x.backend.wait_for_pending.called) def test_get_children(self): tid = uuid() @@ -477,7 +476,7 @@ def get(self, **kwargs): return self.result -class SimpleBackend(object): +class SimpleBackend(SyncBackendMixin): ids = [] def __init__(self, ids=[]): @@ -676,10 +675,12 @@ def test_successful(self): def test_failed(self): self.assertFalse(self.ts.failed()) - def test_maybe_reraise(self): + def test_maybe_throw(self): self.ts.results = [Mock(name='r1')] - self.ts.maybe_reraise() - self.ts.results[0].maybe_reraise.assert_called_with() + self.ts.maybe_throw() + self.ts.results[0].maybe_throw.assert_called_with( + callback=None, propagate=True, + ) def test_join__on_message(self): with self.assertRaises(ImproperlyConfigured): From cb04d8aaaba14f5d2f5b9b5cb2631cf13233eef6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 25 Feb 2016 16:40:32 -0800 Subject: [PATCH 1007/1103] [commands] Fixes support for celery shell --ipython --- celery/bin/celery.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/celery/bin/celery.py b/celery/bin/celery.py index 599875e7d..2b0c74c81 100644 --- a/celery/bin/celery.py +++ b/celery/bin/celery.py @@ -623,12 +623,35 @@ def invoke_fallback_shell(self): code.interact(local=self.locals) def invoke_ipython_shell(self): - try: - from IPython.terminal import embed - embed.TerminalInteractiveShell(user_ns=self.locals).mainloop() - except ImportError: # ipython < 0.11 - from IPython.Shell import IPShell - IPShell(argv=[], user_ns=self.locals).mainloop() + for ip in (self._ipython, self._ipython_pre_10, + self._ipython_terminal, self._ipython_010, + self._no_ipython): + try: + return ip() + except ImportError: + pass + + def _ipython(self): + from IPython import start_ipython + start_ipython(argv=[], user_ns=self.locals) + + def _ipython_pre_10(self): # pragma: no cover + from IPython.frontend.terminal.ipapp import TerminalIPythonApp + app = TerminalIPythonApp.instance() + app.initialize(argv=[]) + app.shell.user_ns.update(self.locals) + app.start() + + def _ipython_terminal(self): # pragma: no cover + from IPython.terminal import embed + embed.TerminalInteractiveShell(user_ns=self.locals).mainloop() + + def _ipython_010(self): # pragma: no cover + from IPython.Shell import IPShell + IPShell(argv=[], user_ns=self.locals).mainloop() + + def _no_ipython(self): # pragma: no cover + raise ImportError("no suitable ipython found") def invoke_bpython_shell(self): import bpython From 984d218f826b8bba6b3c427be5d4a423faa1de0f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 25 Feb 2016 16:42:42 -0800 Subject: [PATCH 1008/1103] flakes --- celery/result.py | 2 +- celery/tests/app/test_control.py | 2 -- celery/tests/backends/test_amqp.py | 4 +--- celery/tests/utils/test_objects.py | 1 - celery/tests/worker/test_consumer.py | 4 ++-- 5 files changed, 4 insertions(+), 9 deletions(-) diff --git a/celery/result.py b/celery/result.py index a37e9e7d0..89a02d849 100644 --- a/celery/result.py +++ b/celery/result.py @@ -14,7 +14,7 @@ from contextlib import contextmanager from copy import copy -from amqp.promise import Thenable, barrier, promise +from amqp.promise import Thenable, promise from kombu.utils import cached_property from . import current_app diff --git a/celery/tests/app/test_control.py b/celery/tests/app/test_control.py index 125bc7682..7a0550680 100644 --- a/celery/tests/app/test_control.py +++ b/celery/tests/app/test_control.py @@ -1,7 +1,5 @@ from __future__ import absolute_import -import warnings - from functools import wraps from kombu.pidbox import Mailbox diff --git a/celery/tests/backends/test_amqp.py b/celery/tests/backends/test_amqp.py index 91d3e6d11..d92ba666d 100644 --- a/celery/tests/backends/test_amqp.py +++ b/celery/tests/backends/test_amqp.py @@ -1,7 +1,6 @@ from __future__ import absolute_import import pickle -import socket from contextlib import contextmanager from datetime import timedelta @@ -11,13 +10,12 @@ from celery import states from celery.backends.amqp import AMQPBackend -from celery.exceptions import TimeoutError from celery.five import Empty, Queue, range from celery.result import AsyncResult from celery.utils import uuid from celery.tests.case import ( - AppCase, Mock, depends_on_current_app, patch, sleepdeprived, + AppCase, Mock, depends_on_current_app, sleepdeprived, ) diff --git a/celery/tests/utils/test_objects.py b/celery/tests/utils/test_objects.py index 88754c1b8..303d14966 100644 --- a/celery/tests/utils/test_objects.py +++ b/celery/tests/utils/test_objects.py @@ -11,4 +11,3 @@ def test(self): x = Bunch(foo='foo', bar=2) self.assertEqual(x.foo, 'foo') self.assertEqual(x.bar, 2) - diff --git a/celery/tests/worker/test_consumer.py b/celery/tests/worker/test_consumer.py index 67870fbea..e41a22e22 100644 --- a/celery/tests/worker/test_consumer.py +++ b/celery/tests/worker/test_consumer.py @@ -178,12 +178,12 @@ def test_register_with_event_loop(self): c.register_with_event_loop(Mock(name='loop')) def test_on_close_clears_semaphore_timer_and_reqs(self): - with patch('celery.worker.consumer.consumer.reserved_requests') as reserv: + with patch('celery.worker.consumer.consumer.reserved_requests') as res: c = self.get_consumer() c.on_close() c.controller.semaphore.clear.assert_called_with() c.timer.clear.assert_called_with() - reserv.clear.assert_called_with() + res.clear.assert_called_with() c.pool.flush.assert_called_with() c.controller = None From 6c78582ea559427881360aa502932239eb8c433a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 26 Feb 2016 12:50:50 -0800 Subject: [PATCH 1009/1103] Fixes group().then(callback) --- celery/result.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/result.py b/celery/result.py index 89a02d849..d4aae59bb 100644 --- a/celery/result.py +++ b/celery/result.py @@ -437,7 +437,7 @@ def __init__(self, results, app=None, ready_barrier=None, **kwargs): self._app = app self._cache = None self.results = results - self.on_ready = promise() + self.on_ready = promise(args=(self,)) self._on_full = ready_barrier if self._on_full: self._on_full.then(promise(self.on_ready)) @@ -457,7 +457,7 @@ def _on_ready(self): self.backend.remove_pending_result(self) if self.backend.is_async: self._cache = [r.get() for r in self.results] - self.on_ready(self) + self.on_ready() def remove(self, result): """Remove result from the set; it must be a member. @@ -864,7 +864,7 @@ def __init__(self, id, ret_value, state, traceback=None): self._result = ret_value self._state = state self._traceback = traceback - self.on_ready = promise() + self.on_ready = promise(args=(self,)) self.on_ready() def then(self, callback, on_error=None): From 11c2a4324fa4dd511d9620970f2fae4ac92a95b7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 26 Feb 2016 13:10:33 -0800 Subject: [PATCH 1010/1103] [4.0][canvas] Fixes regressions with chain. Closes #3066 - chain did not implement .clone properly, so reusing the same chain instance did not work. - chain.freeze() returned the first task in the chain rather than the last. - async backend.get() did not properly account for cached results. --- celery/backends/async.py | 5 ++++- celery/canvas.py | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/celery/backends/async.py b/celery/backends/async.py index d751ab6e6..7fc26c4e1 100644 --- a/celery/backends/async.py +++ b/celery/backends/async.py @@ -98,7 +98,10 @@ def iter_native(self, result, timeout=None, interval=0.5, no_ack=True, bucket = deque() for result in results: - self._collect_into(result, bucket) + if result._cache: + bucket.append(result) + else: + self._collect_into(result, bucket) for _ in self._wait_for_pending( result, diff --git a/celery/canvas.py b/celery/canvas.py index e44ea497d..71f164ba3 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -414,6 +414,11 @@ def __call__(self, *args, **kwargs): if self.tasks: return self.apply_async(args, kwargs) + def clone(self, *args, **kwargs): + s = Signature.clone(self, *args, **kwargs) + s.kwargs['tasks'] = [sig.clone() for sig in s.kwargs['tasks']] + return s + def apply_async(self, args=(), kwargs={}, **options): # python is best at unpacking kwargs, so .run is here to do that. app = self.app @@ -454,7 +459,7 @@ def freeze(self, _id=None, group_id=None, chord=None, self.args, self.tasks, root_id, parent_id, None, self.app, _id, group_id, chord, clone=False, ) - return results[-1] + return results[0] def prepare_steps(self, args, tasks, root_id=None, parent_id=None, link_error=None, app=None, From 01901614f2612f4156dacbc0bff9e4b7c21edb43 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 26 Feb 2016 15:12:11 -0800 Subject: [PATCH 1011/1103] Docs: Clarify what the -P option belongs to. --- docs/configuration.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 2c4be8004..0451f162c 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -2174,8 +2174,9 @@ Name of the pool class used by the worker. .. admonition:: Eventlet/Gevent Never use this option to select the eventlet or gevent pool. - You must use the `-P` option instead, otherwise the monkey patching - will happen too late and things will break in strange and silent ways. + You must use the `-P` option to :program:`celery worker` instead, to + ensure the monkey patches are not applied too late, causing things + to break in strange ways. Default is ``celery.concurrency.prefork:TaskPool``. From e27e42972d01f840ea751778c99cde2edb8cd0dc Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 26 Feb 2016 15:14:39 -0800 Subject: [PATCH 1012/1103] [canvas] maybe_signature should not return list (Issue #3043) --- celery/canvas.py | 9 +++------ celery/tests/tasks/test_canvas.py | 7 ------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 71f164ba3..fd5984c84 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -1041,12 +1041,9 @@ def signature(varies, *args, **kwargs): def maybe_signature(d, app=None): if d is not None: - if isinstance(d, dict): - if not isinstance(d, abstract.CallableSignature): - d = signature(d) - elif isinstance(d, list): - return [maybe_signature(s, app=app) for s in d] - + if (isinstance(d, dict) and + not isinstance(d, abstract.CallableSignature)): + d = signature(d) if app is not None: d._app = app return d diff --git a/celery/tests/tasks/test_canvas.py b/celery/tests/tasks/test_canvas.py index c56394e7d..ea2c45952 100644 --- a/celery/tests/tasks/test_canvas.py +++ b/celery/tests/tasks/test_canvas.py @@ -665,13 +665,6 @@ def test_is_dict(self): maybe_signature(dict(self.add.s()), app=self.app), Signature, ) - def test_is_list(self): - sigs = [dict(self.add.s(2, 2)), dict(self.add.s(4, 4))] - sigs = maybe_signature(sigs, app=self.app) - for sig in sigs: - self.assertIsInstance(sig, Signature) - self.assertIs(sig.app, self.app) - def test_when_sig(self): s = self.add.s() self.assertIs(maybe_signature(s, app=self.app), s) From fa6fbd192bfd40ef6140caed3d85c22a6dd8772a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 26 Feb 2016 15:20:11 -0800 Subject: [PATCH 1013/1103] [canvas] Remove unused localized argument (Issue #3043) --- celery/canvas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index fd5984c84..d6c0ac5e9 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -713,7 +713,7 @@ def from_dict(self, d, app=None): def __len__(self): return len(self.tasks) - def _prepared(self, tasks, partial_args, group_id, root_id, app, dict=dict, + def _prepared(self, tasks, partial_args, group_id, root_id, app, CallableSignature=abstract.CallableSignature, from_dict=Signature.from_dict): for task in tasks: From 560b78e280170dde147fade8df62a0febad8027d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 26 Feb 2016 16:05:07 -0800 Subject: [PATCH 1014/1103] [canvas] Updates localized globals --- celery/canvas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index d6c0ac5e9..db170422c 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -715,7 +715,8 @@ def __len__(self): def _prepared(self, tasks, partial_args, group_id, root_id, app, CallableSignature=abstract.CallableSignature, - from_dict=Signature.from_dict): + from_dict=Signature.from_dict, + isinstance=isinstance, tuple=tuple): for task in tasks: if isinstance(task, CallableSignature): # local sigs are always of type Signature, and we From 4a806a63c521a64c52146550a286e3ed536958da Mon Sep 17 00:00:00 2001 From: Ahmet Demir Date: Tue, 29 Sep 2015 13:56:03 +0100 Subject: [PATCH 1015/1103] Add Elasticsearch Backend --- celery/backends/__init__.py | 1 + celery/backends/elasticsearch.py | 120 ++++++++++++++++++++ celery/tests/backends/test_elasticsearch.py | 86 ++++++++++++++ requirements/extras/elasticsearch.txt | 1 + 4 files changed, 208 insertions(+) create mode 100644 celery/backends/elasticsearch.py create mode 100644 celery/tests/backends/test_elasticsearch.py create mode 100644 requirements/extras/elasticsearch.txt diff --git a/celery/backends/__init__.py b/celery/backends/__init__.py index 2f5b07b52..77c6480e7 100644 --- a/celery/backends/__init__.py +++ b/celery/backends/__init__.py @@ -31,6 +31,7 @@ 'mongodb': 'celery.backends.mongodb:MongoBackend', 'db': 'celery.backends.database:DatabaseBackend', 'database': 'celery.backends.database:DatabaseBackend', + 'elasticsearch': 'celery.backends.elasticsearch:ElasticsearchBackend', 'cassandra': 'celery.backends.cassandra:CassandraBackend', 'couchbase': 'celery.backends.couchbase:CouchBaseBackend', 'couchdb': 'celery.backends.couchdb:CouchDBBackend', diff --git a/celery/backends/elasticsearch.py b/celery/backends/elasticsearch.py new file mode 100644 index 000000000..4031f6385 --- /dev/null +++ b/celery/backends/elasticsearch.py @@ -0,0 +1,120 @@ +# -* coding: utf-8 -*- +""" + celery.backends.elasticsearch + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Elasticsearch result store backend. + Based on CouchDB backend. + +""" +from __future__ import absolute_import + +try: + import elasticsearch +except ImportError: + elasticsearch = None # noqa + +from .base import KeyValueStoreBackend + +import datetime + +from kombu.utils.url import _parse_url + +from celery.exceptions import ImproperlyConfigured + +__all__ = ['ElasticsearchBackend'] + +ERR_LIB_MISSING = """\ +You need to install the elasticsearch library to use the Elasticsearch \ +result backend\ +""" + +class ElasticsearchBackend(KeyValueStoreBackend): + + index = 'celery' + doc_type = 'backend' + scheme = 'http' + host = 'localhost' + port = 9200 + + + def __init__(self, url=None, *args, **kwargs): + """Initialize Elasticsearch backend instance. + + :raises celery.exceptions.ImproperlyConfigured: if + module :mod:`elasticsearch` is not available. + + """ + super(ElasticsearchBackend, self).__init__(*args, **kwargs) + + if elasticsearch is None: + raise ImproperlyConfigured(ERR_LIB_MISSING) + + uindex = udoc_type = uscheme = uhost = uport = None + + if url: + uscheme, uhost, uport, _, _, uuri, _ = _parse_url(url) # noqa + uuri = uuri.strip('/') if uuri else None + uuris = uuri.split("/") + uindex = uuris[0] if len(uuris) > 0 else None + udoc_type = uuris[1] if len(uuris) > 1 else None + + self.index = uindex or self.index + self.doc_type = udoc_type or self.doc_type + self.scheme = uscheme or self.scheme + self.host = uhost or self.host + self.port = uport or self.port + + self._server = None + + + def _get_server(self): + """Connect to the Elasticsearch server.""" + return elasticsearch.Elasticsearch(self.host) + + + @property + def server(self): + if self._server is None: + self._server = self._get_server() + return self._server + + + def get(self, key): + try: + out = self.server.get(index=self.index,\ + doc_type=self.doc_type,\ + id=key) + if isinstance(out, dict) \ + and "found" in out and out["found"] \ + and "_source" in out and key in out["_source"]: + return out["_source"][key] + else: + return None + except elasticsearch.exceptions.NotFoundError: + return None + + + def set(self, key, value): + try: + data = {} + data['@timestamp'] = "{0}Z".format(datetime.datetime.utcnow()\ + .strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3]) + data[key] = value + self.server.index(index=self.index, doc_type=self.doc_type,\ + id=key, body=data) + except elasticsearch.exceptions.ConflictError: + # document already exists, update it + data = self.get(key) + data[key] = value + self.server.index(index=self.index, doc_type=self.doc_type,\ + id=key, body=data, refresh=True) + + + def mget(self, keys): + return [self.get(key) for key in keys] + + + def delete(self, key): + self.server.delete(index=self.index, doc_type=self.doc_type, id=key) + diff --git a/celery/tests/backends/test_elasticsearch.py b/celery/tests/backends/test_elasticsearch.py new file mode 100644 index 000000000..2990172fa --- /dev/null +++ b/celery/tests/backends/test_elasticsearch.py @@ -0,0 +1,86 @@ +from __future__ import absolute_import + +from celery.backends import elasticsearch as module +from celery.backends.elasticsearch import ElasticsearchBackend +from celery.exceptions import ImproperlyConfigured +from celery import backends +from celery.tests.case import ( + AppCase, Mock, SkipTest, sentinel, +) + +try: + import elasticsearch +except ImportError: + elasticsearch = None + + +class test_ElasticsearchBackend(AppCase): + + + def setup(self): + if elasticsearch is None: + raise SkipTest('elasticsearch is not installed.') + self.backend = ElasticsearchBackend(app=self.app) + + + def test_init_no_elasticsearch(self): + prev, module.elasticsearch = module.elasticsearch, None + try: + with self.assertRaises(ImproperlyConfigured): + ElasticsearchBackend(app=self.app) + finally: + module.elasticsearch = prev + + + def test_get(self): + x = ElasticsearchBackend(app=self.app) + x._server = Mock() + x._server.get = Mock() + # expected result + r = dict(found=True, _source={sentinel.task_id: sentinel.result}) + x._server.get.return_value = r + dict_result = x.get(sentinel.task_id) + + self.assertEqual(dict_result, sentinel.result) + x._server.get.assert_called_once_with(doc_type=x.doc_type, id=sentinel.task_id, index=x.index) + + + def test_get_none(self): + x = ElasticsearchBackend(app=self.app) + x._server = Mock() + x._server.get = Mock() + x._server.get.return_value = sentinel.result + none_reusult = x.get(sentinel.task_id) + + self.assertEqual(none_reusult, None) + x._server.get.assert_called_once_with(doc_type=x.doc_type, id=sentinel.task_id, index=x.index) + + + def test_delete(self): + x = ElasticsearchBackend(app=self.app) + x._server = Mock() + x._server.delete = Mock() + x._server.delete.return_value = sentinel.result + + self.assertIsNone(x.delete(sentinel.task_id), sentinel.result) + x._server.delete.assert_called_once_with(doc_type=x.doc_type, id=sentinel.task_id, index=x.index) + + + def test_backend_by_url(self, url='elasticsearch://localhost:9200/index'): + backend, url_ = backends.get_backend_by_url(url, self.app.loader) + + self.assertIs(backend, ElasticsearchBackend) + self.assertEqual(url_, url) + + + def test_backend_params_by_url(self): + url = 'elasticsearch://localhost:9200/index/doc_type' + with self.Celery(backend=url) as app: + x = app.backend + + self.assertEqual(x.index, 'index') + self.assertEqual(x.doc_type, 'doc_type') + self.assertEqual(x.scheme, 'elasticsearch') + self.assertEqual(x.host, 'localhost') + self.assertEqual(x.port, 9200) + diff --git a/requirements/extras/elasticsearch.txt b/requirements/extras/elasticsearch.txt new file mode 100644 index 000000000..174c3f8b3 --- /dev/null +++ b/requirements/extras/elasticsearch.txt @@ -0,0 +1 @@ +elasticsearch From 9364a9ec8939c32b879fc4333dddd5cbaa192439 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 26 Feb 2016 16:36:19 -0800 Subject: [PATCH 1016/1103] Cosmetics for Elasticsearch result backend (Issue #2828) --- README.rst | 4 +- celery/backends/elasticsearch.py | 135 +++++++++--------- celery/tests/backends/test_elasticsearch.py | 39 ++--- docs/configuration.rst | 20 +++ docs/getting-started/introduction.rst | 2 +- docs/includes/installation.txt | 3 + docs/includes/introduction.txt | 2 +- .../celery.backends.elasticsearch.txt | 11 ++ docs/internals/reference/index.rst | 1 + setup.py | 2 +- 10 files changed, 129 insertions(+), 90 deletions(-) create mode 100644 docs/internals/reference/celery.backends.elasticsearch.txt diff --git a/README.rst b/README.rst index f7364034d..0a82f53ca 100644 --- a/README.rst +++ b/README.rst @@ -34,7 +34,7 @@ any language. So far there's RCelery_ for the Ruby programming language, and a `PHP client`, but language interoperability can also be achieved by using webhooks. -.. _RCelery: http://leapfrogonline.github.io/rcelery/ +.. _RCelery: http://leapfrogdevelopment.github.com/rcelery/ .. _`PHP client`: https://github.com/gjedeer/celery-php .. _`using webhooks`: http://docs.celeryproject.org/en/latest/userguide/remote-tasks.html @@ -139,7 +139,7 @@ It supports... - AMQP, Redis - memcached, MongoDB - SQLAlchemy, Django ORM - - Apache Cassandra, IronCache + - Apache Cassandra, IronCache, Elasticsearch - **Serialization** diff --git a/celery/backends/elasticsearch.py b/celery/backends/elasticsearch.py index 4031f6385..95fcd27bb 100644 --- a/celery/backends/elasticsearch.py +++ b/celery/backends/elasticsearch.py @@ -1,35 +1,41 @@ # -* coding: utf-8 -*- """ celery.backends.elasticsearch - ~~~~~~~~~~~~~~~~~~~~~~~~~ + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Elasticsearch result store backend. - Based on CouchDB backend. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals -try: - import elasticsearch -except ImportError: - elasticsearch = None # noqa - -from .base import KeyValueStoreBackend - -import datetime +from datetime import datetime from kombu.utils.url import _parse_url from celery.exceptions import ImproperlyConfigured +from .base import KeyValueStoreBackend + +try: + import elasticsearch +except ImportError: + elasticsearch = None # noqa + __all__ = ['ElasticsearchBackend'] -ERR_LIB_MISSING = """\ +E_LIB_MISSING = """\ You need to install the elasticsearch library to use the Elasticsearch \ -result backend\ +result backend.\ """ + class ElasticsearchBackend(KeyValueStoreBackend): + """Elasticsearch Backend. + + :raises celery.exceptions.ImproperlyConfigured: if + module :mod:`elasticsearch` is not available. + + """ index = 'celery' doc_type = 'backend' @@ -37,84 +43,79 @@ class ElasticsearchBackend(KeyValueStoreBackend): host = 'localhost' port = 9200 - def __init__(self, url=None, *args, **kwargs): - """Initialize Elasticsearch backend instance. - - :raises celery.exceptions.ImproperlyConfigured: if - module :mod:`elasticsearch` is not available. - - """ super(ElasticsearchBackend, self).__init__(*args, **kwargs) if elasticsearch is None: - raise ImproperlyConfigured(ERR_LIB_MISSING) - - uindex = udoc_type = uscheme = uhost = uport = None - - if url: - uscheme, uhost, uport, _, _, uuri, _ = _parse_url(url) # noqa - uuri = uuri.strip('/') if uuri else None - uuris = uuri.split("/") - uindex = uuris[0] if len(uuris) > 0 else None - udoc_type = uuris[1] if len(uuris) > 1 else None - - self.index = uindex or self.index - self.doc_type = udoc_type or self.doc_type - self.scheme = uscheme or self.scheme - self.host = uhost or self.host - self.port = uport or self.port - - self._server = None + raise ImproperlyConfigured(E_LIB_MISSING) + index = doc_type = scheme = host = port = None - def _get_server(self): - """Connect to the Elasticsearch server.""" - return elasticsearch.Elasticsearch(self.host) - + if url: + scheme, host, port, _, _, path, _ = _parse_url(url) # noqa + if path: + path = path.strip('/') + index, _, doc_type = path.partition('/') - @property - def server(self): - if self._server is None: - self._server = self._get_server() - return self._server + self.index = index or self.index + self.doc_type = doc_type or self.doc_type + self.scheme = scheme or self.scheme + self.host = host or self.host + self.port = port or self.port + self._server = None def get(self, key): try: - out = self.server.get(index=self.index,\ - doc_type=self.doc_type,\ - id=key) - if isinstance(out, dict) \ - and "found" in out and out["found"] \ - and "_source" in out and key in out["_source"]: - return out["_source"][key] - else: - return None + res = self.server.get( + index=self.index, + doc_type=self.doc_type, + id=key, + ) + try: + if res['found']: + return res['_source'][key] + except (TypeError, KeyError): + pass except elasticsearch.exceptions.NotFoundError: - return None - + pass def set(self, key, value): try: - data = {} - data['@timestamp'] = "{0}Z".format(datetime.datetime.utcnow()\ - .strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3]) - data[key] = value - self.server.index(index=self.index, doc_type=self.doc_type,\ - id=key, body=data) + self._index( + id=key, + body={ + key: value, + '@timestamp': '{0}Z'.format( + datetime.utcnow().isoformat()[:-3] + ), + }, + ) except elasticsearch.exceptions.ConflictError: # document already exists, update it data = self.get(key) data[key] = value - self.server.index(index=self.index, doc_type=self.doc_type,\ - id=key, body=data, refresh=True) + self._index(key, data, refresh=True) + def _index(self, id, body, **kwargs): + return self.server.index( + index=self.index, + doc_type=self.doc_type, + **kwargs + ) def mget(self, keys): return [self.get(key) for key in keys] - def delete(self, key): self.server.delete(index=self.index, doc_type=self.doc_type, id=key) + def _get_server(self): + """Connect to the Elasticsearch server.""" + return elasticsearch.Elasticsearch(self.host) + + @property + def server(self): + if self._server is None: + self._server = self._get_server() + return self._server diff --git a/celery/tests/backends/test_elasticsearch.py b/celery/tests/backends/test_elasticsearch.py index 2990172fa..cc5d96fdd 100644 --- a/celery/tests/backends/test_elasticsearch.py +++ b/celery/tests/backends/test_elasticsearch.py @@ -1,12 +1,11 @@ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals +from celery import backends from celery.backends import elasticsearch as module from celery.backends.elasticsearch import ElasticsearchBackend from celery.exceptions import ImproperlyConfigured -from celery import backends -from celery.tests.case import ( - AppCase, Mock, SkipTest, sentinel, -) + +from celery.tests.case import AppCase, Mock, SkipTest, sentinel try: import elasticsearch @@ -16,13 +15,11 @@ class test_ElasticsearchBackend(AppCase): - def setup(self): if elasticsearch is None: raise SkipTest('elasticsearch is not installed.') self.backend = ElasticsearchBackend(app=self.app) - def test_init_no_elasticsearch(self): prev, module.elasticsearch = module.elasticsearch, None try: @@ -31,7 +28,6 @@ def test_init_no_elasticsearch(self): finally: module.elasticsearch = prev - def test_get(self): x = ElasticsearchBackend(app=self.app) x._server = Mock() @@ -42,19 +38,25 @@ def test_get(self): dict_result = x.get(sentinel.task_id) self.assertEqual(dict_result, sentinel.result) - x._server.get.assert_called_once_with(doc_type=x.doc_type, id=sentinel.task_id, index=x.index) - + x._server.get.assert_called_once_with( + doc_type=x.doc_type, + id=sentinel.task_id, + index=x.index, + ) def test_get_none(self): x = ElasticsearchBackend(app=self.app) x._server = Mock() x._server.get = Mock() x._server.get.return_value = sentinel.result - none_reusult = x.get(sentinel.task_id) - - self.assertEqual(none_reusult, None) - x._server.get.assert_called_once_with(doc_type=x.doc_type, id=sentinel.task_id, index=x.index) + none_result = x.get(sentinel.task_id) + self.assertEqual(none_result, None) + x._server.get.assert_called_once_with( + doc_type=x.doc_type, + id=sentinel.task_id, + index=x.index, + ) def test_delete(self): x = ElasticsearchBackend(app=self.app) @@ -63,8 +65,11 @@ def test_delete(self): x._server.delete.return_value = sentinel.result self.assertIsNone(x.delete(sentinel.task_id), sentinel.result) - x._server.delete.assert_called_once_with(doc_type=x.doc_type, id=sentinel.task_id, index=x.index) - + x._server.delete.assert_called_once_with( + doc_type=x.doc_type, + id=sentinel.task_id, + index=x.index, + ) def test_backend_by_url(self, url='elasticsearch://localhost:9200/index'): backend, url_ = backends.get_backend_by_url(url, self.app.loader) @@ -72,7 +77,6 @@ def test_backend_by_url(self, url='elasticsearch://localhost:9200/index'): self.assertIs(backend, ElasticsearchBackend) self.assertEqual(url_, url) - def test_backend_params_by_url(self): url = 'elasticsearch://localhost:9200/index/doc_type' with self.Celery(backend=url) as app: @@ -83,4 +87,3 @@ def test_backend_params_by_url(self): self.assertEqual(x.scheme, 'elasticsearch') self.assertEqual(x.host, 'localhost') self.assertEqual(x.port, 9200) - diff --git a/docs/configuration.rst b/docs/configuration.rst index 0451f162c..e3d034b56 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -511,6 +511,10 @@ Can be one of the following: Use `Cassandra`_ to store the results. See :ref:`conf-cassandra-result-backend`. +* elasticsearch + Use `Elasticsearch`_ to store the results. + See :ref:`conf-elasticsearch-result-backend`. + * ironcache Use `IronCache`_ to store the results. See :ref:`conf-ironcache-result-backend`. @@ -541,6 +545,7 @@ Can be one of the following: .. _`MongoDB`: http://mongodb.org .. _`Redis`: http://redis.io .. _`Cassandra`: http://cassandra.apache.org/ +.. _`Elasticsearch`: https://aws.amazon.com/elasticsearch-service/ .. _`IronCache`: http://www.iron.io/cache .. _`CouchDB`: http://www.couchdb.com/ .. _`Couchbase`: http://www.couchbase.com/ @@ -1002,6 +1007,21 @@ Example configuration cassandra_write_consistency = 'ONE' cassandra_entry_ttl = 86400 +.. _conf-elasticsearch-result-backend: + +Elasticsearch backend settings +------------------------------ + +To use `Elasticsearch`_ as the result backend you simply need to +configure the :setting:`result_backend` setting with the correct URL. + +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + result_backend = 'elasticsearch://example.com:9200/index_name/doc_type' + .. _conf-riak-result-backend: Riak backend settings diff --git a/docs/getting-started/introduction.rst b/docs/getting-started/introduction.rst index f7d015932..ad8472497 100644 --- a/docs/getting-started/introduction.rst +++ b/docs/getting-started/introduction.rst @@ -134,7 +134,7 @@ Celery is… - AMQP, Redis - memcached, MongoDB - SQLAlchemy, Django ORM - - Apache Cassandra + - Apache Cassandra, IronCache, Elasticsearch - **Serialization** diff --git a/docs/includes/installation.txt b/docs/includes/installation.txt index 3b4a669d7..fffd8c178 100644 --- a/docs/includes/installation.txt +++ b/docs/includes/installation.txt @@ -86,6 +86,9 @@ Transports and Backends :celery[couchbase]: for using CouchBase as a result backend. +:celery[elasticsearch] + for using Elasticsearch as a result backend. + :celery[riak]: for using Riak as a result backend. diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 69ea7a113..2c37e4a4f 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -133,7 +133,7 @@ It supports… - AMQP, Redis - memcached, MongoDB - SQLAlchemy, Django ORM - - Apache Cassandra, IronCache + - Apache Cassandra, IronCache, Elasticsearch - **Serialization** diff --git a/docs/internals/reference/celery.backends.elasticsearch.txt b/docs/internals/reference/celery.backends.elasticsearch.txt new file mode 100644 index 000000000..ae06fa19f --- /dev/null +++ b/docs/internals/reference/celery.backends.elasticsearch.txt @@ -0,0 +1,11 @@ +=========================================== + celery.backends.elasticsearch +=========================================== + +.. contents:: + :local: +.. currentmodule:: celery.backends.elasticsearch + +.. automodule:: celery.backends.elasticsearch + :members: + :undoc-members: diff --git a/docs/internals/reference/index.rst b/docs/internals/reference/index.rst index 34b513902..d7329cd2e 100644 --- a/docs/internals/reference/index.rst +++ b/docs/internals/reference/index.rst @@ -30,6 +30,7 @@ celery.backends.cache celery.backends.couchdb celery.backends.mongodb + celery.backends.elasticsearch celery.backends.redis celery.backends.riak celery.backends.cassandra diff --git a/setup.py b/setup.py index 8af1a1e25..8f9d3f62e 100644 --- a/setup.py +++ b/setup.py @@ -196,7 +196,7 @@ def extras(*p): # Celery specific features = set([ - 'auth', 'cassandra', 'memcache', 'couchbase', 'threads', + 'auth', 'cassandra', 'elasticsearch', 'memcache', 'couchbase', 'threads', 'eventlet', 'gevent', 'msgpack', 'yaml', 'redis', 'mongodb', 'sqs', 'couchdb', 'riak', 'beanstalk', 'zookeeper', 'zeromq', 'sqlalchemy', 'librabbitmq', 'pyro', 'slmq', From 5295ef8ff5f3aa79e5944dc46ed135346003006a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 26 Feb 2016 18:26:33 -0800 Subject: [PATCH 1017/1103] [task] Raise if countdown/expires is less than INT_MIN. Closes #3078 --- celery/app/amqp.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 518681d4c..455cb5597 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -35,6 +35,9 @@ PY3 = sys.version_info[0] == 3 +#: earliest date supported by time.mktime. +INT_MIN = -2147483648 + # json in Python 2.7 borks if dict contains byte keys. JSON_NEEDS_UNICODE_KEYS = not PY3 and not try_import('simplejson') @@ -313,12 +316,14 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, if not isinstance(kwargs, Mapping): raise TypeError('task keyword arguments must be a mapping') if countdown: # convert countdown to ETA + self._verify_seconds(countdown, 'countdown') now = now or self.app.now() timezone = timezone or self.app.timezone eta = maybe_make_aware( now + timedelta(seconds=countdown), tz=timezone, ) if isinstance(expires, numbers.Real): + self._verify_seconds(expires, 'expires') now = now or self.app.now() timezone = timezone or self.app.timezone expires = maybe_make_aware( @@ -394,12 +399,14 @@ def as_task_v1(self, task_id, name, args=None, kwargs=None, if not isinstance(kwargs, Mapping): raise ValueError('task keyword arguments must be a mapping') if countdown: # convert countdown to ETA + self._verify_seconds(countdown, 'countdown') now = now or self.app.now() timezone = timezone or self.app.timezone eta = now + timedelta(seconds=countdown) if utc: eta = to_utc(eta).astimezone(timezone) if isinstance(expires, numbers.Real): + self._verify_seconds(expires, 'expires') now = now or self.app.now() timezone = timezone or self.app.timezone expires = now + timedelta(seconds=expires) @@ -449,6 +456,11 @@ def as_task_v1(self, task_id, name, args=None, kwargs=None, } if create_sent_event else None, ) + def _verify_seconds(self, s, what): + if s < INT_MIN: + raise ValueError('%s is out of range: %r' % (what, s)) + return s + def _create_task_sender(self): default_retry = self.app.conf.task_publish_retry default_policy = self.app.conf.task_publish_retry_policy From 0b488c0779ca411c81f8791e03a9d4b1d18a0e0f Mon Sep 17 00:00:00 2001 From: Ahmet Demir Date: Sat, 27 Feb 2016 15:43:38 +0100 Subject: [PATCH 1018/1103] contribute #2828 --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 54c8b06d5..da16a006f 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -204,3 +204,4 @@ Valentyn Klindukh, 2016/01/15 Wayne Chang, 2016/01/15 Mike Attwood, 2016/01/22 David Harrigan, 2016/02/01 +Ahmet Demir, 2016/02/27 From 61a2427fa5442cd5ae884758ffbe1aee2f73e56a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 29 Feb 2016 13:41:53 -0800 Subject: [PATCH 1019/1103] [utils] Stop argument to mro_lookup is now a set --- celery/app/base.py | 2 +- celery/app/trace.py | 2 +- celery/tests/worker/test_request.py | 2 +- celery/utils/objects.py | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 2d662e0ea..f3816ac0e 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -71,7 +71,7 @@ def app_has_custom(app, attr): - return mro_lookup(app.__class__, attr, stop=(Celery, object), + return mro_lookup(app.__class__, attr, stop={Celery, object}, monkey_patched=[__name__]) diff --git a/celery/app/trace.py b/celery/app/trace.py index 7fd459f01..e29d9d990 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -117,7 +117,7 @@ def task_has_custom(task, attr): """Return true if the task or one of its bases defines ``attr`` (excluding the one in BaseTask).""" - return mro_lookup(task.__class__, attr, stop=(BaseTask, object), + return mro_lookup(task.__class__, attr, stop={BaseTask, object}, monkey_patched=['celery.app.task']) diff --git a/celery/tests/worker/test_request.py b/celery/tests/worker/test_request.py index 5b50ff389..72c4a7d41 100644 --- a/celery/tests/worker/test_request.py +++ b/celery/tests/worker/test_request.py @@ -107,7 +107,7 @@ def mro(cls): A.x = 10 self.assertEqual(mro_lookup(C, 'x'), A) - self.assertIsNone(mro_lookup(C, 'x', stop=(A,))) + self.assertIsNone(mro_lookup(C, 'x', stop={A})) B.x = 10 self.assertEqual(mro_lookup(C, 'x'), B) C.x = 10 diff --git a/celery/utils/objects.py b/celery/utils/objects.py index 8a2f7f639..f6bd0ba28 100644 --- a/celery/utils/objects.py +++ b/celery/utils/objects.py @@ -18,7 +18,7 @@ def __init__(self, **kwargs): self.__dict__.update(kwargs) -def mro_lookup(cls, attr, stop=(), monkey_patched=[]): +def mro_lookup(cls, attr, stop=set(), monkey_patched=[]): """Return the first node by MRO order that defines an attribute. :keyword stop: A list of types that if reached will stop the search. @@ -32,8 +32,8 @@ def mro_lookup(cls, attr, stop=(), monkey_patched=[]): for node in cls.mro(): if node in stop: try: - attr = node.__dict__[attr] - module_origin = attr.__module__ + value = node.__dict__[attr] + module_origin = value.__module__ except (AttributeError, KeyError): pass else: From f8bcdfed799e0e041118b19d450104e0eb761d9c Mon Sep 17 00:00:00 2001 From: m-vdb Date: Thu, 25 Feb 2016 14:32:32 -0800 Subject: [PATCH 1020/1103] [Results] Adds new Backend.as_uri() This can be used to get the URL used when configuring the backend, and also supports an include_password argument that if set to False sanitizes the URL for use in logs, etc. The :program:`celery worker` startup banner is updated to use this for sanitization. Closes #3079 Closes #3045 Closes #3049 Closes #3068 Closes #3073 --- celery/apps/worker.py | 5 +---- celery/backends/base.py | 15 ++++++++++++++- celery/backends/cache.py | 9 +++++++++ celery/backends/mongodb.py | 19 ++++++++++++++++--- celery/tests/backends/test_base.py | 18 ++++++++++++++++++ celery/tests/backends/test_cache.py | 22 ++++++++++++++++++++-- celery/tests/backends/test_mongodb.py | 21 ++++++++++++++++++++- celery/tests/bin/test_worker.py | 14 ++++++++++++++ 8 files changed, 112 insertions(+), 11 deletions(-) diff --git a/celery/apps/worker.py b/celery/apps/worker.py index 7198172fe..873ac0b8a 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -22,7 +22,6 @@ from billiard.process import current_process from kombu.utils.encoding import safe_str -from kombu.utils.url import maybe_sanitize_url from celery import VERSION_BANNER, platforms, signals from celery.app import trace @@ -206,9 +205,7 @@ def startup_info(self): timestamp=datetime.now().replace(microsecond=0), version=VERSION_BANNER, conninfo=self.app.connection().as_uri(), - results=maybe_sanitize_url( - self.app.conf.result_backend or 'disabled', - ), + results=self.app.backend.as_uri(), concurrency=concurrency, platform=safe_str(_platform.platform()), events=events, diff --git a/celery/backends/base.py b/celery/backends/base.py index 9030d4225..6be3ffa6f 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -24,6 +24,7 @@ registry as serializer_registry, ) from kombu.utils.encoding import bytes_to_str, ensure_bytes, from_utf8 +from kombu.utils.url import maybe_sanitize_url from celery import states from celery import current_app, group, maybe_signature @@ -93,7 +94,7 @@ class Backend(object): def __init__(self, app, serializer=None, max_cached_results=None, accept=None, - expires=None, expires_type=None, **kwargs): + expires=None, expires_type=None, url=None, **kwargs): self.app = app conf = self.app.conf self.serializer = serializer or conf.result_serializer @@ -108,6 +109,14 @@ def __init__(self, app, conf.accept_content if accept is None else accept, ) self._pending_results = {} + self.url = url + + def as_uri(self, include_password=False): + """Return the backend as an URI, sanitizing the password or not""" + # when using maybe_sanitize_url(), "/" is added + # we're stripping it for consistency + return (self.url if include_password + else maybe_sanitize_url(self.url).rstrip("/")) def mark_as_started(self, task_id, **meta): """Mark a task as started""" @@ -682,5 +691,9 @@ def _is_disabled(self, *args, **kwargs): raise NotImplementedError( 'No result backend configured. ' 'Please see the documentation for more information.') + + def as_uri(self, *args, **kwargs): + return 'disabled://' + get_state = get_status = get_result = get_traceback = _is_disabled wait_for = get_many = _is_disabled diff --git a/celery/backends/cache.py b/celery/backends/cache.py index 9d8f7c97e..7da40bac2 100644 --- a/celery/backends/cache.py +++ b/celery/backends/cache.py @@ -149,3 +149,12 @@ def __reduce__(self, args=(), kwargs={}): expires=self.expires, options=self.options)) return super(CacheBackend, self).__reduce__(args, kwargs) + + def as_uri(self, *args, **kwargs): + """ + Return the backend as an URI. It properly handles the + case of multiple servers. It doesn't try to sanitize + password because memcached URIs doesn't support them. + """ + servers = ';'.join(self.servers) + return '{0}://{1}/'.format(self.backend, servers) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 2f755a24a..c4d6c18b5 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -11,6 +11,7 @@ from datetime import datetime, timedelta from kombu.utils import cached_property +from kombu.utils.url import maybe_sanitize_url from kombu.exceptions import EncodeError from celery import states from celery.exceptions import ImproperlyConfigured @@ -55,7 +56,7 @@ class MongoBackend(BaseBackend): _connection = None - def __init__(self, app=None, url=None, **kwargs): + def __init__(self, app=None, **kwargs): """Initialize MongoDB backend instance. :raises celery.exceptions.ImproperlyConfigured: if @@ -71,8 +72,6 @@ def __init__(self, app=None, url=None, **kwargs): 'You need to install the pymongo library to use the ' 'MongoDB backend.') - self.url = url - # Set option defaults for key, value in items(self._prepare_client_options()): self.options.setdefault(key, value) @@ -295,3 +294,17 @@ def group_collection(self): @cached_property def expires_delta(self): return timedelta(seconds=self.expires) + + def as_uri(self, include_password=False): + """ + Return the backend as an URI, sanitizing the password or not. + It properly handles the case of a replica set. + """ + if include_password: + return self.url + + if "," not in self.url: + return maybe_sanitize_url(self.url).rstrip("/") + + uri1, remainder = self.url.split(",", 1) + return ",".join([maybe_sanitize_url(uri1).rstrip("/"), remainder]) diff --git a/celery/tests/backends/test_base.py b/celery/tests/backends/test_base.py index 226bb0d7a..c0e01afc0 100644 --- a/celery/tests/backends/test_base.py +++ b/celery/tests/backends/test_base.py @@ -584,3 +584,21 @@ def test_store_result(self): def test_is_disabled(self): with self.assertRaises(NotImplementedError): DisabledBackend(self.app).get_state('foo') + + def test_as_uri(self): + self.assertEqual(DisabledBackend(self.app).as_uri(), 'disabled://') + + +class test_as_uri(AppCase): + + def setup(self): + self.b = BaseBackend( + app=self.app, + url="sch://uuuu:pwpw@hostname.dom" + ) + + def test_as_uri_include_password(self): + self.assertEqual(self.b.as_uri(True), "sch://uuuu:pwpw@hostname.dom") + + def test_as_uri_exclude_password(self): + self.assertEqual(self.b.as_uri(), "sch://uuuu:**@hostname.dom") diff --git a/celery/tests/backends/test_cache.py b/celery/tests/backends/test_cache.py index ee3291295..9fb5053c7 100644 --- a/celery/tests/backends/test_cache.py +++ b/celery/tests/backends/test_cache.py @@ -5,12 +5,12 @@ from contextlib import contextmanager -from kombu.utils.encoding import str_to_bytes +from kombu.utils.encoding import str_to_bytes, ensure_bytes from celery import signature from celery import states from celery import group -from celery.backends.cache import CacheBackend, DummyClient +from celery.backends.cache import CacheBackend, DummyClient, backends from celery.exceptions import ImproperlyConfigured from celery.five import items, string, text_t from celery.utils import uuid @@ -34,6 +34,11 @@ def setup(self): self.app.conf.result_serializer = 'pickle' self.tb = CacheBackend(backend='memory://', app=self.app) self.tid = uuid() + self.old_get_best_memcached = backends['memcache'] + backends['memcache'] = lambda: (DummyClient, ensure_bytes) + + def teardown(self): + backends['memcache'] = self.old_get_best_memcached def test_no_backend(self): self.app.conf.cache_backend = None @@ -118,6 +123,19 @@ def test_unknown_backend_raises_ImproperlyConfigured(self): with self.assertRaises(ImproperlyConfigured): CacheBackend(backend='unknown://', app=self.app) + def test_as_uri_no_servers(self): + self.assertEqual(self.tb.as_uri(), 'memory:///') + + def test_as_uri_one_server(self): + backend = 'memcache://127.0.0.1:11211/' + b = CacheBackend(backend=backend, app=self.app) + self.assertEqual(b.as_uri(), backend) + + def test_as_uri_multiple_servers(self): + backend = 'memcache://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/' + b = CacheBackend(backend=backend, app=self.app) + self.assertEqual(b.as_uri(), backend) + class MyMemcachedStringEncodingError(Exception): pass diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index d2fa023bd..a8b4164a3 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -31,6 +31,11 @@ class test_MongoBackend(AppCase): + default_url = "mongodb://uuuu:pwpw@hostname.dom/database" + replica_set_url = "mongodb://uuuu:pwpw@hostname.dom,hostname.dom/database?replicaSet=rs" + sanitized_default_url = default_url.replace("pwpw", "**") + sanitized_replica_set_url = replica_set_url.replace("pwpw", "**") + def setup(self): if pymongo is None: raise SkipTest('pymongo is not installed.') @@ -41,7 +46,7 @@ def setup(self): R['Binary'], module.Binary = module.Binary, Mock() R['datetime'], datetime.datetime = datetime.datetime, Mock() - self.backend = MongoBackend(app=self.app) + self.backend = MongoBackend(app=self.app, url=self.default_url) def teardown(self): MongoBackend.encode = self._reset['encode'] @@ -385,6 +390,20 @@ def test_prepare_client_options(self): 'maxPoolSize': self.backend.max_pool_size }) + def test_as_uri_include_password(self): + self.assertEqual(self.backend.as_uri(True), self.default_url) + + def test_as_uri_exclude_password(self): + self.assertEqual(self.backend.as_uri(), self.sanitized_default_url) + + def test_as_uri_include_password_replica_set(self): + backend = MongoBackend(app=self.app, url=self.replica_set_url) + self.assertEqual(backend.as_uri(True), self.replica_set_url) + + def test_as_uri_exclude_password_replica_set(self): + backend = MongoBackend(app=self.app, url=self.replica_set_url) + self.assertEqual(backend.as_uri(), self.sanitized_replica_set_url) + class test_MongoBackend_no_mock(AppCase): diff --git a/celery/tests/bin/test_worker.py b/celery/tests/bin/test_worker.py index 75e58cb7d..98ffdf0ca 100644 --- a/celery/tests/bin/test_worker.py +++ b/celery/tests/bin/test_worker.py @@ -209,6 +209,20 @@ def test_startup_info(self): finally: cd.ARTLINES = prev + @disable_stdouts + def test_startup_info_mongo_result_backend(self): + self.app.conf.result_backend = "mongodb://user:password@host0.com:43437,host1.com:43437/work4us?replicaSet=rs&ssl=true" + worker = self.Worker(app=self.app) + worker.on_start() + self.assertTrue(worker.startup_info()) + + @disable_stdouts + def test_startup_info_memcached_result_backend(self): + self.app.conf.result_backend = "cache+memcached://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/" + worker = self.Worker(app=self.app) + worker.on_start() + self.assertTrue(worker.startup_info()) + @disable_stdouts def test_run(self): self.Worker(app=self.app).on_start() From 426292262a8976ea85822d80c8e6b55100691d79 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 29 Feb 2016 13:55:47 -0800 Subject: [PATCH 1021/1103] Cosmetics for #3079 --- celery/tests/backends/test_cache.py | 11 +++++++- celery/tests/backends/test_mongodb.py | 17 ++++++++++-- celery/tests/bin/test_worker.py | 37 +-------------------------- celery/tests/case.py | 16 ++++++++++-- 4 files changed, 40 insertions(+), 41 deletions(-) diff --git a/celery/tests/backends/test_cache.py b/celery/tests/backends/test_cache.py index 9fb5053c7..3b95fe0c3 100644 --- a/celery/tests/backends/test_cache.py +++ b/celery/tests/backends/test_cache.py @@ -16,7 +16,7 @@ from celery.utils import uuid from celery.tests.case import ( - AppCase, Mock, mask_modules, patch, reset_modules, + AppCase, Mock, disable_stdouts, mask_modules, patch, reset_modules, ) PY3 = sys.version_info[0] == 3 @@ -136,6 +136,15 @@ def test_as_uri_multiple_servers(self): b = CacheBackend(backend=backend, app=self.app) self.assertEqual(b.as_uri(), backend) + @disable_stdouts + def test_regression_worker_startup_info(self): + self.app.conf.result_backend = ( + "cache+memcached://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/" + ) + worker = self.app.Worker() + worker.on_start() + self.assertTrue(worker.startup_info()) + class MyMemcachedStringEncodingError(Exception): pass diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index a8b4164a3..1dfb12b7a 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -15,7 +15,7 @@ from celery.exceptions import ImproperlyConfigured from celery.tests.case import ( AppCase, MagicMock, Mock, SkipTest, ANY, - depends_on_current_app, patch, sentinel, + depends_on_current_app, disable_stdouts, patch, sentinel, ) COLLECTION = 'taskmeta_celery' @@ -32,7 +32,10 @@ class test_MongoBackend(AppCase): default_url = "mongodb://uuuu:pwpw@hostname.dom/database" - replica_set_url = "mongodb://uuuu:pwpw@hostname.dom,hostname.dom/database?replicaSet=rs" + replica_set_url = ( + "mongodb://uuuu:pwpw@hostname.dom," + "hostname.dom/database?replicaSet=rs" + ) sanitized_default_url = default_url.replace("pwpw", "**") sanitized_replica_set_url = replica_set_url.replace("pwpw", "**") @@ -404,6 +407,16 @@ def test_as_uri_exclude_password_replica_set(self): backend = MongoBackend(app=self.app, url=self.replica_set_url) self.assertEqual(backend.as_uri(), self.sanitized_replica_set_url) + @disable_stdouts + def test_regression_worker_startup_info(self): + self.app.conf.result_backend = ( + "mongodb://user:password@host0.com:43437,host1.com:43437" + "/work4us?replicaSet=rs&ssl=true" + ) + worker = self.app.Worker() + worker.on_start() + self.assertTrue(worker.startup_info()) + class test_MongoBackend_no_mock(AppCase): diff --git a/celery/tests/bin/test_worker.py b/celery/tests/bin/test_worker.py index 98ffdf0ca..c69c9502b 100644 --- a/celery/tests/bin/test_worker.py +++ b/celery/tests/bin/test_worker.py @@ -4,8 +4,6 @@ import os import sys -from functools import wraps - from billiard.process import current_process from kombu import Exchange, Queue @@ -24,7 +22,7 @@ AppCase, Mock, SkipTest, - WhateverIO, + disable_stdouts, patch, skip_if_pypy, skip_if_jython, @@ -38,25 +36,6 @@ def tearDown(self): trace.reset_worker_optimizations() -def disable_stdouts(fun): - - @wraps(fun) - def disable(*args, **kwargs): - prev_out, prev_err = sys.stdout, sys.stderr - prev_rout, prev_rerr = sys.__stdout__, sys.__stderr__ - sys.stdout = sys.__stdout__ = WhateverIO() - sys.stderr = sys.__stderr__ = WhateverIO() - try: - return fun(*args, **kwargs) - finally: - sys.stdout = prev_out - sys.stderr = prev_err - sys.__stdout__ = prev_rout - sys.__stderr__ = prev_rerr - - return disable - - class Worker(cd.Worker): redirect_stdouts = False @@ -209,20 +188,6 @@ def test_startup_info(self): finally: cd.ARTLINES = prev - @disable_stdouts - def test_startup_info_mongo_result_backend(self): - self.app.conf.result_backend = "mongodb://user:password@host0.com:43437,host1.com:43437/work4us?replicaSet=rs&ssl=true" - worker = self.Worker(app=self.app) - worker.on_start() - self.assertTrue(worker.startup_info()) - - @disable_stdouts - def test_startup_info_memcached_result_backend(self): - self.app.conf.result_backend = "cache+memcached://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/" - worker = self.Worker(app=self.app) - worker.on_start() - self.assertTrue(worker.startup_info()) - @disable_stdouts def test_run(self): self.Worker(app=self.app).on_start() diff --git a/celery/tests/case.py b/celery/tests/case.py index c93e6bbaf..da19a4ff5 100644 --- a/celery/tests/case.py +++ b/celery/tests/case.py @@ -715,6 +715,7 @@ def myimp(name, *args, **kwargs): def override_stdouts(): """Override `sys.stdout` and `sys.stderr` with `WhateverIO`.""" prev_out, prev_err = sys.stdout, sys.stderr + prev_rout, prev_rerr = sys.__stdout__, sys.__stderr__ mystdout, mystderr = WhateverIO(), WhateverIO() sys.stdout = sys.__stdout__ = mystdout sys.stderr = sys.__stderr__ = mystderr @@ -722,8 +723,19 @@ def override_stdouts(): try: yield mystdout, mystderr finally: - sys.stdout = sys.__stdout__ = prev_out - sys.stderr = sys.__stderr__ = prev_err + sys.stdout = prev_out + sys.stderr = prev_err + sys.__stdout__ = prev_rout + sys.__stderr__ = prev_rerr + + +def disable_stdouts(fun): + + @wraps(fun) + def disable(*args, **kwargs): + with override_stdouts(): + return fun(*args, **kwargs) + return disable def _old_patch(module, name, mocked): From d86ad1ff36d7c9d85566f1dc792f63ce6b8887cb Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 29 Feb 2016 14:28:14 -0800 Subject: [PATCH 1022/1103] [Travis] Adds pymemcache requirement to run cache backend tests Issue #3079 --- docs/includes/installation.txt | 5 ++++- requirements/extras/pymemcache.txt | 1 + requirements/test-ci-base.txt | 1 + setup.py | 6 +++--- 4 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 requirements/extras/pymemcache.txt diff --git a/docs/includes/installation.txt b/docs/includes/installation.txt index fffd8c178..25ae7eef9 100644 --- a/docs/includes/installation.txt +++ b/docs/includes/installation.txt @@ -75,7 +75,10 @@ Transports and Backends for using Amazon SQS as a message transport (*experimental*). :celery[memcache]: - for using memcached as a result backend. + for using memcached as a result backend (using pylibmc) + +:celery[pymemcache]: + for using memcached as a result backend (pure-python implementation). :celery[cassandra]: for using Apache Cassandra as a result backend with DataStax driver. diff --git a/requirements/extras/pymemcache.txt b/requirements/extras/pymemcache.txt new file mode 100644 index 000000000..851bfd86d --- /dev/null +++ b/requirements/extras/pymemcache.txt @@ -0,0 +1 @@ +python-memcached diff --git a/requirements/test-ci-base.txt b/requirements/test-ci-base.txt index 9f14178c3..71fbfea0e 100644 --- a/requirements/test-ci-base.txt +++ b/requirements/test-ci-base.txt @@ -3,4 +3,5 @@ codecov -r extras/redis.txt -r extras/mongodb.txt -r extras/sqlalchemy.txt +-r extras/pymemcache.txt -r dev.txt diff --git a/setup.py b/setup.py index 8f9d3f62e..da34e97c1 100644 --- a/setup.py +++ b/setup.py @@ -196,9 +196,9 @@ def extras(*p): # Celery specific features = set([ - 'auth', 'cassandra', 'elasticsearch', 'memcache', 'couchbase', 'threads', - 'eventlet', 'gevent', 'msgpack', 'yaml', 'redis', - 'mongodb', 'sqs', 'couchdb', 'riak', 'beanstalk', 'zookeeper', + 'auth', 'cassandra', 'elasticsearch', 'memcache', 'pymemcache', + 'couchbase', 'threads', 'eventlet', 'gevent', 'msgpack', 'yaml', + 'redis', 'mongodb', 'sqs', 'couchdb', 'riak', 'beanstalk', 'zookeeper', 'zeromq', 'sqlalchemy', 'librabbitmq', 'pyro', 'slmq', ]) extras_require = dict((x, extras(x + '.txt')) for x in features) From ec1ad9c2dbc83a4be2bd81ffbbd1e5da518dc44a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 29 Feb 2016 14:49:09 -0800 Subject: [PATCH 1023/1103] Cosmetics for #3079 --- celery/backends/cache.py | 8 ++++---- celery/backends/mongodb.py | 25 +++++++++++++------------ celery/tests/backends/test_base.py | 6 +++--- celery/tests/backends/test_cache.py | 2 +- celery/tests/backends/test_mongodb.py | 14 +++++++------- 5 files changed, 28 insertions(+), 27 deletions(-) diff --git a/celery/backends/cache.py b/celery/backends/cache.py index 7da40bac2..005737845 100644 --- a/celery/backends/cache.py +++ b/celery/backends/cache.py @@ -151,10 +151,10 @@ def __reduce__(self, args=(), kwargs={}): return super(CacheBackend, self).__reduce__(args, kwargs) def as_uri(self, *args, **kwargs): - """ - Return the backend as an URI. It properly handles the - case of multiple servers. It doesn't try to sanitize - password because memcached URIs doesn't support them. + """Return the backend as an URI. + + This properly handles the case of multiple servers. + """ servers = ';'.join(self.servers) return '{0}://{1}/'.format(self.backend, servers) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index c4d6c18b5..e48a68371 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -84,7 +84,7 @@ def __init__(self, app=None, **kwargs): uri_data = pymongo.uri_parser.parse_uri(self.url) # build the hosts list to create a mongo connection hostslist = [ - "{0}:{1}".format(x[0], x[1]) for x in uri_data['nodelist'] + '{0}:{1}'.format(x[0], x[1]) for x in uri_data['nodelist'] ] self.user = uri_data['username'] self.password = uri_data['password'] @@ -230,11 +230,11 @@ def _delete_group(self, group_id): self.group_collection.remove({'_id': group_id}) def _forget(self, task_id): - """ - Remove result from MongoDB. + """Remove result from MongoDB. + + :raises celery.exceptions.OperationsError: + if the task_id could not be removed. - :raises celery.exceptions.OperationsError: if the task_id could not be - removed. """ # By using safe=True, this will wait until it receives a response from # the server. Likewise, it will raise an OperationsError if the @@ -296,15 +296,16 @@ def expires_delta(self): return timedelta(seconds=self.expires) def as_uri(self, include_password=False): - """ - Return the backend as an URI, sanitizing the password or not. - It properly handles the case of a replica set. + """Return the backend as an URI. + + :keyword include_password: Censor passwords. + """ if include_password: return self.url - if "," not in self.url: - return maybe_sanitize_url(self.url).rstrip("/") + if ',' not in self.url: + return maybe_sanitize_url(self.url).rstrip('/') - uri1, remainder = self.url.split(",", 1) - return ",".join([maybe_sanitize_url(uri1).rstrip("/"), remainder]) + uri1, remainder = self.url.split(',', 1) + return ','.join([maybe_sanitize_url(uri1).rstrip('/'), remainder]) diff --git a/celery/tests/backends/test_base.py b/celery/tests/backends/test_base.py index c0e01afc0..fa6a5bac7 100644 --- a/celery/tests/backends/test_base.py +++ b/celery/tests/backends/test_base.py @@ -594,11 +594,11 @@ class test_as_uri(AppCase): def setup(self): self.b = BaseBackend( app=self.app, - url="sch://uuuu:pwpw@hostname.dom" + url='sch://uuuu:pwpw@hostname.dom' ) def test_as_uri_include_password(self): - self.assertEqual(self.b.as_uri(True), "sch://uuuu:pwpw@hostname.dom") + self.assertEqual(self.b.as_uri(True), 'sch://uuuu:pwpw@hostname.dom') def test_as_uri_exclude_password(self): - self.assertEqual(self.b.as_uri(), "sch://uuuu:**@hostname.dom") + self.assertEqual(self.b.as_uri(), 'sch://uuuu:**@hostname.dom') diff --git a/celery/tests/backends/test_cache.py b/celery/tests/backends/test_cache.py index 3b95fe0c3..b888e85ec 100644 --- a/celery/tests/backends/test_cache.py +++ b/celery/tests/backends/test_cache.py @@ -139,7 +139,7 @@ def test_as_uri_multiple_servers(self): @disable_stdouts def test_regression_worker_startup_info(self): self.app.conf.result_backend = ( - "cache+memcached://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/" + 'cache+memcached://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/' ) worker = self.app.Worker() worker.on_start() diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index 1dfb12b7a..fed11b207 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -31,13 +31,13 @@ class test_MongoBackend(AppCase): - default_url = "mongodb://uuuu:pwpw@hostname.dom/database" + default_url = 'mongodb://uuuu:pwpw@hostname.dom/database' replica_set_url = ( - "mongodb://uuuu:pwpw@hostname.dom," - "hostname.dom/database?replicaSet=rs" + 'mongodb://uuuu:pwpw@hostname.dom,' + 'hostname.dom/database?replicaSet=rs' ) - sanitized_default_url = default_url.replace("pwpw", "**") - sanitized_replica_set_url = replica_set_url.replace("pwpw", "**") + sanitized_default_url = default_url.replace('pwpw', '**') + sanitized_replica_set_url = replica_set_url.replace('pwpw', '**') def setup(self): if pymongo is None: @@ -410,8 +410,8 @@ def test_as_uri_exclude_password_replica_set(self): @disable_stdouts def test_regression_worker_startup_info(self): self.app.conf.result_backend = ( - "mongodb://user:password@host0.com:43437,host1.com:43437" - "/work4us?replicaSet=rs&ssl=true" + 'mongodb://user:password@host0.com:43437,host1.com:43437' + '/work4us?replicaSet=rs&ssl=true' ) worker = self.app.Worker() worker.on_start() From 10d34b471dafe321cd25006035941b9b6b683572 Mon Sep 17 00:00:00 2001 From: Maxime Vdb Date: Mon, 29 Feb 2016 14:55:10 -0800 Subject: [PATCH 1024/1103] update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index da16a006f..5ddcf8ca9 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -205,3 +205,4 @@ Wayne Chang, 2016/01/15 Mike Attwood, 2016/01/22 David Harrigan, 2016/02/01 Ahmet Demir, 2016/02/27 +Maxime Verger, 2016/02/29 From db35ccd14d8803822ed04b606014f4416eb136fd Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 29 Feb 2016 16:36:36 -0800 Subject: [PATCH 1025/1103] [Stress] Support both 3.1 and 4.0 --- funtests/stress/stress/app.py | 21 ++++++++++++++++++--- funtests/stress/stress/templates.py | 29 +++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/funtests/stress/stress/app.py b/funtests/stress/stress/app.py index ac35f0cfe..fbf283a47 100644 --- a/funtests/stress/stress/app.py +++ b/funtests/stress/stress/app.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals +import celery import os import sys import signal from time import sleep -from celery import Celery from celery import signals from celery.bin.base import Option from celery.exceptions import SoftTimeLimitExceeded @@ -17,8 +17,10 @@ logger = get_task_logger(__name__) +IS_CELERY_4 = celery.VERSION[0] >= 4 -class App(Celery): + +class App(celery.Celery): template_selected = False def __init__(self, *args, **kwargs): @@ -33,7 +35,8 @@ def __init__(self, *args, **kwargs): ) ) signals.user_preload_options.connect(self.on_preload_parsed) - self.on_configure.connect(self._maybe_use_default_template) + if IS_CELERY_4: + self.on_configure.connect(self._maybe_use_default_template) def on_preload_parsed(self, options=None, **kwargs): self.use_template(options['template']) @@ -48,6 +51,18 @@ def _maybe_use_default_template(self, **kwargs): if not self.template_selected: self.use_template('default') + if not IS_CELERY_4: + after_configure = None + + def _get_config(self): + ret = super(App, self)._get_config() + if self.after_configure: + self.after_configure(ret) + return ret + + def on_configure(self): + self._maybe_use_default_template() + app = App('stress', set_as_current=False) diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index bc5cb7ff9..741fe14da 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -1,9 +1,12 @@ from __future__ import absolute_import +import celery import os +from functools import partial + from celery.five import items -from kombu import Exchange, Queue +from kombu import Queue from kombu.utils import symbol_by_name CSTRESS_TRANS = os.environ.get('CSTRESS_TRANS', False) @@ -12,6 +15,8 @@ templates = {} +IS_CELERY_4 = celery.VERSION[0] >= 4 + def template(name=None): @@ -21,15 +26,23 @@ def _register(cls): return _register -def use_template(app, template='default'): - template = template.split(',') +if IS_CELERY_4: + + def use_template(app, template='default'): + template = template.split(',') + + # mixin the rest of the templates when the config is needed + @app.on_after_configure.connect(weak=False) + def load_template(sender, source, **kwargs): + mixin_templates(template[1:], source) - # mixin the rest of the templates when the config is needed - @app.on_after_configure.connect(weak=False) - def load_template(sender, source, **kwargs): - mixin_templates(template[1:], source) + app.config_from_object(templates[template[0]]) +else: - app.config_from_object(templates[template[0]]) + def use_template(app, template='default'): # noqa + template = template.split(',') + app.after_configure = partial(mixin_templates, template[1:]) + app.config_from_object(templates[template[0]]) def mixin_templates(templates, conf): From 531bb97e89848c61c94c31160c39a6ef51a60037 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 29 Feb 2016 16:37:25 -0800 Subject: [PATCH 1026/1103] Adds .vagrant/ to .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0f856d445..70d602b25 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,4 @@ Documentation/ celery/tests/cover/ .ve* cover/ - +.vagrant/ From 6aa33f71c4225f8a28140a2a8bd4f67a7d5419fb Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 1 Mar 2016 11:51:39 -0800 Subject: [PATCH 1027/1103] [asynpool] Include exception/traceback in "process inqueue damaged" error --- celery/concurrency/asynpool.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 4b9aeff67..0f549475e 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -794,8 +794,9 @@ def send_job(tup): put_message(job) self._quick_put = send_job - def on_not_recovering(proc, fd, job): - error('Process inqueue damaged: %r %r' % (proc, proc.exitcode)) + def on_not_recovering(proc, fd, job, exc): + error('Process inqueue damaged: %r %r: %r', + proc, proc.exitcode, exc, exc_info=1)) if proc._is_alive(): proc.terminate() hub.remove(fd) @@ -824,7 +825,7 @@ def _write_job(proc, fd, job): # suspend until more data errors += 1 if errors > 100: - on_not_recovering(proc, fd, job) + on_not_recovering(proc, fd, job, exc) raise StopIteration() yield else: @@ -840,7 +841,7 @@ def _write_job(proc, fd, job): # suspend until more data errors += 1 if errors > 100: - on_not_recovering(proc, fd, job) + on_not_recovering(proc, fd, job, exc) raise StopIteration() yield else: From 16a17c976d1b59599c2a0e5372f5270569dbea56 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 1 Mar 2016 13:32:07 -0800 Subject: [PATCH 1028/1103] [Stress] Provision script now installs htop --- funtests/stress/run/provision/provision.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/funtests/stress/run/provision/provision.sh b/funtests/stress/run/provision/provision.sh index d4de824d2..927261902 100644 --- a/funtests/stress/run/provision/provision.sh +++ b/funtests/stress/run/provision/provision.sh @@ -186,6 +186,7 @@ provision () { apt_update configure_system apt_install powertop + apt_install htop install_git install_rabbitmq install_redis From 8653cb61d9bdf1aed1f92ab667b050ed6fc8e6b2 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 1 Mar 2016 13:32:28 -0800 Subject: [PATCH 1029/1103] [Stress] now uses uppercase settings for 3.1 support --- funtests/stress/stress/templates.py | 69 ++++++++++++++++------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index 741fe14da..6bd2705b1 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -63,89 +63,94 @@ def template_names(): @template() class default(object): - accept_content = ['json'] - broker_url = os.environ.get('CSTRESS_BROKER', 'pyamqp://') - broker_heartbeat = 30 - result_backend = os.environ.get('CSTRESS_BACKEND', 'rpc://') - result_serializer = 'json' - result_persistent = True - result_expires = 300 - result_cache_max = 100 - task_default_queue = CSTRESS_QUEUE - task_queues = [ + CELERY_ACCEPT_CONTENT = ['json'] + BROKER_URL = os.environ.get('CSTRESS_BROKER', 'pyamqp://') + BROKER_HEARTBEAT = 30 + CELERY_RESULT_BACKEND = os.environ.get('CSTRESS_BACKEND', 'rpc://') + CELERY_RESULT_SERIALIZER = 'json' + CELERY_RESULT_PERSISTENT = True + CELERY_RESULT_EXPIRES = 300 + CELERY_MAX_CACHED_RESULTS = 100 + CELERY_DEFAULT_QUEUE = CSTRESS_QUEUE + CELERY_TASK_QUEUES = [ Queue(CSTRESS_QUEUE, durable=not CSTRESS_TRANS, no_ack=CSTRESS_TRANS), ] - task_serializer = 'json' - task_publish_retry_policy = { + CELERY_TASK_SERIALIZER = 'json' + CELERY_TASK_PUBLISH_RETRY_POLICY = { 'max_retries': 100, 'interval_max': 2, 'interval_step': 0.1, } - task_protocol = 2 + CELERY_TASK_PROTOCOL = 2 if CSTRESS_TRANS: - task_default_delivery_mode = 1 - worker_prefetch_multiplier = int(os.environ.get('CSTRESS_PREFETCH', 10)) + CELERY_DEFAULT_DELIVERY_MODE = 1 + CELERYD_PREFETCH_MULTIPLIER = int(os.environ.get('CSTRESS_PREFETCH', 10)) @template() class redis(default): - broker_url = os.environ.get('CSTRESS_BROKER', 'redis://') - broker_transport_options = { + BROKER_URL = os.environ.get('CSTRESS_BROKER', 'redis://') + BROKER_TRANSPORT_OPTIONS = { 'fanout_prefix': True, 'fanout_patterns': True, } - result_backend = os.environ.get('CSTRESS_BACKEND', 'redis://') + CELERY_RESULT_BACKEND = os.environ.get('CSTRESS_BACKEND', 'redis://') @template() class redistore(default): - result_backend = 'redis://' + CELERY_RESULT_BACKEND = 'redis://' @template() class acks_late(default): - task_acks_late = True + CELERY_ACKS_LATE = True @template() class pickle(default): - accept_content = ['pickle', 'json'] - task_serializer = 'pickle' - result_serializer = 'pickle' + CELERY_ACCEPT_CONTENT = ['pickle', 'json'] + CELERY_TASK_SERIALIZER = 'pickle' + CELERY_RESULT_SERIALIZER = 'pickle' @template() class confirms(default): - broker_url = 'pyamqp://' - broker_transport_options = {'confirm_publish': True} + BROKER_URL = 'pyamqp://' + BROKER_TRANSPORT_OPTIONS = {'confirm_publish': True} @template() class events(default): - task_send_events = True - task_send_sent_event = True + CELERY_SEND_EVENTS = True + CELERY_SEND_TASK_SENT_EVENT = True @template() class execv(default): - worker_force_execv = True + CELERYD_FORCE_EXECV = True @template() class sqs(default): - broker_url = 'sqs://' - broker_transport_options = { + BROKER_URL = 'sqs://' + BROKER_TRANSPORT_OPTIONS = { 'region': os.environ.get('AWS_REGION', 'us-east-1'), } @template() class proto1(default): - task_protocol = 1 + CELERY_TASK_PROTOCOL = 1 @template() class vagrant1(default): - broker_url = 'pyamqp://testing:t3s71ng@192.168.33.123//testing' + BROKER_URL = 'pyamqp://testing:t3s71ng@192.168.33.123//testing' + + +@template() +class vagrant1_redis(default): + BROKER_URL = 'redis://192.168.33.123' From f496b84a566e1e0b0abde01daa187aea68aeddfa Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 1 Mar 2016 13:39:59 -0800 Subject: [PATCH 1030/1103] [asynpool] Fixes SyntaxError in last commit --- celery/concurrency/asynpool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 0f549475e..1714fecaa 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -796,7 +796,7 @@ def send_job(tup): def on_not_recovering(proc, fd, job, exc): error('Process inqueue damaged: %r %r: %r', - proc, proc.exitcode, exc, exc_info=1)) + proc, proc.exitcode, exc, exc_info=1) if proc._is_alive(): proc.terminate() hub.remove(fd) From 7ee9afa170cd40d780ffdf4979afdaed3f16a5bb Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 1 Mar 2016 17:34:45 -0800 Subject: [PATCH 1031/1103] [Stress] Adds vagrant1_redis template --- funtests/stress/stress/templates.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/funtests/stress/stress/templates.py b/funtests/stress/stress/templates.py index 6bd2705b1..7d3802971 100644 --- a/funtests/stress/stress/templates.py +++ b/funtests/stress/stress/templates.py @@ -152,5 +152,6 @@ class vagrant1(default): @template() -class vagrant1_redis(default): +class vagrant1_redis(redis): BROKER_URL = 'redis://192.168.33.123' + CELERY_RESULT_BACKEND = 'redis://192.168.33.123' From ca57e722b25f8fca817084ec7562be3698c7ee02 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 1 Mar 2016 17:38:42 -0800 Subject: [PATCH 1032/1103] [Asynpool] Fixes 100% CPU loop in epoll (round 2) The billiard patch celery/billiard@4f4759b8a92c117b2694faa18f1f6d6108065773 was initially submitted to fix this problem, but on closer investigation we were only leaking file descriptors. I monkey patched os.open/socket/os.close etc to track what was going on, and I found no evidence of the code closing random sockets, instead I found out that: 1) epoll_wait always returned an error state for a Popen pipe fd. 2) the worker was trying to unregister this fd from epoll, but 3) ``epoll.unregister`` refused to do so giving an IOError(ENOENT) error. So turns out this is an epoll quirk, and the solution is to duplicate the pipe fd so that we can carefully control when it's removed from the process file descriptor table. Closes celery/celery#1845 Could fix: celery/celery#2142 celery/celery#2606 --- celery/concurrency/asynpool.py | 40 +++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 1714fecaa..ae73567f0 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -426,11 +426,28 @@ def __init__(self, processes=None, synack=False, self._timeout_handler, 'on_hard_timeout', noop, ) - def _event_process_exit(self, hub, fd): + def _event_process_exit(self, hub, proc): # This method is called whenever the process sentinel is readable. - hub.remove(fd) + self._untrack_child_process(proc, hub) self.maintain_pool() + def _track_child_process(self, proc, hub): + try: + fd = proc._sentinel_poll + except AttributeError: + # we need to duplicate the fd here to carefully + # control when the fd is removed from the process table, + # as once the original fd is closed we cannot unregister + # the fd from epoll(7) anymore, causing a 100% CPU poll loop. + fd = proc._sentinel_poll = os.dup(proc._popen.sentinel) + hub.add_reader(fd, self._event_process_exit, hub, proc) + + def _untrack_child_process(self, proc, hub): + if proc._sentinel_poll is not None: + fd, proc._sentinel_poll = proc._sentinel_poll, None + hub.remove(fd) + os.close(fd) + def register_with_event_loop(self, hub): """Registers the async pool with the current event loop.""" self._result_handler.register_with_event_loop(hub) @@ -440,8 +457,7 @@ def register_with_event_loop(self, hub): self._create_write_handlers(hub) # Add handler for when a process exits (calls maintain_pool) - [hub.add_reader(fd, self._event_process_exit, hub, fd) - for fd in self.process_sentinels] + [self._track_child_process(w, hub) for w in self._pool] # Handle_result_event is called whenever one of the # result queues are readable. [hub.add_reader(fd, self.handle_result_event, fd) @@ -528,7 +544,6 @@ def _create_process_handlers(self, hub, READ=READ, ERR=ERR): fileno_to_outq = self._fileno_to_outq fileno_to_synq = self._fileno_to_synq busy_workers = self._busy_workers - event_process_exit = self._event_process_exit handle_result_event = self.handle_result_event process_flush_queues = self.process_flush_queues waiting_to_start = self._waiting_to_start @@ -554,10 +569,9 @@ def on_process_up(proc): if job._scheduled_for and job._scheduled_for.inqW_fd == infd: job._scheduled_for = proc fileno_to_outq[proc.outqR_fd] = proc + # maintain_pool is called whenever a process exits. - add_reader( - proc.sentinel, event_process_exit, hub, proc.sentinel, - ) + self._track_child_process(proc, hub) assert not isblocking(proc.outq._reader) @@ -611,16 +625,16 @@ def on_process_down(proc): ) if inq: busy_workers.discard(inq) - remove_reader(proc.sentinel) + self._untrack_child_process(proc, hub) waiting_to_start.discard(proc) self._active_writes.discard(proc.inqW_fd) - remove_writer(proc.inqW_fd) - remove_reader(proc.outqR_fd) + remove_writer(proc.inq._writer) + remove_reader(proc.outq._reader) if proc.synqR_fd: - remove_reader(proc.synqR_fd) + remove_reader(proc.synq._reader) if proc.synqW_fd: self._active_writes.discard(proc.synqW_fd) - remove_reader(proc.synqW_fd) + remove_reader(proc.synq._writer) self.on_process_down = on_process_down def _create_write_handlers(self, hub, From fd2f712f1b279cdec2afadb7e4494448cb1fd5e7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 1 Mar 2016 18:40:00 -0800 Subject: [PATCH 1033/1103] [Stress] Remove unused periodic_task --- funtests/stress/stress/app.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/funtests/stress/stress/app.py b/funtests/stress/stress/app.py index fbf283a47..d4541961c 100644 --- a/funtests/stress/stress/app.py +++ b/funtests/stress/stress/app.py @@ -175,8 +175,3 @@ def marker(s, sep='-'): return _marker.delay(s, sep) except Exception as exc: print("Retrying marker.delay(). It failed to start: %s" % exc) - - -@app.on_after_configure.connect -def setup_periodic_tasks(sender, **kwargs): - sender.add_periodic_task(10, add.s(2, 2), expires=10) From 22eba29948f8657be440295c342434e464d6c0a0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Mar 2016 13:50:38 -0800 Subject: [PATCH 1034/1103] Use vine for promises --- celery/app/base.py | 2 +- celery/canvas.py | 2 +- celery/concurrency/asynpool.py | 3 ++- celery/result.py | 2 +- celery/tests/app/test_app.py | 2 +- celery/utils/functional.py | 2 +- celery/worker/consumer/consumer.py | 2 +- docs/conf.py | 1 + requirements/dev.txt | 3 ++- 9 files changed, 11 insertions(+), 8 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index f3816ac0e..cd8c250ac 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -16,11 +16,11 @@ from operator import attrgetter from functools import wraps -from amqp import starpromise from kombu import pools from kombu.clocks import LamportClock from kombu.common import oid_from from kombu.utils import cached_property, register_after_fork, uuid +from vine import starpromise from celery import platforms from celery import signals diff --git a/celery/canvas.py b/celery/canvas.py index db170422c..f01c12b4f 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -20,8 +20,8 @@ from operator import itemgetter from itertools import chain as _chain -from amqp.promise import barrier from kombu.utils import cached_property, fxrange, reprcall, uuid +from vine import barrier from celery._state import current_app from celery.local import try_import diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index ae73567f0..31a815019 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -33,7 +33,6 @@ from time import sleep from weakref import WeakValueDictionary, ref -from amqp import promise from billiard.pool import RUN, TERMINATE, ACK, NACK, WorkersJoined from billiard import pool as _pool from billiard.compat import buf_t, setblocking, isblocking @@ -42,6 +41,8 @@ from kombu.serialization import pickle as _pickle from kombu.utils import fxrange from kombu.utils.eventio import SELECT_BAD_FD +from vine import promise + from celery.five import Counter, items, values from celery.utils.functional import noop from celery.utils.log import get_logger diff --git a/celery/result.py b/celery/result.py index d4aae59bb..1efde9ddd 100644 --- a/celery/result.py +++ b/celery/result.py @@ -14,8 +14,8 @@ from contextlib import contextmanager from copy import copy -from amqp.promise import Thenable, promise from kombu.utils import cached_property +from vine import Thenable, promise from . import current_app from . import states diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index 7a8a415a2..70fe7351c 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -7,7 +7,7 @@ from copy import deepcopy from pickle import loads, dumps -from amqp import promise +from vine import promise from celery import Celery from celery import shared_task, current_app diff --git a/celery/utils/functional.py b/celery/utils/functional.py index c691d45a3..0084f5dd4 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -19,10 +19,10 @@ from inspect import isfunction, getargspec # noqa from itertools import chain, islice -from amqp import promise from kombu.utils.functional import ( dictfilter, lazy, maybe_evaluate, is_list, maybe_list, ) +from vine import promise from celery.five import UserDict, UserList, keys, range diff --git a/celery/worker/consumer/consumer.py b/celery/worker/consumer/consumer.py index 41ae346c1..ea2d93e96 100644 --- a/celery/worker/consumer/consumer.py +++ b/celery/worker/consumer/consumer.py @@ -17,7 +17,6 @@ from collections import defaultdict from time import sleep -from amqp.promise import ppartial, promise from billiard.common import restart_state from billiard.exceptions import RestartFreqExceeded from kombu.async.semaphore import DummyLock @@ -25,6 +24,7 @@ from kombu.syn import _detect_environment from kombu.utils.encoding import safe_repr, bytes_t from kombu.utils.limits import TokenBucket +from vine import ppartial, promise from celery import bootsteps from celery import signals diff --git a/docs/conf.py b/docs/conf.py index 867025d40..05352f36e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -76,6 +76,7 @@ def linkcode_resolve(domain, info): 'djcelery': ('http://django-celery.readthedocs.org/en/master', None), 'cyme': ('http://cyme.readthedocs.org/en/latest', None), 'amqp': ('http://amqp.readthedocs.org/en/latest', None), + 'vine': ('http://vine.readthedocs.org/en/latest', None), 'flower': ('http://flower.readthedocs.org/en/latest', None), } diff --git a/requirements/dev.txt b/requirements/dev.txt index 567243863..6d8fb7307 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,3 +1,4 @@ -https://github.com/celery/py-amqp/zipball/master https://github.com/celery/billiard/zipball/master https://github.com/celery/kombu/zipball/master +https://github.com/celery/py-amqp/zipball/master +https://github.com/celery/vine/zipball/master From da43974ebc76ab70fe3e9f7f0f663ab7f2b391c8 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Mar 2016 13:57:32 -0800 Subject: [PATCH 1035/1103] [Docs] Updates repository list in Contributing guide --- CONTRIBUTING.rst | 19 ++++++++++++++++++- docs/contributing.rst | 19 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 1b5dde68d..91160cefd 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -187,7 +187,7 @@ the developers fix the bug. A bug could be fixed by some other improvements and fixes - it might not have an existing report in the bug tracker. Make sure you're using the latest releases of -celery, billiard and kombu. +celery, billiard, kombu, amqp and vine. 5) **Collect information about the bug.** @@ -246,6 +246,7 @@ issue tracker. * Celery: http://github.com/celery/celery/issues/ * Kombu: http://github.com/celery/kombu/issues * pyamqp: http://github.com/celery/pyamqp/issues +* vine: http://github.com/celery/vine/issues * librabbitmq: http://github.com/celery/librabbitmq/issues * Django-Celery: http://github.com/celery/django-celery/issues @@ -887,6 +888,7 @@ celery :git: https://github.com/celery/celery :CI: http://travis-ci.org/#!/celery/celery +:Windows-CI: https://ci.appveyor.com/project/ask/celery :PyPI: http://pypi.python.org/pypi/celery :docs: http://docs.celeryproject.org @@ -897,6 +899,7 @@ Messaging library. :git: https://github.com/celery/kombu :CI: http://travis-ci.org/#!/celery/kombu +:Windows-CI: https://ci.appveyor.com/project/ask/kombu :PyPI: http://pypi.python.org/pypi/kombu :docs: http://kombu.readthedocs.org @@ -907,9 +910,21 @@ Python AMQP 0.9.1 client. :git: https://github.com/celery/py-amqp :CI: http://travis-ci.org/#!/celery/py-amqp +:Windows-CI: https://ci.appveyor.com/project/ask/py-amqp :PyPI: http://pypi.python.org/pypi/amqp :docs: http://amqp.readthedocs.org +vine +---- + +Promise/deferred implementation. + +:git: https://github.com/celery/vine/ +:CI: http://travis-ci.org/#!/celery/vine/ +:Windows-CI: https://ci.appveyor.com/project/ask/vine +:PyPI: http://pypi.python.org/pypi/vine +:docs: http://vine.readthedocs.org + billiard -------- @@ -917,6 +932,8 @@ Fork of multiprocessing containing improvements that will eventually be merged into the Python stdlib. :git: https://github.com/celery/billiard +:CI: http://travis-ci.org/#!/celery/billiard/ +:Windows-CI: https://ci.appveyor.com/project/ask/billiard :PyPI: http://pypi.python.org/pypi/billiard librabbitmq diff --git a/docs/contributing.rst b/docs/contributing.rst index a51c54e75..931b8883a 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -187,7 +187,7 @@ the developers fix the bug. A bug could be fixed by some other improvements and fixes - it might not have an existing report in the bug tracker. Make sure you're using the latest releases of -celery, billiard and kombu. +celery, billiard, kombu, amqp and vine. 5) **Collect information about the bug.** @@ -247,6 +247,7 @@ issue tracker. * Celery: http://github.com/celery/celery/issues/ * Kombu: http://github.com/celery/kombu/issues * pyamqp: http://github.com/celery/pyamqp/issues +* vine: http://github.com/celery/vine/issues * librabbitmq: http://github.com/celery/librabbitmq/issues * Django-Celery: http://github.com/celery/django-celery/issues @@ -916,6 +917,7 @@ celery :git: https://github.com/celery/celery :CI: http://travis-ci.org/#!/celery/celery +:Windows-CI: https://ci.appveyor.com/project/ask/celery :PyPI: http://pypi.python.org/pypi/celery :docs: http://docs.celeryproject.org @@ -926,6 +928,7 @@ Messaging library. :git: https://github.com/celery/kombu :CI: http://travis-ci.org/#!/celery/kombu +:Windows-CI: https://ci.appveyor.com/project/ask/kombu :PyPI: http://pypi.python.org/pypi/kombu :docs: http://kombu.readthedocs.org @@ -936,9 +939,21 @@ Python AMQP 0.9.1 client. :git: https://github.com/celery/py-amqp :CI: http://travis-ci.org/#!/celery/py-amqp +:Windows-CI: https://ci.appveyor.com/project/ask/py-amqp :PyPI: http://pypi.python.org/pypi/amqp :docs: http://amqp.readthedocs.org +vine +---- + +Promise/deferred implementation. + +:git: https://github.com/celery/vine/ +:CI: http://travis-ci.org/#!/celery/vine/ +:Windows-CI: https://ci.appveyor.com/project/ask/vine +:PyPI: http://pypi.python.org/pypi/vine +:docs: http://vine.readthedocs.org + billiard -------- @@ -946,6 +961,8 @@ Fork of multiprocessing containing improvements that will eventually be merged into the Python stdlib. :git: https://github.com/celery/billiard +:CI: http://travis-ci.org/#!/celery/billiard/ +:Windows-CI: https://ci.appveyor.com/project/ask/billiard :PyPI: http://pypi.python.org/pypi/billiard librabbitmq From 72ba1a4c52f241b62fad0d5660caaa4025c178fa Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Mar 2016 14:53:35 -0800 Subject: [PATCH 1036/1103] [Travis] Show logs for pip install dev.txt --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 18d35e40a..b5c1ddcde 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ deps= sitepackages = False recreate = False commands = {toxinidir}/extra/release/removepyc.sh {toxinidir} - pip install -q -U -r{toxinidir}/requirements/dev.txt + pip install -U -r{toxinidir}/requirements/dev.txt nosetests -xsv --with-coverage \ --cover-inclusive --cover-min-percentage=94 --cover-erase [] From 07035f627b9cfe81e3febf86531533f8d34991f4 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Mar 2016 14:54:26 -0800 Subject: [PATCH 1037/1103] [Worker] Refactor Mingle to be reusable --- celery/tests/worker/test_consumer.py | 74 ++++++++++++++-------------- celery/worker/consumer/mingle.py | 42 ++++++++++------ 2 files changed, 63 insertions(+), 53 deletions(-) diff --git a/celery/tests/worker/test_consumer.py b/celery/tests/worker/test_consumer.py index e41a22e22..fcd883f5a 100644 --- a/celery/tests/worker/test_consumer.py +++ b/celery/tests/worker/test_consumer.py @@ -6,7 +6,6 @@ from billiard.exceptions import RestartFreqExceeded from celery.datastructures import LimitedSet -from celery.worker import state as worker_state from celery.worker.consumer.agent import Agent from celery.worker.consumer.consumer import CLOSE, Consumer, dump_body from celery.worker.consumer.gossip import Gossip @@ -278,43 +277,42 @@ def test_start_no_replies(self): mingle.start(c) def test_start(self): - try: - c = Mock() - c.app.connection_for_read = _amqp_connection() - mingle = Mingle(c) - self.assertTrue(mingle.enabled) - - Aig = LimitedSet() - Big = LimitedSet() - Aig.add('Aig-1') - Aig.add('Aig-2') - Big.add('Big-1') - - I = c.app.control.inspect.return_value = Mock() - I.hello.return_value = { - 'A@example.com': { - 'clock': 312, - 'revoked': Aig._data, - }, - 'B@example.com': { - 'clock': 29, - 'revoked': Big._data, - }, - 'C@example.com': { - 'error': 'unknown method', - }, - } - - mingle.start(c) - I.hello.assert_called_with(c.hostname, worker_state.revoked._data) - c.app.clock.adjust.assert_has_calls([ - call(312), call(29), - ], any_order=True) - self.assertIn('Aig-1', worker_state.revoked) - self.assertIn('Aig-2', worker_state.revoked) - self.assertIn('Big-1', worker_state.revoked) - finally: - worker_state.revoked.clear() + c = Mock() + c.app.connection_for_read = _amqp_connection() + mingle = Mingle(c) + self.assertTrue(mingle.enabled) + + Aig = LimitedSet() + Big = LimitedSet() + Aig.add('Aig-1') + Aig.add('Aig-2') + Big.add('Big-1') + + I = c.app.control.inspect.return_value = Mock() + I.hello.return_value = { + 'A@example.com': { + 'clock': 312, + 'revoked': Aig._data, + }, + 'B@example.com': { + 'clock': 29, + 'revoked': Big._data, + }, + 'C@example.com': { + 'error': 'unknown method', + }, + } + + our_revoked = c.controller.state.revoked = LimitedSet() + + mingle.start(c) + I.hello.assert_called_with(c.hostname, our_revoked._data) + c.app.clock.adjust.assert_has_calls([ + call(312), call(29), + ], any_order=True) + self.assertIn('Aig-1', our_revoked) + self.assertIn('Aig-2', our_revoked) + self.assertIn('Big-1', our_revoked) def _amqp_connection(): diff --git a/celery/worker/consumer/mingle.py b/celery/worker/consumer/mingle.py index 70f07f6b3..2ca059149 100644 --- a/celery/worker/consumer/mingle.py +++ b/celery/worker/consumer/mingle.py @@ -3,11 +3,9 @@ from operator import itemgetter from celery import bootsteps -from celery.five import items, values +from celery.five import items from celery.utils.log import get_logger -from celery.worker.state import revoked - from .events import Events __all__ = ['Mingle'] @@ -15,7 +13,7 @@ MINGLE_GET_FIELDS = itemgetter('clock', 'revoked') logger = get_logger(__name__) -info = logger.info +debug, info, exception = logger.debug, logger.info, logger.exception class Mingle(bootsteps.StartStopStep): @@ -34,20 +32,34 @@ def compatible_transport(self, app): def start(self, c): info('mingle: searching for neighbors') I = c.app.control.inspect(timeout=1.0, connection=c.connection) - replies = I.hello(c.hostname, revoked._data) or {} - replies.pop(c.hostname, None) + our_revoked = c.controller.state.revoked + replies = I.hello(c.hostname, our_revoked._data) or {} + replies.pop(c.hostname, None) # delete my own response if replies: info('mingle: sync with %s nodes', len([reply for reply, value in items(replies) if value])) - for reply in values(replies): - if reply: - try: - other_clock, other_revoked = MINGLE_GET_FIELDS(reply) - except KeyError: # reply from pre-3.1 worker - pass - else: - c.app.clock.adjust(other_clock) - revoked.update(other_revoked) + [self.on_node_reply(c, nodename, reply) + for nodename, reply in items(replies) if reply] info('mingle: sync complete') else: info('mingle: all alone') + + def on_node_reply(self, c, nodename, reply): + debug('mingle: processing reply from %s', nodename) + try: + self.sync_with_node(c, **reply) + except MemoryError: + raise + except Exception as exc: + exception('mingle: sync with %s failed: %r', nodename, exc) + + def sync_with_node(self, c, clock=None, revoked=None, **kwargs): + self.on_clock_event(c, clock) + self.on_revoked_received(c, revoked) + + def on_clock_event(self, c, clock): + c.app.clock.adjust(clock) if clock else c.app.clock.forward() + + def on_revoked_received(self, c, revoked): + if revoked: + c.controller.state.revoked.update(revoked) From 0fe91e3378574e45b35a3906548031847d1f9c73 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 2 Mar 2016 16:16:25 -0800 Subject: [PATCH 1038/1103] Attempt to fix travis build --- requirements/dev.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 6d8fb7307..0f3f526b2 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,4 +1,4 @@ +https://github.com/celery/vine/zipball/master +https://github.com/celery/py-amqp/zipball/master https://github.com/celery/billiard/zipball/master https://github.com/celery/kombu/zipball/master -https://github.com/celery/py-amqp/zipball/master -https://github.com/celery/vine/zipball/master From d6fccea1432e4d65a812f49592914dc4ef854f23 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 3 Mar 2016 16:53:30 -0800 Subject: [PATCH 1039/1103] [Stress] Fixes JSON serialization of Big data. Depends on celery/kombu@b94f1bab9b76177a525b7ef0fb44621031a607fa --- funtests/stress/stress/__init__.py | 3 +- funtests/stress/stress/data.py | 65 ++++++++++++++++-------------- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/funtests/stress/stress/__init__.py b/funtests/stress/stress/__init__.py index 089130cba..d000f8a20 100644 --- a/funtests/stress/stress/__init__.py +++ b/funtests/stress/stress/__init__.py @@ -4,6 +4,8 @@ import os import time +from .data import install_json # noqa + if os.environ.get('C_SLEEP'): _orig_sleep = time.sleep @@ -15,5 +17,4 @@ def _sleep(n): _orig_sleep(n) time.sleep = _sleep - from .app import app # noqa diff --git a/funtests/stress/stress/data.py b/funtests/stress/stress/data.py index bc6b37a46..040147203 100644 --- a/funtests/stress/stress/data.py +++ b/funtests/stress/stress/data.py @@ -1,14 +1,45 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -import json - -from celery.utils.debug import humanbytes -from celery.utils.imports import qualname +try: + import simplejson as json +except ImportError: + import json # noqa type_registry = {} +class JSONEncoder(json.JSONEncoder): + + def default(self, obj): + try: + return super(JSONEncoder, self).default(obj) + except TypeError: + reducer = getattr(obj, '__to_json__', None) + if reducer: + return reducer() + raise + + +def decode_hook(d): + try: + d = d['py/obj'] + except KeyError: + return d + type_registry[d['type']](**d['attrs']) + + +def install_json(): + json._default_encoder = JSONEncoder() + json._default_decoder.object_hook = decode_hook +install_json() # ugh, ugly but it's a test suite after all + + +# this imports kombu.utils.json, so can only import after install_json() +from celery.utils.debug import humanbytes # noqa +from celery.utils.imports import qualname # noqa + + def json_reduce(obj, attrs): return {'py/obj': {'type': qualname(obj), 'attrs': attrs}} @@ -43,29 +74,3 @@ def __reduce__(self): BIG = Data('BIG', 'x' * 2 ** 20 * 8) SMALL = Data('SMALL', 'e' * 1024) - - -class JSONEncoder(json.JSONEncoder): - - def default(self, obj): - try: - return super(JSONEncoder, self).default(obj) - except TypeError: - reducer = getattr(obj, '__to_json__', None) - if reducer: - return reducer() - raise - - -def decode_hook(d): - try: - d = d['py/obj'] - except KeyError: - return d - type_registry[d['type']](**d['attrs']) - - -def install_json(): - json._default_encoder = JSONEncoder() - json._default_decoder.object_hook = decode_hook -install_json() # ugh, ugly but it's a test suite after all From d356050ccb52605e655edb477b83b1940239da71 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 3 Mar 2016 16:55:18 -0800 Subject: [PATCH 1040/1103] [Docs] Optimizing: Improves -Ofair figures --- docs/userguide/optimizing.rst | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/docs/userguide/optimizing.rst b/docs/userguide/optimizing.rst index 67cd2c964..a7c0446b5 100644 --- a/docs/userguide/optimizing.rst +++ b/docs/userguide/optimizing.rst @@ -219,15 +219,17 @@ tasks. This benefits performance but it also means that tasks may be stuck waiting for long running tasks to complete:: - -> send T1 to Process A + -> send task T1 to process A # A executes T1 - -> send T2 to Process B + -> send task T2 to process B # B executes T2 - <- T2 complete + <- T2 complete sent by process B - -> send T3 to Process A + -> send task T3 to process A # A still executing T1, T3 stuck in local buffer and will not start until # T1 returns, and other queued tasks will not be sent to idle processes + <- T1 complete sent by process A + # A executes T3 The worker will send tasks to the process as long as the pipe buffer is writable. The pipe buffer size varies based on the operating system: some may @@ -242,4 +244,17 @@ worker option: $ celery -A proj worker -l info -Ofair With this option enabled the worker will only write to processes that are -available for work, disabling the prefetch behavior. +available for work, disabling the prefetch behavior:: + +-> send task T1 to process A +# A executes T1 +-> send task T2 to process B +# B executes T2 +<- T2 complete sent by process B + +-> send T3 to process B +# B executes T3 + +<- T3 complete sent by process B +<- T1 complete sent by process A + From f91658d3517248a7b4aa90cae369d8eb14daca22 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 3 Mar 2016 16:56:27 -0800 Subject: [PATCH 1041/1103] [Prefork] Fixes memory leak in async pool on process exit. Closes #2927 --- celery/concurrency/asynpool.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 31a815019..7d6f94a0e 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -550,7 +550,9 @@ def _create_process_handlers(self, hub, READ=READ, ERR=ERR): waiting_to_start = self._waiting_to_start def verify_process_alive(proc): - if proc._is_alive() and proc in waiting_to_start: + proc = proc() # is a weakref + if (proc is not None and proc._is_alive() and + proc in waiting_to_start): assert proc.outqR_fd in fileno_to_outq assert fileno_to_outq[proc.outqR_fd] is proc assert proc.outqR_fd in hub.readers @@ -582,7 +584,7 @@ def on_process_up(proc): waiting_to_start.add(proc) hub.call_later( - self._proc_alive_timeout, verify_process_alive, proc, + self._proc_alive_timeout, verify_process_alive, ref(proc), ) self.on_process_up = on_process_up From 8ef663e8b3694be5a5f3babe10a554435c36c3e7 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 3 Mar 2016 17:01:18 -0800 Subject: [PATCH 1042/1103] [Prefork] Reset celery.worker.state after fork --- celery/concurrency/prefork.py | 2 ++ celery/worker/state.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/celery/concurrency/prefork.py b/celery/concurrency/prefork.py index 173316e6d..b4054d4c8 100644 --- a/celery/concurrency/prefork.py +++ b/celery/concurrency/prefork.py @@ -81,6 +81,8 @@ def process_initializer(app, hostname): for name, task in items(app.tasks): task.__trace__ = build_tracer(name, task, app.loader, hostname, app=app) + from celery.worker import state as worker_state + worker_state.reset_state() signals.worker_process_init.send(sender=None) diff --git a/celery/worker/state.py b/celery/worker/state.py index 51f55a44a..4e86e723a 100644 --- a/celery/worker/state.py +++ b/celery/worker/state.py @@ -63,6 +63,14 @@ should_terminate = None +def reset_state(): + reserved_requests.clear() + active_requests.clear() + total_count.clear() + all_total_count[:] = [0] + revoked.clear() + + def maybe_shutdown(): if should_stop is not None and should_stop is not False: raise WorkerShutdown(should_stop) From 4d90a281da7ad5a7ac6d7fff2343303e0c0d917f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 3 Mar 2016 17:21:12 -0800 Subject: [PATCH 1043/1103] [Prefork] Forgot to commit part of memory leak fix (Issue #2927) --- celery/concurrency/asynpool.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 7d6f94a0e..d8e64acb3 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -19,6 +19,7 @@ from __future__ import absolute_import import errno +import gc import os import select import socket @@ -427,6 +428,10 @@ def __init__(self, processes=None, synack=False, self._timeout_handler, 'on_hard_timeout', noop, ) + def _create_worker_process(self, i): + gc.collect() # Issue #2927 + return super(AsynPool, self)._create_worker_process(i) + def _event_process_exit(self, hub, proc): # This method is called whenever the process sentinel is readable. self._untrack_child_process(proc, hub) From 0b751092e6b12b084536b4131041a7147239f2a8 Mon Sep 17 00:00:00 2001 From: Dave Smith Date: Wed, 2 Mar 2016 17:54:15 -0700 Subject: [PATCH 1044/1103] [datastructures] Fix LimitedSet.discard() This was raising ValueError every time it was called, because the argument order was backward, resulting in unbounded memory growth for callers using discard() to remove items from LimitedSet. Closes #3087 --- celery/datastructures.py | 2 +- celery/tests/utils/test_datastructures.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/celery/datastructures.py b/celery/datastructures.py index e889e5e8b..a4258657d 100644 --- a/celery/datastructures.py +++ b/celery/datastructures.py @@ -633,7 +633,7 @@ def discard(self, value): except KeyError: return try: - self._heap.remove((value, itime)) + self._heap.remove((itime, value)) except ValueError: pass self._data.pop(value, None) diff --git a/celery/tests/utils/test_datastructures.py b/celery/tests/utils/test_datastructures.py index be81c364b..49be7a90e 100644 --- a/celery/tests/utils/test_datastructures.py +++ b/celery/tests/utils/test_datastructures.py @@ -259,6 +259,8 @@ def test_discard(self): s.add('foo') s.discard('foo') self.assertNotIn('foo', s) + self.assertEqual(len(s._data), 0) + self.assertEqual(len(s._heap), 0) s.discard('foo') def test_clear(self): From 131f44f1d4e985b42eb6e0fe2228733ced6db07a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Mar 2016 11:29:58 -0800 Subject: [PATCH 1045/1103] Merge changelog from 3.1 branch --- docs/history/changelog-3.1.rst | 58 ++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/docs/history/changelog-3.1.rst b/docs/history/changelog-3.1.rst index a5f38b92c..c47ce4db6 100644 --- a/docs/history/changelog-3.1.rst +++ b/docs/history/changelog-3.1.rst @@ -8,6 +8,64 @@ This document contains change notes for bugfix releases in the 3.1.x series (Cipater), please see :ref:`whatsnew-3.1` for an overview of what's new in Celery 3.1. +.. _version-3.1.21: + +3.1.21 +====== +:release-date: 2016-03-04 11:16 A.M PST +:release-by: Ask Solem + +- **Requirements** + + - Now depends on :ref:`Kombu 3.0.34 `. + + - Now depends on :mod:`billiard` 3.3.0.23. + +- **Prefork pool**: Fixes 100% CPU loop on Linux epoll (Issue #1845). + + Also potential fix for: Issue #2142, Issue #2606 + +- **Prefork pool**: Fixes memory leak related to processes exiting + (Issue #2927). + +- **Worker**: Fixes crash at startup when trying to censor passwords + in MongoDB and Cache result backend URLs (Issue #3079, Issue #3045, + Issue #3049, Issue #3068, Issue #3073). + + Fix contributed by Maxime Verger. + +- **Task**: An exception is now raised if countdown/expires is less + than -2147483648 (Issue #3078). + +- **Programs**: :program:`celery shell --ipython` now compatible with newer + IPython versions. + +- **Programs**: The DuplicateNodeName warning emitted by inspect/control + now includes a list of the node names returned. + + Contributed by Sebastian Kalinowski. + +- **Utils**: The ``.discard(item)`` method of + :class:`~celery.datastructures.LimitedSet` did not actually remove the item + (Issue #3087). + + Fix contributed by Dave Smith. + +- **Worker**: Node name formatting now emits less confusing error message + for unmatched format keys (Issue #3016). + +- **Results**: amqp/rpc backends: Fixed deserialization of JSON exceptions + (Issue #2518). + + Fix contributed by Allard Hoeve. + +- **Prefork pool: The `process inqueue damaged` error message now includes + the original exception raised. + +- **Documentation**: Includes improvements by: + + - Jeff Widman. + .. _version-3.1.20: 3.1.20 From 90a417716388e13e8a41ed3b365adf5c333d35de Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Mar 2016 11:33:12 -0800 Subject: [PATCH 1046/1103] Fixes rst syntax error in Changelog --- docs/history/changelog-3.1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/history/changelog-3.1.rst b/docs/history/changelog-3.1.rst index c47ce4db6..d9263f2b3 100644 --- a/docs/history/changelog-3.1.rst +++ b/docs/history/changelog-3.1.rst @@ -59,7 +59,7 @@ new in Celery 3.1. Fix contributed by Allard Hoeve. -- **Prefork pool: The `process inqueue damaged` error message now includes +- **Prefork pool**: The `process inqueue damaged` error message now includes the original exception raised. - **Documentation**: Includes improvements by: From 02f95470a781369d2b9c4fa7105d879fb0dae3b1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Mar 2016 15:38:05 -0800 Subject: [PATCH 1047/1103] task_routes entries can now be glob patterns or even regular expressions. Closes #1137 --- celery/app/routes.py | 31 ++++++++++++-- celery/tests/app/test_routes.py | 17 ++++++++ docs/configuration.rst | 73 ++++++++++++++++++--------------- docs/userguide/routing.rst | 23 ++++++++++- docs/whatsnew-4.0.rst | 6 +++ 5 files changed, 112 insertions(+), 38 deletions(-) diff --git a/celery/app/routes.py b/celery/app/routes.py index c428035b8..4b7047697 100644 --- a/celery/app/routes.py +++ b/celery/app/routes.py @@ -9,10 +9,15 @@ """ from __future__ import absolute_import +import re +import string + +from collections import Mapping, OrderedDict + from kombu import Queue from celery.exceptions import QueueNotFound -from celery.five import string_t +from celery.five import items, string_t from celery.utils import lpmerge from celery.utils.functional import firstmethod, mlazy from celery.utils.imports import instantiate @@ -22,11 +27,25 @@ _first_route = firstmethod('route_for_task') +def glob_to_re(glob, quote=string.punctuation.replace('*', '')): + glob = ''.join('\\' + c if c in quote else c for c in glob) + return glob.replace('*', '.+?') + + class MapRoute(object): """Creates a router out of a :class:`dict`.""" def __init__(self, map): - self.map = map + map = items(map) if isinstance(map, Mapping) else map + self.map = {} + self.patterns = OrderedDict() + for k, v in map: + if isinstance(k, re._pattern_type): + self.patterns[k] = v + elif '*' in k: + self.patterns[re.compile(glob_to_re(k))] = v + else: + self.map[k] = v def route_for_task(self, task, *args, **kwargs): try: @@ -35,6 +54,12 @@ def route_for_task(self, task, *args, **kwargs): pass except ValueError: return {'queue': self.map[task]} + for regex, route in items(self.patterns): + if regex.match(task): + try: + return dict(route) + except ValueError: + return {'queue': route} class Router(object): @@ -85,7 +110,7 @@ def prepare(routes): """Expands the :setting:`task_routes` setting.""" def expand_route(route): - if isinstance(route, dict): + if isinstance(route, (Mapping, list, tuple)): return MapRoute(route) if isinstance(route, string_t): return mlazy(instantiate, route) diff --git a/celery/tests/app/test_routes.py b/celery/tests/app/test_routes.py index 9730aab05..81f511fb5 100644 --- a/celery/tests/app/test_routes.py +++ b/celery/tests/app/test_routes.py @@ -72,6 +72,23 @@ def test_route_for_task(self): ) self.assertIsNone(route.route_for_task('celery.awesome')) + def test_route_for_task__glob(self): + route = routes.MapRoute([ + ('proj.tasks.*', 'routeA'), + ('demoapp.tasks.bar.*', {'exchange': 'routeB'}), + ]) + self.assertDictEqual( + route.route_for_task('proj.tasks.foo'), + {'queue': 'routeA'}, + ) + self.assertDictEqual( + route.route_for_task('demoapp.tasks.bar.moo'), + {'exchange': 'routeB'}, + ) + self.assertIsNone( + route.route_for_task('demoapp.foo.bar.moo'), + ) + def test_expand_route_not_found(self): expand = E(self.app, self.app.amqp.Queues( self.app.conf.task_queues, False)) diff --git a/docs/configuration.rst b/docs/configuration.rst index e3d034b56..81846ed8f 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -128,7 +128,7 @@ rush in moving to the new settings format. ``CELERY_DEFAULT_QUEUE`` :setting:`task_default_queue` ``CELERY_DEFAULT_RATE_LIMIT`` :setting:`task_default_rate_limit` ``CELERY_DEFAULT_ROUTING_KEY`` :setting:`task_default_routing_key` -``-"-_EAGER_PROPAGATES_EXCEPTIONS`` :setting:`task_eager_propagates` +``-'-_EAGER_PROPAGATES_EXCEPTIONS`` :setting:`task_eager_propagates` ``CELERY_IGNORE_RESULT`` :setting:`task_ignore_result` ``CELERY_TASK_PUBLISH_RETRY`` :setting:`task_publish_retry` ``CELERY_TASK_PUBLISH_RETRY_POLICY`` :setting:`task_publish_retry_policy` @@ -389,10 +389,10 @@ If set, the worker stores all task errors in the result store even if task_track_started ~~~~~~~~~~~~~~~~~~ -If :const:`True` the task will report its status as "started" when the +If :const:`True` the task will report its status as 'started' when the task is executed by a worker. The default value is :const:`False` as the normal behaviour is to not report that level of granularity. Tasks -are either pending, finished, or waiting to be retried. Having a "started" +are either pending, finished, or waiting to be retried. Having a 'started' state can be useful for when there are long running tasks and there is a need to report which task is currently running. @@ -599,7 +599,7 @@ Default is to expire after 1 day. result_cache_max ~~~~~~~~~~~~~~~~ -Enables client caching of results, which can be useful for the old "amqp" +Enables client caching of results, which can be useful for the old 'amqp' backend where the result is unavailable as soon as one result instance consumes it. @@ -1041,21 +1041,21 @@ Riak backend settings This backend requires the :setting:`result_backend` setting to be set to a Riak URL:: - result_backend = "riak://host:port/bucket" + result_backend = 'riak://host:port/bucket' For example:: - result_backend = "riak://localhost/celery + result_backend = 'riak://localhost/celery which is the same as:: - result_backend = "riak://" + result_backend = 'riak://' The fields of the URL are defined as follows: - *host* -Host name or IP address of the Riak server. e.g. `"localhost"`. +Host name or IP address of the Riak server. e.g. `'localhost'`. - *port* @@ -1307,25 +1307,30 @@ in order. A router can be specified as either: -* A router class instances +* A router class instance. * A string which provides the path to a router class -* A dict containing router specification. It will be converted to a :class:`celery.routes.MapRoute` instance. +* A dict containing router specification: + Will be converted to a :class:`celery.routes.MapRoute` instance. +* A list of ``(pattern, route)`` tuples: + Will be converted to a :class:`celery.routes.MapRoute` instance. Examples: .. code-block:: python task_routes = { - "celery.ping": "default", - "mytasks.add": "cpu-bound", - "video.encode": { - "queue": "video", - "exchange": "media" - "routing_key": "media.video.encode", + 'celery.ping': 'default', + 'mytasks.add': 'cpu-bound', + 'feed.tasks.*': 'feeds', # <-- glob pattern + re.compile(r'(image|video)\.tasks\..*'): 'media', # <-- regex + 'video.encode': { + 'queue': 'video', + 'exchange': 'media' + 'routing_key': 'media.video.encode', }, } - task_routes = ("myapp.tasks.Router", {"celery.ping": "default}) + task_routes = ('myapp.tasks.Router', {'celery.ping': 'default}) Where ``myapp.tasks.Router`` could be: @@ -1334,8 +1339,8 @@ Where ``myapp.tasks.Router`` could be: class Router(object): def route_for_task(self, task, args=None, kwargs=None): - if task == "celery.ping": - return "default" + if task == 'celery.ping': + return {'queue': 'default'} ``route_for_task`` may return a string or a dict. A string then means it's a queue name in :setting:`task_queues`, a dict means it's a custom route. @@ -1349,20 +1354,20 @@ Example if :func:`~celery.execute.apply_async` has these arguments: .. code-block:: python - Task.apply_async(immediate=False, exchange="video", - routing_key="video.compress") + Task.apply_async(immediate=False, exchange='video', + routing_key='video.compress') and a router returns: .. code-block:: python - {"immediate": True, "exchange": "urgent"} + {'immediate': True, 'exchange': 'urgent'} the final message options will be: .. code-block:: python - immediate=True, exchange="urgent", routing_key="video.compress" + immediate=True, exchange='urgent', routing_key='video.compress' (and any default message options defined in the :class:`~celery.task.base.Task` class) @@ -1375,17 +1380,17 @@ With the follow settings: .. code-block:: python task_queues = { - "cpubound": { - "exchange": "cpubound", - "routing_key": "cpubound", + 'cpubound': { + 'exchange': 'cpubound', + 'routing_key': 'cpubound', }, } task_routes = { - "tasks.add": { - "queue": "cpubound", - "routing_key": "tasks.add", - "serializer": "json", + 'tasks.add': { + 'queue': 'cpubound', + 'routing_key': 'tasks.add', + 'serializer': 'json', }, } @@ -1393,9 +1398,9 @@ The final routing options for ``tasks.add`` will become: .. code-block:: javascript - {"exchange": "cpubound", - "routing_key": "tasks.add", - "serializer": "json"} + {'exchange': 'cpubound', + 'routing_key': 'tasks.add', + 'serializer': 'json'} See :ref:`routers` for more examples. @@ -1970,7 +1975,7 @@ email_charset ~~~~~~~~~~~~~ .. versionadded:: 4.0 -Charset for outgoing emails. Default is "utf-8". +Charset for outgoing emails. Default is 'utf-8'. .. _conf-example-error-mail-config: diff --git a/docs/userguide/routing.rst b/docs/userguide/routing.rst index 4183a5303..5c485b5ea 100644 --- a/docs/userguide/routing.rst +++ b/docs/userguide/routing.rst @@ -41,7 +41,28 @@ With this route enabled import feed tasks will be routed to the `"feeds"` queue, while all other tasks will be routed to the default queue (named `"celery"` for historical reasons). -Now you can start server `z` to only process the feeds queue like this: +Alternatively, you can use glob pattern matching, or even regular expressions, +to match all tasks in the ``feed.tasks`` namespace:: + + task_routes = {'feed.tasks.*': {'queue': 'feeds'}} + +If the order in which the patterns are matched is important you should should +specify a tuple as the task router instead:: + + task_routes = ([ + ('feed.tasks.*': {'queue': 'feeds'}), + ('web.tasks.*': {'queue': 'web'}), + (re.compile(r'(video|image)\.tasks\..*'), {'queue': 'media'}), + ],) + +.. note:: + + The :setting:`task_routes` setting can either be a dictionary, or a + list of router objects, so in this case we need to specify the setting + as a tuple containing a list. + +After installing the router, you can start server `z` to only process the feeds +queue like this: .. code-block:: console diff --git a/docs/whatsnew-4.0.rst b/docs/whatsnew-4.0.rst index ddb2cc201..49a82672f 100644 --- a/docs/whatsnew-4.0.rst +++ b/docs/whatsnew-4.0.rst @@ -382,6 +382,12 @@ Task Autoretry Decorator Contributed by Dmitry Malinovsky. + +:setting:`task_routes` can now contain glob patterns and regexes. +================================================================= + +See examples in :setting:`task_routes` and :ref:`routing-automatic`. + In Other News ------------- From 7be9626e6c19e735a7a35ed41d3d90426e2af6f1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Mar 2016 15:43:05 -0800 Subject: [PATCH 1048/1103] [docs] revoke+terminate only supported by prefork --- docs/userguide/workers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index 7a2294a30..0b8d0d949 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -307,7 +307,7 @@ Commands ``revoke``: Revoking tasks -------------------------- -:pool support: all +:pool support: all, terminate only supported by prefork :broker support: *amqp, redis* :command: :program:`celery -A proj control revoke ` From 5e3559926c8ba3af6863dbf2eb345a2e000b8fdc Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Mar 2016 16:18:47 -0800 Subject: [PATCH 1049/1103] Adds "disable prefetch" FAQ. Closes #1736 --- docs/faq.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index cf45f5f80..c374f9748 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -722,6 +722,21 @@ and a worker can bind to as many queues as it wants. See :doc:`userguide/routing` for more information. +.. _faq-disable-prefetch: + +Can I disable prefetching of tasks? +----------------------------------- + +**Answer**: The term prefetch must have confused you, as as in Celery it's only used +to describe the task prefetching *limits*. + +Disabling the prefetch limits is possible, but that means the worker will +consume as many tasks as it can, as fast as possible. + +A discussion on prefetch limits, and configuration settings for a worker +that only reserves one task at a time is found here: +:ref:`optimizing-prefetch-limit`. + .. _faq-change-periodic-task-interval-at-runtime: Can I change the interval of a periodic task at runtime? From beb450be0de4030865c7fda621f4cbfc31876325 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Mar 2016 16:27:33 -0800 Subject: [PATCH 1050/1103] [result][database] Fixes database backend .as_uri() --- celery/backends/base.py | 5 +++-- celery/backends/database/__init__.py | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 6be3ffa6f..5468d75d7 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -115,8 +115,9 @@ def as_uri(self, include_password=False): """Return the backend as an URI, sanitizing the password or not""" # when using maybe_sanitize_url(), "/" is added # we're stripping it for consistency - return (self.url if include_password - else maybe_sanitize_url(self.url).rstrip("/")) + if self.url: + return (self.url if include_password + else maybe_sanitize_url(self.url).rstrip("/")) def mark_as_started(self, task_id, **meta): """Mark a task as started""" diff --git a/celery/backends/database/__init__.py b/celery/backends/database/__init__.py index 3c423960d..b63adb816 100644 --- a/celery/backends/database/__init__.py +++ b/celery/backends/database/__init__.py @@ -80,7 +80,7 @@ def __init__(self, dburi=None, engine_options=None, url=None, **kwargs): expires_type=maybe_timedelta, **kwargs ) conf = self.app.conf - self.dburi = url or dburi or conf.sqlalchemy_dburi + self.url = url or dburi or conf.sqlalchemy_dburi self.engine_options = dict( engine_options or {}, **conf.sqlalchemy_engine_options or {}) @@ -93,14 +93,14 @@ def __init__(self, dburi=None, engine_options=None, url=None, **kwargs): Task.__table__.name = tablenames.get('task', 'celery_taskmeta') TaskSet.__table__.name = tablenames.get('group', 'celery_tasksetmeta') - if not self.dburi: + if not self.url: raise ImproperlyConfigured( 'Missing connection string! Do you have the' ' sqlalchemy_dburi setting set to a real value?') def ResultSession(self, session_manager=SessionManager()): return session_manager.session_factory( - dburi=self.dburi, + dburi=self.url, short_lived_sessions=self.short_lived_sessions, **self.engine_options ) @@ -189,7 +189,7 @@ def cleanup(self): def __reduce__(self, args=(), kwargs={}): kwargs.update( - dict(dburi=self.dburi, + dict(dburi=self.url, expires=self.expires, engine_options=self.engine_options)) return super(DatabaseBackend, self).__reduce__(args, kwargs) From fa1820439e8ee1d8b6a09ff1c609b4121c171184 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Mar 2016 16:30:06 -0800 Subject: [PATCH 1051/1103] [result][database] Set max varchar size to 155 to deal with MySQL brain damage. Closes #1748 --- celery/backends/database/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/backends/database/models.py b/celery/backends/database/models.py index 2802a007c..82bc20d8f 100644 --- a/celery/backends/database/models.py +++ b/celery/backends/database/models.py @@ -28,7 +28,7 @@ class Task(ResultModelBase): id = sa.Column(sa.Integer, sa.Sequence('task_id_sequence'), primary_key=True, autoincrement=True) - task_id = sa.Column(sa.String(255), unique=True) + task_id = sa.Column(sa.String(155), unique=True) status = sa.Column(sa.String(50), default=states.PENDING) result = sa.Column(PickleType, nullable=True) date_done = sa.Column(sa.DateTime, default=datetime.utcnow, @@ -56,7 +56,7 @@ class TaskSet(ResultModelBase): id = sa.Column(sa.Integer, sa.Sequence('taskset_id_sequence'), autoincrement=True, primary_key=True) - taskset_id = sa.Column(sa.String(255), unique=True) + taskset_id = sa.Column(sa.String(155), unique=True) result = sa.Column(PickleType, nullable=True) date_done = sa.Column(sa.DateTime, default=datetime.utcnow, nullable=True) From b0a9990ead4132f07268f7abadb4887c6f93f0f6 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 7 Mar 2016 11:52:18 -0800 Subject: [PATCH 1052/1103] Adds .five.getfullargspec --- celery/bin/base.py | 5 ++--- celery/contrib/sphinx.py | 8 ++------ celery/five.py | 5 +++++ celery/utils/functional.py | 9 +++------ 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/celery/bin/base.py b/celery/bin/base.py index bc0095045..3b729d2fb 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -79,7 +79,6 @@ from collections import defaultdict from heapq import heappush -from inspect import getargspec from optparse import ( OptionParser, OptionGroup, IndentedHelpFormatter, make_option as Option, ) @@ -88,7 +87,7 @@ from celery import VERSION_BANNER, Celery, maybe_patch_concurrency from celery import signals from celery.exceptions import CDeprecationWarning, CPendingDeprecationWarning -from celery.five import items, string, string_t +from celery.five import getfullargspec, items, string, string_t from celery.platforms import EX_FAILURE, EX_OK, EX_USAGE from celery.utils import term from celery.utils import text @@ -283,7 +282,7 @@ def __call__(self, *args, **kwargs): return exc.status def verify_args(self, given, _index=0): - S = getargspec(self.run) + S = getfullargspec(self.run) _index = 1 if S.args and S.args[0] == 'self' else _index required = S.args[_index:-len(S.defaults) if S.defaults else None] missing = required[len(given):] diff --git a/celery/contrib/sphinx.py b/celery/contrib/sphinx.py index 2e5743123..c72513545 100644 --- a/celery/contrib/sphinx.py +++ b/celery/contrib/sphinx.py @@ -32,15 +32,11 @@ """ from __future__ import absolute_import -try: - from inspect import formatargspec, getfullargspec as getargspec -except ImportError: # Py2 - from inspect import formatargspec, getargspec # noqa - from sphinx.domains.python import PyModulelevel from sphinx.ext.autodoc import FunctionDocumenter from celery.app.task import BaseTask +from celery.five import formatargspec, getfullargspec class TaskDocumenter(FunctionDocumenter): @@ -54,7 +50,7 @@ def can_document_member(cls, member, membername, isattr, parent): def format_args(self): wrapped = getattr(self.object, '__wrapped__') if wrapped is not None: - argspec = getargspec(wrapped) + argspec = getfullargspec(wrapped) fmt = formatargspec(*argspec) fmt = fmt.replace('\\', '\\\\') return fmt diff --git a/celery/five.py b/celery/five.py index d6ec040cc..20462acad 100644 --- a/celery/five.py +++ b/celery/five.py @@ -25,6 +25,11 @@ except ImportError: pass +try: # pragma: no cover + from inspect import formatargspec, getfullargspec +except ImportError: # Py2 + from inspect import formatargspec, getargspec as getfullargspec # noqa + __all__ = [ 'class_property', 'reclassmethod', 'create_module', 'recreate_module', ] diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 0084f5dd4..0cf5c844a 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -13,10 +13,7 @@ from collections import OrderedDict from functools import partial, wraps -try: - from inspect import isfunction, getfullargspec as getargspec -except ImportError: # Py2 - from inspect import isfunction, getargspec # noqa +from inspect import isfunction from itertools import chain, islice from kombu.utils.functional import ( @@ -24,7 +21,7 @@ ) from vine import promise -from celery.five import UserDict, UserList, keys, range +from celery.five import UserDict, UserList, getfullargspec, keys, range __all__ = ['LRUCache', 'is_list', 'maybe_list', 'memoize', 'mlazy', 'noop', 'first', 'firstmethod', 'chunks', 'padlist', 'mattrgetter', 'uniq', @@ -388,7 +385,7 @@ def head_from_fun(fun, bound=False, debug=False): name = fun.__name__ definition = FUNHEAD_TEMPLATE.format( fun_name=name, - fun_args=_argsfromspec(getargspec(fun)), + fun_args=_argsfromspec(getfullargspec(fun)), fun_value=1, ) if debug: # pragma: no cover From 06b99aeda3ce7a1c5f380b87b1e117511ab70e2c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 7 Mar 2016 11:59:41 -0800 Subject: [PATCH 1053/1103] Adds .utils.functional.fun_takes_arguments --- celery/tests/utils/test_functional.py | 17 +++++++++++++++++ celery/utils/functional.py | 5 +++++ 2 files changed, 22 insertions(+) diff --git a/celery/tests/utils/test_functional.py b/celery/tests/utils/test_functional.py index c358351aa..d9d14ddea 100644 --- a/celery/tests/utils/test_functional.py +++ b/celery/tests/utils/test_functional.py @@ -11,6 +11,7 @@ from celery.utils.functional import ( DummyContext, LRUCache, + fun_takes_argument, head_from_fun, firstmethod, first, @@ -308,3 +309,19 @@ def test_from_fun_with_hints(self): g(1) g(1, 2) g(1, 2, kwarg=3) + + +class test_fun_takes_argument(Case): + + def test_starkwargs(self): + self.assertTrue(fun_takes_argument('foo', lambda **kw: 1)) + + def test_named(self): + self.assertTrue(fun_takes_argument('foo', lambda a, foo, bar: 1)) + + def test_starargs(self): + self.assertTrue(fun_takes_argument('foo', lambda a, *args: 1)) + + def test_does_not(self): + self.assertFalse(fun_takes_argument('foo', lambda a, bar, baz: 1)) + self.assertFalse(fun_takes_argument('foo', lambda: 1)) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 0cf5c844a..716939abe 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -397,3 +397,8 @@ def head_from_fun(fun, bound=False, debug=False): if bound: return partial(result, object()) return result + + +def fun_takes_argument(name, fun): + spec = getfullargspec(fun) + return spec.keywords or spec.varargs or name in spec.args From 1b7a9f6187bfeae34b9abe38a721b0937ff08848 Mon Sep 17 00:00:00 2001 From: Ali Bozorgkhan Date: Wed, 27 Aug 2014 11:58:53 -0700 Subject: [PATCH 1054/1103] pass options to route_for_task --- celery/app/routes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/app/routes.py b/celery/app/routes.py index 4b7047697..c6fbb508c 100644 --- a/celery/app/routes.py +++ b/celery/app/routes.py @@ -74,7 +74,7 @@ def __init__(self, routes=None, queues=None, def route(self, options, task, args=(), kwargs={}): options = self.expand_destination(options) # expands 'queue' if self.routes: - route = self.lookup_route(task, args, kwargs) + route = self.lookup_route(task, args, kwargs, options) if route: # expands 'queue' in route. return lpmerge(self.expand_destination(route), options) if 'queue' not in options: @@ -102,8 +102,8 @@ def expand_destination(self, route): 'Queue {0!r} missing from task_queues'.format(queue)) return route - def lookup_route(self, task, args=None, kwargs=None): - return _first_route(self.routes, task, args, kwargs) + def lookup_route(self, task, args=None, kwargs=None, options=None): + return _first_route(self.routes, task, args, kwargs, options) def prepare(routes): From 16f927185d9bd6f18f0eaa612e57168cb2534640 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 7 Mar 2016 12:22:41 -0800 Subject: [PATCH 1055/1103] Fixes backward compatibility for #2217 --- celery/app/routes.py | 8 +++++++- celery/tests/utils/test_functional.py | 10 ++++++++++ celery/utils/functional.py | 18 +++++++++++------- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/celery/app/routes.py b/celery/app/routes.py index c6fbb508c..5a367d651 100644 --- a/celery/app/routes.py +++ b/celery/app/routes.py @@ -19,11 +19,17 @@ from celery.exceptions import QueueNotFound from celery.five import items, string_t from celery.utils import lpmerge -from celery.utils.functional import firstmethod, mlazy +from celery.utils.functional import firstmethod, fun_takes_argument, mlazy from celery.utils.imports import instantiate __all__ = ['MapRoute', 'Router', 'prepare'] + +def _try_route(meth, task, args, kwargs, options=None): + if fun_takes_argument('options', meth, position=4): + return meth(task, args, kwargs, options) + return meth(task, args, kwargs) + _first_route = firstmethod('route_for_task') diff --git a/celery/tests/utils/test_functional.py b/celery/tests/utils/test_functional.py index d9d14ddea..2b37e140b 100644 --- a/celery/tests/utils/test_functional.py +++ b/celery/tests/utils/test_functional.py @@ -319,9 +319,19 @@ def test_starkwargs(self): def test_named(self): self.assertTrue(fun_takes_argument('foo', lambda a, foo, bar: 1)) + def fun(a, b, c, d): + return 1 + + self.assertTrue(fun_takes_argument('foo', fun, position=4)) + def test_starargs(self): self.assertTrue(fun_takes_argument('foo', lambda a, *args: 1)) def test_does_not(self): self.assertFalse(fun_takes_argument('foo', lambda a, bar, baz: 1)) self.assertFalse(fun_takes_argument('foo', lambda: 1)) + + def fun(a, b, foo): + return 1 + + self.assertFalse(fun_takes_argument('foo', fun, position=4)) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 716939abe..a41d464a8 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -237,7 +237,7 @@ def first(predicate, it): ) -def firstmethod(method): +def firstmethod(method, on_call=None): """Return a function that with a list of instances, finds the first instance that gives a value for the given method. @@ -249,13 +249,14 @@ def firstmethod(method): def _matcher(it, *args, **kwargs): for obj in it: try: - answer = getattr(maybe_evaluate(obj), method)(*args, **kwargs) + meth = getattr(maybe_evaluate(obj), method) + reply = (on_call(meth, *args, **kwargs) if on_call + else meth(*args, **kwargs)) except AttributeError: pass else: - if answer is not None: - return answer - + if reply is not None: + return reply return _matcher @@ -399,6 +400,9 @@ def head_from_fun(fun, bound=False, debug=False): return result -def fun_takes_argument(name, fun): +def fun_takes_argument(name, fun, position=None): spec = getfullargspec(fun) - return spec.keywords or spec.varargs or name in spec.args + return ( + spec.keywords or spec.varargs or + (len(spec.args) >= position if position else name in spec.args) + ) From c766fb4531f19aa37ba70a205eb5959377a9ccfa Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 7 Mar 2016 12:35:27 -0800 Subject: [PATCH 1056/1103] [Programs][worker] --detach would create extraenous logfile with literal %I in the filename. Closes #3096 --- celery/bin/celeryd_detach.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/celery/bin/celeryd_detach.py b/celery/bin/celeryd_detach.py index 43fd5c665..f29c05e8e 100644 --- a/celery/bin/celeryd_detach.py +++ b/celery/bin/celeryd_detach.py @@ -19,6 +19,7 @@ from optparse import OptionParser, BadOptionError from celery.platforms import EX_FAILURE, detached +from celery.utils import default_nodename, node_format from celery.utils.log import get_logger from celery.bin.base import daemon_options @@ -32,7 +33,10 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, gid=None, umask=None, working_directory=None, fake=False, app=None, - executable=None): + executable=None, hostname=None): + hostname = default_nodename(hostname) + logfile = node_format(logfile, hostname) + pidfile = node_format(pidfile, hostname) fake = 1 if C_FAKEFORK else fake with detached(logfile, pidfile, uid, gid, umask, working_directory, fake, after_forkers=False): @@ -44,7 +48,8 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, if app is None: from celery import current_app app = current_app - app.log.setup_logging_subsystem('ERROR', logfile) + app.log.setup_logging_subsystem( + 'ERROR', logfile, hostname=hostname) logger.critical("Can't exec %r", ' '.join([path] + argv), exc_info=True) return EX_FAILURE @@ -159,6 +164,7 @@ def execute_from_commandline(self, argv=None): def prepare_arguments(self, parser): daemon_options(parser, default_pidfile='celeryd.pid') parser.add_option('--workdir', default=None, dest='working_directory') + parser.add_option('-n', '--hostname') parser.add_option( '--fake', default=False, action='store_true', dest='fake', From 925fb7df44518c3148a3140433c202ef94070700 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 7 Mar 2016 12:41:23 -0800 Subject: [PATCH 1057/1103] Fixes build --- celery/tests/bin/test_celeryd_detach.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/celery/tests/bin/test_celeryd_detach.py b/celery/tests/bin/test_celeryd_detach.py index 0e1d0169a..e36abcdc6 100644 --- a/celery/tests/bin/test_celeryd_detach.py +++ b/celery/tests/bin/test_celeryd_detach.py @@ -23,7 +23,7 @@ def test_execs(self, setup_logs, logger, execv, detached): context.__exit__ = Mock() detach('/bin/boo', ['a', 'b', 'c'], logfile='/var/log', - pidfile='/var/pid') + pidfile='/var/pid', hostname='foo@example.com') detached.assert_called_with( '/var/log', '/var/pid', None, None, None, None, False, after_forkers=False, @@ -36,11 +36,14 @@ def test_execs(self, setup_logs, logger, execv, detached): execv.assert_called_with('/bin/foo', ['/bin/foo', 'a', 'b', 'c']) execv.side_effect = Exception('foo') - r = detach('/bin/boo', ['a', 'b', 'c'], - logfile='/var/log', pidfile='/var/pid', app=self.app) + r = detach( + '/bin/boo', ['a', 'b', 'c'], + logfile='/var/log', pidfile='/var/pid', + hostname='foo@example.com', app=self.app) context.__enter__.assert_called_with() self.assertTrue(logger.critical.called) - setup_logs.assert_called_with('ERROR', '/var/log') + setup_logs.assert_called_with( + 'ERROR', '/var/log', hostname='foo@example.com') self.assertEqual(r, 1) self.patch('celery.current_app') @@ -108,7 +111,7 @@ def test_execute_from_commandline(self, detach, exit): detach.assert_called_with( path=x.execv_path, uid=None, gid=None, umask=None, fake=False, logfile='/var/log', pidfile='celeryd.pid', - working_directory=None, executable=None, + working_directory=None, executable=None, hostname=None, argv=x.execv_argv + [ '-c', '1', '-lDEBUG', '--logfile=/var/log', '--pidfile=celeryd.pid', From ffaf8cf940aa5e99f956e954d8bdb984089e5f24 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 7 Mar 2016 13:02:18 -0800 Subject: [PATCH 1058/1103] [Result] backend.as_uri() must return proper schemes --- celery/backends/base.py | 7 ++++--- celery/tests/backends/test_base.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 5468d75d7..c1793fa83 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -115,9 +115,10 @@ def as_uri(self, include_password=False): """Return the backend as an URI, sanitizing the password or not""" # when using maybe_sanitize_url(), "/" is added # we're stripping it for consistency - if self.url: - return (self.url if include_password - else maybe_sanitize_url(self.url).rstrip("/")) + if include_password: + return self.url + url = maybe_sanitize_url(self.url or '') + return url[:-1] if url.endswith(':///') else url def mark_as_started(self, task_id, **meta): """Mark a task as started""" diff --git a/celery/tests/backends/test_base.py b/celery/tests/backends/test_base.py index fa6a5bac7..4d9607c68 100644 --- a/celery/tests/backends/test_base.py +++ b/celery/tests/backends/test_base.py @@ -598,7 +598,7 @@ def setup(self): ) def test_as_uri_include_password(self): - self.assertEqual(self.b.as_uri(True), 'sch://uuuu:pwpw@hostname.dom') + self.assertEqual(self.b.as_uri(True), self.b.url) def test_as_uri_exclude_password(self): - self.assertEqual(self.b.as_uri(), 'sch://uuuu:**@hostname.dom') + self.assertEqual(self.b.as_uri(), 'sch://uuuu:**@hostname.dom/') From 62d15a08cecb96ff6e730f0b4e181edf7e41df63 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 7 Mar 2016 13:06:17 -0800 Subject: [PATCH 1059/1103] Fixes build --- celery/tests/bin/test_celeryd_detach.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/celery/tests/bin/test_celeryd_detach.py b/celery/tests/bin/test_celeryd_detach.py index e36abcdc6..a2bbe5b2d 100644 --- a/celery/tests/bin/test_celeryd_detach.py +++ b/celery/tests/bin/test_celeryd_detach.py @@ -48,10 +48,12 @@ def test_execs(self, setup_logs, logger, execv, detached): self.patch('celery.current_app') from celery import current_app - r = detach('/bin/boo', ['a', 'b', 'c'], - logfile='/var/log', pidfile='/var/pid', app=None) + r = detach( + '/bin/boo', ['a', 'b', 'c'], + logfile='/var/log', pidfile='/var/pid', + hostname='foo@example.com', app=None) current_app.log.setup_logging_subsystem.assert_called_with( - 'ERROR', '/var/log', + 'ERROR', '/var/log', hostname='foo@example.com', ) From 0989790b278da2a874f68ba5100e4dc316366f97 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 7 Mar 2016 13:20:49 -0800 Subject: [PATCH 1060/1103] All result backends must properly set self.url --- celery/backends/amqp.py | 3 +++ celery/backends/cache.py | 1 + celery/backends/cassandra.py | 3 +++ celery/backends/couchbase.py | 14 +++++++------- celery/backends/couchdb.py | 13 +++++++------ celery/backends/elasticsearch.py | 1 + celery/backends/filesystem.py | 22 +++++++++++----------- celery/backends/mongodb.py | 16 ++++++++-------- celery/backends/riak.py | 12 +++++++----- celery/backends/rpc.py | 3 +++ celery/tests/backends/test_mongodb.py | 7 +++++-- 11 files changed, 56 insertions(+), 39 deletions(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index 89ee6a423..cfaaafaa8 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -249,6 +249,9 @@ def delete_group(self, group_id): raise NotImplementedError( 'delete_group is not supported by this backend.') + def as_uri(self, include_password=True): + return 'amqp://' + def __reduce__(self, args=(), kwargs={}): kwargs.update( connection=self._connection, diff --git a/celery/backends/cache.py b/celery/backends/cache.py index 005737845..122e70f6b 100644 --- a/celery/backends/cache.py +++ b/celery/backends/cache.py @@ -100,6 +100,7 @@ class CacheBackend(KeyValueStoreBackend): def __init__(self, app, expires=None, backend=None, options={}, url=None, **kwargs): super(CacheBackend, self).__init__(app, **kwargs) + self.url = url self.options = dict(self.app.conf.cache_backend_options, **options) diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index e6a3f02e7..2bd2a78e4 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -218,6 +218,9 @@ def _store_result(self, task_id, result, state, buf_t(self.encode(self.current_task_children(request))) )) + def as_uri(self, include_password=True): + return 'cassandra://' + def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" self._get_connection() diff --git a/celery/backends/couchbase.py b/celery/backends/couchbase.py index 1cf9a7b59..0f3483072 100644 --- a/celery/backends/couchbase.py +++ b/celery/backends/couchbase.py @@ -28,6 +28,12 @@ class CouchBaseBackend(KeyValueStoreBackend): + """CouchBase backend. + + :raises celery.exceptions.ImproperlyConfigured: if + module :mod:`couchbase` is not available. + + """ bucket = 'default' host = 'localhost' port = 8091 @@ -38,19 +44,13 @@ class CouchBaseBackend(KeyValueStoreBackend): unlock_gil = True timeout = 2.5 transcoder = None - # supports_autoexpire = False # Use str as couchbase key not bytes key_t = str_t def __init__(self, url=None, *args, **kwargs): - """Initialize CouchBase backend instance. - - :raises celery.exceptions.ImproperlyConfigured: if - module :mod:`couchbase` is not available. - - """ super(CouchBaseBackend, self).__init__(*args, **kwargs) + self.url = url if Couchbase is None: raise ImproperlyConfigured( diff --git a/celery/backends/couchdb.py b/celery/backends/couchdb.py index f1a3ebde5..32ae7826f 100644 --- a/celery/backends/couchdb.py +++ b/celery/backends/couchdb.py @@ -27,6 +27,12 @@ class CouchBackend(KeyValueStoreBackend): + """CouchDB backend. + + :raises celery.exceptions.ImproperlyConfigured: if + module :mod:`pycouchdb` is not available. + + """ container = 'default' scheme = 'http' host = 'localhost' @@ -35,13 +41,8 @@ class CouchBackend(KeyValueStoreBackend): password = None def __init__(self, url=None, *args, **kwargs): - """Initialize CouchDB backend instance. - - :raises celery.exceptions.ImproperlyConfigured: if - module :mod:`pycouchdb` is not available. - - """ super(CouchBackend, self).__init__(*args, **kwargs) + self.url = url if pycouchdb is None: raise ImproperlyConfigured(ERR_LIB_MISSING) diff --git a/celery/backends/elasticsearch.py b/celery/backends/elasticsearch.py index 95fcd27bb..78d1aa3e2 100644 --- a/celery/backends/elasticsearch.py +++ b/celery/backends/elasticsearch.py @@ -45,6 +45,7 @@ class ElasticsearchBackend(KeyValueStoreBackend): def __init__(self, url=None, *args, **kwargs): super(ElasticsearchBackend, self).__init__(*args, **kwargs) + self.url = url if elasticsearch is None: raise ImproperlyConfigured(E_LIB_MISSING) diff --git a/celery/backends/filesystem.py b/celery/backends/filesystem.py index 5368de4db..e42a5eeaf 100644 --- a/celery/backends/filesystem.py +++ b/celery/backends/filesystem.py @@ -32,22 +32,22 @@ class FilesystemBackend(KeyValueStoreBackend): + """Filesystem result backend. - def __init__(self, url=None, open=open, unlink=os.unlink, sep=os.sep, - encoding=default_encoding, *args, **kwargs): - """Initialize the filesystem backend. - - Keyword arguments (in addition to those of KeyValueStoreBackend): + Keyword arguments (in addition to those of KeyValueStoreBackend): - :param url: URL to the directory we should use - :param open: open function to use when opening files - :param unlink: unlink function to use when deleting files - :param sep: directory seperator (to join the directory with the key) - :param encoding: encoding used on the filesystem + :param url: URL to the directory we should use + :param open: open function to use when opening files + :param unlink: unlink function to use when deleting files + :param sep: directory seperator (to join the directory with the key) + :param encoding: encoding used on the filesystem - """ + """ + def __init__(self, url=None, open=open, unlink=os.unlink, sep=os.sep, + encoding=default_encoding, *args, **kwargs): super(FilesystemBackend, self).__init__(*args, **kwargs) + self.url = url path = self._find_path(url) # We need the path and seperator as bytes objects diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index e48a68371..fd11f4764 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -40,6 +40,12 @@ class InvalidDocument(Exception): # noqa class MongoBackend(BaseBackend): + """MongoDB result backend. + + :raises celery.exceptions.ImproperlyConfigured: if + module :mod:`pymongo` is not available. + + """ mongo_host = None host = 'localhost' @@ -57,12 +63,6 @@ class MongoBackend(BaseBackend): _connection = None def __init__(self, app=None, **kwargs): - """Initialize MongoDB backend instance. - - :raises celery.exceptions.ImproperlyConfigured: if - module :mod:`pymongo` is not available. - - """ self.options = {} super(MongoBackend, self).__init__(app, **kwargs) @@ -305,7 +305,7 @@ def as_uri(self, include_password=False): return self.url if ',' not in self.url: - return maybe_sanitize_url(self.url).rstrip('/') + return maybe_sanitize_url(self.url) uri1, remainder = self.url.split(',', 1) - return ','.join([maybe_sanitize_url(uri1).rstrip('/'), remainder]) + return ','.join([maybe_sanitize_url(uri1), remainder]) diff --git a/celery/backends/riak.py b/celery/backends/riak.py index 005be46b9..de2138e3d 100644 --- a/celery/backends/riak.py +++ b/celery/backends/riak.py @@ -50,6 +50,12 @@ def is_ascii(s): class RiakBackend(KeyValueStoreBackend): + """Riak result backend. + + :raises celery.exceptions.ImproperlyConfigured: if + module :mod:`riak` is not available. + + """ # TODO: allow using other protocols than protobuf ? #: default protocol used to connect to Riak, might be `http` or `pbc` protocol = 'pbc' @@ -67,12 +73,8 @@ class RiakBackend(KeyValueStoreBackend): def __init__(self, host=None, port=None, bucket_name=None, protocol=None, url=None, *args, **kwargs): - """Initialize Riak backend instance. - - :raises celery.exceptions.ImproperlyConfigured: if - module :mod:`riak` is not available. - """ super(RiakBackend, self).__init__(*args, **kwargs) + self.url = url if not riak: raise ImproperlyConfigured( diff --git a/celery/backends/rpc.py b/celery/backends/rpc.py index ee282eed1..7c6c68ebb 100644 --- a/celery/backends/rpc.py +++ b/celery/backends/rpc.py @@ -57,6 +57,9 @@ def on_reply_declare(self, task_id): def on_result_fulfilled(self, result): pass + def as_uri(self, include_password=True): + return 'rpc://' + @property def binding(self): return self.Queue(self.oid, self.exchange, self.oid, diff --git a/celery/tests/backends/test_mongodb.py b/celery/tests/backends/test_mongodb.py index fed11b207..96a8db4b3 100644 --- a/celery/tests/backends/test_mongodb.py +++ b/celery/tests/backends/test_mongodb.py @@ -36,8 +36,11 @@ class test_MongoBackend(AppCase): 'mongodb://uuuu:pwpw@hostname.dom,' 'hostname.dom/database?replicaSet=rs' ) - sanitized_default_url = default_url.replace('pwpw', '**') - sanitized_replica_set_url = replica_set_url.replace('pwpw', '**') + sanitized_default_url = 'mongodb://uuuu:**@hostname.dom/database' + sanitized_replica_set_url = ( + 'mongodb://uuuu:**@hostname.dom/,' + 'hostname.dom/database?replicaSet=rs' + ) def setup(self): if pymongo is None: From d8d19de9d83499c272b95cbf91cd0ca59853e5e3 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 8 Mar 2016 13:30:48 -0800 Subject: [PATCH 1061/1103] [result][rpc] RPC backend get_task_meta needs to handle out of band messages --- celery/backends/amqp.py | 35 ++++++++++++++++++++++++------ celery/backends/async.py | 3 +++ celery/tests/backends/test_amqp.py | 1 + 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index cfaaafaa8..0bb925d19 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -85,7 +85,8 @@ def consume_from(self, queue): self._consumer.consume() def cancel_for(self, queue): - self._consumer.cancel_by_queue(queue.name) + if self._consumer: + self._consumer.cancel_by_queue(queue.name) class AMQPBackend(base.Backend, AsyncBackendMixin): @@ -115,6 +116,7 @@ def __init__(self, app, connection=None, exchange=None, exchange_type=None, super(AMQPBackend, self).__init__(app, **kwargs) conf = self.app.conf self._connection = connection + self._out_of_band = {} self.persistent = self.prepare_persistent(persistent) self.delivery_mode = 2 if self.persistent else 1 exchange = exchange or conf.result_exchange @@ -191,7 +193,20 @@ def store_result(self, task_id, result, state, def on_reply_declare(self, task_id): return [self._create_binding(task_id)] + def on_out_of_band_result(self, task_id, message): + if self.result_consumer: + self.result_consumer.on_out_of_band_result(message) + self._out_of_band[task_id] = message + def get_task_meta(self, task_id, backlog_limit=1000): + try: + buffered = self._out_of_band.pop(task_id) + except KeyError: + pass + else: + payload = self._cache[task_id] = self.meta_from_decoded( + buffered.payload) + return payload # Polling and using basic_get with self.app.pool.acquire_channel(block=True) as (_, channel): binding = self._create_binding(task_id)(channel) @@ -204,13 +219,19 @@ def get_task_meta(self, task_id, backlog_limit=1000): ) if not acc: # no more messages break - if acc.payload['task_id'] == task_id: + try: + message_task_id = acc.properties['correlation_id'] + except (AttributeError, KeyError): + message_task_id = acc.payload['task_id'] + if message_task_id == task_id: prev, latest = latest, acc - if prev: - # backends are not expected to keep history, - # so we delete everything except the most recent state. - prev.ack() - prev = None + if prev: + # backends are not expected to keep history, + # so we delete everything except the most recent state. + prev.ack() + prev = None + else: + self.on_out_of_band_result(message_task_id, acc) else: raise self.BacklogLimitExceeded(task_id) diff --git a/celery/backends/async.py b/celery/backends/async.py index 7fc26c4e1..4d0a71725 100644 --- a/celery/backends/async.py +++ b/celery/backends/async.py @@ -190,6 +190,9 @@ def _wait_for_pending(self, result, timeout=None, interval=0.5, finally: self.on_message = prev_on_m + def on_out_of_band_result(self, message): + self.on_state_change(message.payload, message) + def on_state_change(self, meta, message): if self.on_message: self.on_message(meta) diff --git a/celery/tests/backends/test_amqp.py b/celery/tests/backends/test_amqp.py index d92ba666d..0f57b3b88 100644 --- a/celery/tests/backends/test_amqp.py +++ b/celery/tests/backends/test_amqp.py @@ -155,6 +155,7 @@ class Message(object): def __init__(self, **merge): self.payload = dict({'status': states.STARTED, 'result': None}, **merge) + self.properties = {'correlation_id': merge.get('task_id')} self.body = pickle.dumps(self.payload) self.content_type = 'application/x-python-serialize' self.content_encoding = 'binary' From 7604070c9590b482fac6a32cba19d31f9dbf7165 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 8 Mar 2016 19:24:12 -0800 Subject: [PATCH 1062/1103] [backend][async] Implements gevent result drainer --- celery/backends/async.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/celery/backends/async.py b/celery/backends/async.py index 4d0a71725..ddb56287f 100644 --- a/celery/backends/async.py +++ b/celery/backends/async.py @@ -14,6 +14,7 @@ from weakref import WeakKeyDictionary from kombu.syn import detect_environment +from kombu.utils import cached_property from celery import states from celery.exceptions import TimeoutError @@ -58,22 +59,21 @@ def wait_for(self, p, wait, timeout=None): wait(timeout=timeout) -@register_drainer('eventlet') -class EventletDrainer(Drainer): +class greenletDrainer(Drainer): + spawn = None _g = None _stopped = False def run(self): while not self._stopped: try: - self.result_consumer.drain_events(timeout=10) + self.result_consumer.drain_events(timeout=1) except socket.timeout: pass def start(self): - from eventlet import spawn if self._g is None: - self._g = spawn(self.run) + self._g = self.spawn(self.run) def stop(self): self._stopped = True @@ -85,6 +85,24 @@ def wait_for(self, p, wait, timeout=None): time.sleep(0) +@register_drainer('eventlet') +class eventletDrainer(greenletDrainer): + + @cached_property + def spawn(self): + from eventlet import spawn + return spawn + + +@register_drainer('gevent') +class geventDrainer(greenletDrainer): + + @cached_property + def spawn(self): + from gevent import spawn + return spawn + + class AsyncBackendMixin(object): def _collect_into(self, result, bucket): From 1f995b37c1f828cb2ae984d19866aabad8e2cc2b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 8 Mar 2016 19:29:43 -0800 Subject: [PATCH 1063/1103] [result][redis] Use pubsub for consuming results, and use the new async backend interface (Issue #2511) --- celery/backends/amqp.py | 21 ++++++++------- celery/backends/async.py | 18 ++++++++++--- celery/backends/base.py | 11 +++++--- celery/backends/redis.py | 57 ++++++++++++++++++++++++++++++++++++++-- celery/backends/rpc.py | 4 ++- 5 files changed, 92 insertions(+), 19 deletions(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index 0bb925d19..6af14a192 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -49,13 +49,16 @@ class NoCacheQueue(Queue): class ResultConsumer(BaseResultConsumer): Consumer = Consumer + _connection = None + _consumer = None + def __init__(self, *args, **kwargs): super(ResultConsumer, self).__init__(*args, **kwargs) - self._connection = None - self._consumer = None + self._create_binding = self.backend._create_binding - def start(self, initial_queue, no_ack=True): + def start(self, initial_task_id, no_ack=True): self._connection = self.app.connection() + initial_queue = self._create_binding(initial_task_id) self._consumer = self.Consumer( self._connection.default_channel, [initial_queue], callbacks=[self.on_state_change], no_ack=no_ack, @@ -77,16 +80,17 @@ def on_after_fork(self): self._connection.collect() self._connection = None - def consume_from(self, queue): + def consume_from(self, task_id): if self._consumer is None: - return self.start(queue) + return self.start(task_id) + queue = self._create_binding(task_id) if not self._consumer.consuming_from(queue): self._consumer.add_queue(queue) self._consumer.consume() - def cancel_for(self, queue): + def cancel_for(self, task_id): if self._consumer: - self._consumer.cancel_by_queue(queue.name) + self._consumer.cancel_by_queue(self._create_binding(task_id).name) class AMQPBackend(base.Backend, AsyncBackendMixin): @@ -138,9 +142,6 @@ def _after_fork(self): self._pending_results.clear() self.result_consumer._after_fork() - def on_result_fulfilled(self, result): - self.result_consumer.cancel_for(self._create_binding(result.id)) - def _create_exchange(self, name, type='direct', delivery_mode=2): return self.Exchange(name=name, type=type, diff --git a/celery/backends/async.py b/celery/backends/async.py index ddb56287f..0ff5ac045 100644 --- a/celery/backends/async.py +++ b/celery/backends/async.py @@ -135,7 +135,7 @@ def iter_native(self, result, timeout=None, interval=0.5, no_ack=True, def add_pending_result(self, result): if result.id not in self._pending_results: self._pending_results[result.id] = result - self.result_consumer.consume_from(self._create_binding(result.id)) + self.result_consumer.consume_from(result.id) return result def remove_pending_result(self, result): @@ -144,7 +144,7 @@ def remove_pending_result(self, result): return result def on_result_fulfilled(self, result): - pass + self.result_consumer.cancel_for(result.id) def wait_for_pending(self, result, callback=None, propagate=True, **kwargs): @@ -177,8 +177,20 @@ def __init__(self, backend, app, accept, pending_results): self.buckets = WeakKeyDictionary() self.drainer = drainers[detect_environment()](self) + def start(self): + raise NotImplementedError() + + def stop(self): + pass + def drain_events(self, timeout=None): - raise NotImplementedError('subclass responsibility') + raise NotImplementedError() + + def consume_from(self, task_id): + raise NotImplementedError() + + def cancel_for(self, task_id): + raise NotImplementedError() def _after_fork(self): self.bucket.clear() diff --git a/celery/backends/base.py b/celery/backends/base.py index c1793fa83..4077a5ac8 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -448,7 +448,7 @@ class BaseBackend(Backend, SyncBackendMixin): BaseDictBackend = BaseBackend # XXX compat -class KeyValueStoreBackend(BaseBackend): +class BaseKeyValueStoreBackend(Backend): key_t = ensure_bytes task_keyprefix = 'celery-task-meta-' group_keyprefix = 'celery-taskset-meta-' @@ -459,7 +459,7 @@ def __init__(self, *args, **kwargs): if hasattr(self.key_t, '__func__'): # pragma: no cover self.key_t = self.key_t.__func__ # remove binding self._encode_prefixes() - super(KeyValueStoreBackend, self).__init__(*args, **kwargs) + super(BaseKeyValueStoreBackend, self).__init__(*args, **kwargs) if self.implements_incr: self.apply_chord = self._apply_chord_incr @@ -578,7 +578,8 @@ def _forget(self, task_id): def _store_result(self, task_id, result, state, traceback=None, request=None, **kwargs): meta = {'status': state, 'result': result, 'traceback': traceback, - 'children': self.current_task_children(request)} + 'children': self.current_task_children(request), + 'task_id': task_id} self.set(self.get_key_for_task(task_id), self.encode(meta)) return result @@ -683,6 +684,10 @@ def on_chord_part_return(self, request, state, result, **kwargs): self.expire(key, 86400) +class KeyValueStoreBackend(BaseKeyValueStoreBackend, SyncBackendMixin): + pass + + class DisabledBackend(BaseBackend): _cache = {} # need this attribute to reset cache in tests. diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 00bc01227..8cbb8fe27 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -14,6 +14,7 @@ from kombu.utils.url import _parse_url from celery import states +from celery._state import task_join_will_block from celery.canvas import maybe_signature from celery.exceptions import ChordError, ImproperlyConfigured from celery.five import string_t @@ -22,7 +23,8 @@ from celery.utils.log import get_logger from celery.utils.timeutils import humanize_seconds -from .base import KeyValueStoreBackend +from . import async +from . import base try: import redis @@ -47,9 +49,54 @@ error = logger.error -class RedisBackend(KeyValueStoreBackend): +class ResultConsumer(async.BaseResultConsumer): + + _pubsub = None + + def __init__(self, *args, **kwargs): + super(ResultConsumer, self).__init__(*args, **kwargs) + self._get_key_for_task = self.backend.get_key_for_task + self._decode_result = self.backend.decode_result + self.subscribed_to = set() + + def start(self, initial_task_id): + self._pubsub = self.backend.client.pubsub( + ignore_subscribe_messages=True, + ) + self._consume_from(initial_task_id) + + def stop(self): + if self._pubsub is not None: + self._pubsub.close() + + def drain_events(self, timeout=None): + m = self._pubsub.get_message(timeout=timeout) + if m and m['type'] == 'message': + self.on_state_change(self._decode_result(m['data']), m) + + def consume_from(self, task_id): + if self._pubsub is None: + return self.start(task_id) + self._consume_from(task_id) + + def _consume_from(self, task_id): + key = self._get_key_for_task(task_id) + if key not in self.subscribed_to: + self.subscribed_to.add(key) + self._pubsub.subscribe(key) + + def cancel_for(self, task_id): + if self._pubsub: + key = self._get_key_for_task(task_id) + self.subscribed_to.discard(key) + self._pubsub.unsubscribe(key) + + +class RedisBackend(base.BaseKeyValueStoreBackend, async.AsyncBackendMixin): """Redis task result store.""" + ResultConsumer = ResultConsumer + #: redis-py client module. redis = redis @@ -93,6 +140,8 @@ def __init__(self, host=None, port=None, db=None, password=None, self.connection_errors, self.channel_errors = ( get_redis_error_classes() if get_redis_error_classes else ((), ())) + self.result_consumer = self.ResultConsumer( + self, self.app, self.accept, self._pending_results) def _params_from_url(self, url, defaults): scheme, host, port, user, password, path, query = _parse_url(url) @@ -124,6 +173,10 @@ def _params_from_url(self, url, defaults): connparams.update(query) return connparams + def on_task_call(self, producer, task_id): + if not task_join_will_block(): + self.result_consumer.consume_from(task_id) + def get(self, key): return self.client.get(key) diff --git a/celery/backends/rpc.py b/celery/backends/rpc.py index 7c6c68ebb..620055583 100644 --- a/celery/backends/rpc.py +++ b/celery/backends/rpc.py @@ -13,6 +13,7 @@ from kombu.utils import cached_property from celery import current_task +from celery._state import task_join_will_block from celery.backends import amqp __all__ = ['RPCBackend'] @@ -29,7 +30,8 @@ def _create_exchange(self, name, type='direct', delivery_mode=2): return Exchange(None) def on_task_call(self, producer, task_id): - maybe_declare(self.binding(producer.channel), retry=True) + if not task_join_will_block(): + maybe_declare(self.binding(producer.channel), retry=True) def _create_binding(self, task_id): return self.binding From de6feab8c17d50bbf7b47be03412364c080bfc7a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 9 Mar 2016 13:49:56 -0800 Subject: [PATCH 1064/1103] Tiny doc fixes --- docs/configuration.rst | 2 +- examples/django/README.rst | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 81846ed8f..56a22ba16 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -910,7 +910,7 @@ cassandra backend settings To install, use `pip` or `easy_install`: - .. code-block:: bash + .. code-block:: console $ pip install cassandra-driver diff --git a/examples/django/README.rst b/examples/django/README.rst index e41e9b84e..e8e091e96 100644 --- a/examples/django/README.rst +++ b/examples/django/README.rst @@ -37,22 +37,22 @@ http://docs.celeryproject.org/en/latest/getting-started/brokers/rabbitmq.html In addition, some Python requirements must also be satisfied: -.. code-block:: bash +.. code-block:: console $ pip install -r requirements.txt Starting the worker =================== -.. code-block:: bash +.. code-block:: console $ celery -A proj worker -l info Running a task =================== -.. code-block:: bash - +.. code-block:: console + $ python ./manage.sh shell >>> from demoapp.tasks import add, mul, xsum >>> res = add.delay(2,3) From 2dcda99ea6db6c4b655d007692163a94bb7323d1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 9 Mar 2016 13:54:38 -0800 Subject: [PATCH 1065/1103] Fixes regression with worker detach ignoring hostname argument. Closes #3103 --- celery/bin/celeryd_detach.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/celery/bin/celeryd_detach.py b/celery/bin/celeryd_detach.py index f29c05e8e..ed3f0bf9a 100644 --- a/celery/bin/celeryd_detach.py +++ b/celery/bin/celeryd_detach.py @@ -140,6 +140,8 @@ def parse_options(self, prog_name, argv): parser.leftovers.append('--logfile={0}'.format(options.logfile)) if options.pidfile: parser.leftovers.append('--pidfile={0}'.format(options.pidfile)) + if options.hostname: + parser.leftovers.append('--hostname={0}'.format(options.hostname)) return options, values, parser.leftovers def execute_from_commandline(self, argv=None): From 029d443626946aad9ce0a7b41ad7bfcf3b6d9406 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 9 Mar 2016 17:53:52 -0800 Subject: [PATCH 1066/1103] [stress] Fixes parentids_chain test --- funtests/stress/stress/suite.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/funtests/stress/stress/suite.py b/funtests/stress/stress/suite.py index 0f4298aba..e6e1d4d65 100755 --- a/funtests/stress/stress/suite.py +++ b/funtests/stress/stress/suite.py @@ -288,12 +288,12 @@ def chaincomplex(self): assert_equal(res.get(), [32, 33, 34, 35]) @testcase('all', 'green', iterations=1) - def parentids_chain(self): - c = chain(ids.si(i) for i in range(248)) + def parentids_chain(self, num=248): + c = chain(ids.si(i) for i in range(num)) c.freeze() res = c() res.get(timeout=5) - self.assert_ids(res, len(c.tasks) - 1) + self.assert_ids(res, num - 1) @testcase('all', 'green', iterations=1) def parentids_group(self): @@ -309,8 +309,8 @@ def parentids_group(self): assert_equal(parent_id, expected_parent_id) assert_equal(value, i + 2) - def assert_ids(self, res, len): - i, root = len, res + def assert_ids(self, res, size): + i, root = size, res while root.parent: root = root.parent node = res From 5df1ce67d7d9ed783a0f75b0a94ae58411d9efb9 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 9 Mar 2016 17:56:08 -0800 Subject: [PATCH 1067/1103] [result][mongodb] Fixes as_uri() when not configured by url (Issue #3094) --- celery/backends/mongodb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index fd11f4764..938b7e193 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -301,6 +301,8 @@ def as_uri(self, include_password=False): :keyword include_password: Censor passwords. """ + if not self.url: + return 'mongodb://' if include_password: return self.url From 74d5bff213e2ae820d87396b63adceedd4dd40e5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 9 Mar 2016 18:25:59 -0800 Subject: [PATCH 1068/1103] [utils] .five.getfullargspec now returns the same fields as on Python3 --- celery/five.py | 12 +++++++++++- celery/utils/functional.py | 8 ++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/celery/five.py b/celery/five.py index 20462acad..1379f1dc3 100644 --- a/celery/five.py +++ b/celery/five.py @@ -28,7 +28,17 @@ try: # pragma: no cover from inspect import formatargspec, getfullargspec except ImportError: # Py2 - from inspect import formatargspec, getargspec as getfullargspec # noqa + from collections import namedtuple + from inspect import formatargspec, getargspec as _getargspec # noqa + + FullArgSpec = namedtuple('FullArgSpec', ( + 'args', 'varargs', 'varkw', 'defaults', + 'kwonlyargs', 'kwonlydefaults', 'annotations', + )) + + def getfullargspec(fun, _fill=(None, ) * 3): # noqa + s = _getargspec(fun) + return FullArgSpec(*s + _fill) __all__ = [ 'class_property', 'reclassmethod', 'create_module', 'recreate_module', diff --git a/celery/utils/functional.py b/celery/utils/functional.py index a41d464a8..2715743f2 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -367,15 +367,11 @@ def _argsfromspec(spec, replace_defaults=True): optional = list(zip(spec.args[-split:], defaults)) else: positional, optional = spec.args, [] - if IS_PY3: # pragma: no cover - keywords = spec.varkw - elif IS_PY2: - keywords = spec.keywords # noqa return ', '.join(filter(None, [ ', '.join(positional), ', '.join('{0}={1}'.format(k, v) for k, v in optional), '*{0}'.format(spec.varargs) if spec.varargs else None, - '**{0}'.format(keywords) if keywords else None, + '**{0}'.format(spec.varkw) if spec.varkw else None, ])) @@ -403,6 +399,6 @@ def head_from_fun(fun, bound=False, debug=False): def fun_takes_argument(name, fun, position=None): spec = getfullargspec(fun) return ( - spec.keywords or spec.varargs or + spec.varkw or spec.varargs or (len(spec.args) >= position if position else name in spec.args) ) From dc9093c345fe953be0b624b6318cd018672ad02a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 8 Mar 2016 19:29:43 -0800 Subject: [PATCH 1069/1103] [result][redis] Use pubsub for consuming results, and use the new async backend interface (Issue #2511) --- celery/backends/amqp.py | 21 ++++++++------- celery/backends/async.py | 18 ++++++++++--- celery/backends/base.py | 11 +++++--- celery/backends/redis.py | 57 ++++++++++++++++++++++++++++++++++++++-- celery/backends/rpc.py | 4 ++- 5 files changed, 92 insertions(+), 19 deletions(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index 0bb925d19..6af14a192 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -49,13 +49,16 @@ class NoCacheQueue(Queue): class ResultConsumer(BaseResultConsumer): Consumer = Consumer + _connection = None + _consumer = None + def __init__(self, *args, **kwargs): super(ResultConsumer, self).__init__(*args, **kwargs) - self._connection = None - self._consumer = None + self._create_binding = self.backend._create_binding - def start(self, initial_queue, no_ack=True): + def start(self, initial_task_id, no_ack=True): self._connection = self.app.connection() + initial_queue = self._create_binding(initial_task_id) self._consumer = self.Consumer( self._connection.default_channel, [initial_queue], callbacks=[self.on_state_change], no_ack=no_ack, @@ -77,16 +80,17 @@ def on_after_fork(self): self._connection.collect() self._connection = None - def consume_from(self, queue): + def consume_from(self, task_id): if self._consumer is None: - return self.start(queue) + return self.start(task_id) + queue = self._create_binding(task_id) if not self._consumer.consuming_from(queue): self._consumer.add_queue(queue) self._consumer.consume() - def cancel_for(self, queue): + def cancel_for(self, task_id): if self._consumer: - self._consumer.cancel_by_queue(queue.name) + self._consumer.cancel_by_queue(self._create_binding(task_id).name) class AMQPBackend(base.Backend, AsyncBackendMixin): @@ -138,9 +142,6 @@ def _after_fork(self): self._pending_results.clear() self.result_consumer._after_fork() - def on_result_fulfilled(self, result): - self.result_consumer.cancel_for(self._create_binding(result.id)) - def _create_exchange(self, name, type='direct', delivery_mode=2): return self.Exchange(name=name, type=type, diff --git a/celery/backends/async.py b/celery/backends/async.py index ddb56287f..0ff5ac045 100644 --- a/celery/backends/async.py +++ b/celery/backends/async.py @@ -135,7 +135,7 @@ def iter_native(self, result, timeout=None, interval=0.5, no_ack=True, def add_pending_result(self, result): if result.id not in self._pending_results: self._pending_results[result.id] = result - self.result_consumer.consume_from(self._create_binding(result.id)) + self.result_consumer.consume_from(result.id) return result def remove_pending_result(self, result): @@ -144,7 +144,7 @@ def remove_pending_result(self, result): return result def on_result_fulfilled(self, result): - pass + self.result_consumer.cancel_for(result.id) def wait_for_pending(self, result, callback=None, propagate=True, **kwargs): @@ -177,8 +177,20 @@ def __init__(self, backend, app, accept, pending_results): self.buckets = WeakKeyDictionary() self.drainer = drainers[detect_environment()](self) + def start(self): + raise NotImplementedError() + + def stop(self): + pass + def drain_events(self, timeout=None): - raise NotImplementedError('subclass responsibility') + raise NotImplementedError() + + def consume_from(self, task_id): + raise NotImplementedError() + + def cancel_for(self, task_id): + raise NotImplementedError() def _after_fork(self): self.bucket.clear() diff --git a/celery/backends/base.py b/celery/backends/base.py index c1793fa83..4077a5ac8 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -448,7 +448,7 @@ class BaseBackend(Backend, SyncBackendMixin): BaseDictBackend = BaseBackend # XXX compat -class KeyValueStoreBackend(BaseBackend): +class BaseKeyValueStoreBackend(Backend): key_t = ensure_bytes task_keyprefix = 'celery-task-meta-' group_keyprefix = 'celery-taskset-meta-' @@ -459,7 +459,7 @@ def __init__(self, *args, **kwargs): if hasattr(self.key_t, '__func__'): # pragma: no cover self.key_t = self.key_t.__func__ # remove binding self._encode_prefixes() - super(KeyValueStoreBackend, self).__init__(*args, **kwargs) + super(BaseKeyValueStoreBackend, self).__init__(*args, **kwargs) if self.implements_incr: self.apply_chord = self._apply_chord_incr @@ -578,7 +578,8 @@ def _forget(self, task_id): def _store_result(self, task_id, result, state, traceback=None, request=None, **kwargs): meta = {'status': state, 'result': result, 'traceback': traceback, - 'children': self.current_task_children(request)} + 'children': self.current_task_children(request), + 'task_id': task_id} self.set(self.get_key_for_task(task_id), self.encode(meta)) return result @@ -683,6 +684,10 @@ def on_chord_part_return(self, request, state, result, **kwargs): self.expire(key, 86400) +class KeyValueStoreBackend(BaseKeyValueStoreBackend, SyncBackendMixin): + pass + + class DisabledBackend(BaseBackend): _cache = {} # need this attribute to reset cache in tests. diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 00bc01227..8cbb8fe27 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -14,6 +14,7 @@ from kombu.utils.url import _parse_url from celery import states +from celery._state import task_join_will_block from celery.canvas import maybe_signature from celery.exceptions import ChordError, ImproperlyConfigured from celery.five import string_t @@ -22,7 +23,8 @@ from celery.utils.log import get_logger from celery.utils.timeutils import humanize_seconds -from .base import KeyValueStoreBackend +from . import async +from . import base try: import redis @@ -47,9 +49,54 @@ error = logger.error -class RedisBackend(KeyValueStoreBackend): +class ResultConsumer(async.BaseResultConsumer): + + _pubsub = None + + def __init__(self, *args, **kwargs): + super(ResultConsumer, self).__init__(*args, **kwargs) + self._get_key_for_task = self.backend.get_key_for_task + self._decode_result = self.backend.decode_result + self.subscribed_to = set() + + def start(self, initial_task_id): + self._pubsub = self.backend.client.pubsub( + ignore_subscribe_messages=True, + ) + self._consume_from(initial_task_id) + + def stop(self): + if self._pubsub is not None: + self._pubsub.close() + + def drain_events(self, timeout=None): + m = self._pubsub.get_message(timeout=timeout) + if m and m['type'] == 'message': + self.on_state_change(self._decode_result(m['data']), m) + + def consume_from(self, task_id): + if self._pubsub is None: + return self.start(task_id) + self._consume_from(task_id) + + def _consume_from(self, task_id): + key = self._get_key_for_task(task_id) + if key not in self.subscribed_to: + self.subscribed_to.add(key) + self._pubsub.subscribe(key) + + def cancel_for(self, task_id): + if self._pubsub: + key = self._get_key_for_task(task_id) + self.subscribed_to.discard(key) + self._pubsub.unsubscribe(key) + + +class RedisBackend(base.BaseKeyValueStoreBackend, async.AsyncBackendMixin): """Redis task result store.""" + ResultConsumer = ResultConsumer + #: redis-py client module. redis = redis @@ -93,6 +140,8 @@ def __init__(self, host=None, port=None, db=None, password=None, self.connection_errors, self.channel_errors = ( get_redis_error_classes() if get_redis_error_classes else ((), ())) + self.result_consumer = self.ResultConsumer( + self, self.app, self.accept, self._pending_results) def _params_from_url(self, url, defaults): scheme, host, port, user, password, path, query = _parse_url(url) @@ -124,6 +173,10 @@ def _params_from_url(self, url, defaults): connparams.update(query) return connparams + def on_task_call(self, producer, task_id): + if not task_join_will_block(): + self.result_consumer.consume_from(task_id) + def get(self, key): return self.client.get(key) diff --git a/celery/backends/rpc.py b/celery/backends/rpc.py index 7c6c68ebb..620055583 100644 --- a/celery/backends/rpc.py +++ b/celery/backends/rpc.py @@ -13,6 +13,7 @@ from kombu.utils import cached_property from celery import current_task +from celery._state import task_join_will_block from celery.backends import amqp __all__ = ['RPCBackend'] @@ -29,7 +30,8 @@ def _create_exchange(self, name, type='direct', delivery_mode=2): return Exchange(None) def on_task_call(self, producer, task_id): - maybe_declare(self.binding(producer.channel), retry=True) + if not task_join_will_block(): + maybe_declare(self.binding(producer.channel), retry=True) def _create_binding(self, task_id): return self.binding From 868d4b17a51fcbc346cae07c7896e96c9ac2e6d9 Mon Sep 17 00:00:00 2001 From: Alexandru Chirila Date: Thu, 10 Mar 2016 11:00:42 +0200 Subject: [PATCH 1070/1103] Describe the `virtual_host` parameter Add details about changing the database number while using UNIX socket with a Redis broker. --- docs/getting-started/brokers/redis.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/getting-started/brokers/redis.rst b/docs/getting-started/brokers/redis.rst index ac6ef7c85..ef2f6b8eb 100644 --- a/docs/getting-started/brokers/redis.rst +++ b/docs/getting-started/brokers/redis.rst @@ -38,6 +38,11 @@ If a unix socket connection should be used, the URL needs to be in the format:: redis+socket:///path/to/redis.sock +Specifying a different database while using a unix socket is possible by adding the +`virtual_host` parameter to the URL:: + + redis+socket:///path/to/redis.sock?virtual_host=db_number + .. _redis-visibility_timeout: Visibility Timeout From d07fb545df1b464851bad0e10be191c0bbb9e30a Mon Sep 17 00:00:00 2001 From: Alexander Oblovatniy Date: Thu, 10 Mar 2016 19:01:05 +0200 Subject: [PATCH 1071/1103] Update gevent.py Return missing information about pool size. This will allow to get pool size via `control.inspect().stats()`. `eventlet`, `prefork` and `solo` implementations already have this. This is essential to have such feature, for example, to know how much tasks a single pool can consume at one time. --- celery/concurrency/gevent.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/celery/concurrency/gevent.py b/celery/concurrency/gevent.py index dc0f13203..1e79a8ff5 100644 --- a/celery/concurrency/gevent.py +++ b/celery/concurrency/gevent.py @@ -121,3 +121,8 @@ def shrink(self, n=1): @property def num_processes(self): return len(self._pool) + + def _get_info(self): + info = super(TaskPool, self)._get_info() + info['max-concurrency'] = self.limit + return info From 2ed53953b0017058ef496d9ab9db1c7b3633756f Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Mar 2016 10:34:41 -0800 Subject: [PATCH 1072/1103] Cosmetics for #3108 --- celery/concurrency/base.py | 4 +++- celery/concurrency/gevent.py | 5 ----- celery/tests/concurrency/test_concurrency.py | 4 +++- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/celery/concurrency/base.py b/celery/concurrency/base.py index 4b2e7a15d..e40d1d1a6 100644 --- a/celery/concurrency/base.py +++ b/celery/concurrency/base.py @@ -159,7 +159,9 @@ def apply_async(self, target, args=[], kwargs={}, **options): **options) def _get_info(self): - return {} + return { + 'max-concurrency': self.limit, + } @property def info(self): diff --git a/celery/concurrency/gevent.py b/celery/concurrency/gevent.py index 1e79a8ff5..dc0f13203 100644 --- a/celery/concurrency/gevent.py +++ b/celery/concurrency/gevent.py @@ -121,8 +121,3 @@ def shrink(self, n=1): @property def num_processes(self): return len(self._pool) - - def _get_info(self): - info = super(TaskPool, self)._get_info() - info['max-concurrency'] = self.limit - return info diff --git a/celery/tests/concurrency/test_concurrency.py b/celery/tests/concurrency/test_concurrency.py index 0ea7d6567..7bc021c0c 100644 --- a/celery/tests/concurrency/test_concurrency.py +++ b/celery/tests/concurrency/test_concurrency.py @@ -107,7 +107,9 @@ def test_interface_on_apply(self): BasePool(10).on_apply() def test_interface_info(self): - self.assertDictEqual(BasePool(10).info, {}) + self.assertDictEqual(BasePool(10).info, { + 'max-concurrency': 10, + }) def test_interface_flush(self): self.assertIsNone(BasePool(10).flush()) From 7aacb808658b5caa27e6874638da41726a1438da Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Mar 2016 10:41:50 -0800 Subject: [PATCH 1073/1103] [pool][eventlet] ._get_info now calls super --- celery/concurrency/eventlet.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/celery/concurrency/eventlet.py b/celery/concurrency/eventlet.py index 6991e0608..c867fd01b 100644 --- a/celery/concurrency/eventlet.py +++ b/celery/concurrency/eventlet.py @@ -143,8 +143,10 @@ def shrink(self, n=1): self.limit = limit def _get_info(self): - return { + info = super(TaskPool, self)._get_info() + info.update({ 'max-concurrency': self.limit, 'free-threads': self._pool.free(), 'running-threads': self._pool.running(), - } + }) + return info From cf07612639378f86c77b53329bc70439f299b033 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Mar 2016 10:42:58 -0800 Subject: [PATCH 1074/1103] Cosmetics for #3106 --- docs/getting-started/brokers/redis.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started/brokers/redis.rst b/docs/getting-started/brokers/redis.rst index ef2f6b8eb..c2329efed 100644 --- a/docs/getting-started/brokers/redis.rst +++ b/docs/getting-started/brokers/redis.rst @@ -38,8 +38,8 @@ If a unix socket connection should be used, the URL needs to be in the format:: redis+socket:///path/to/redis.sock -Specifying a different database while using a unix socket is possible by adding the -`virtual_host` parameter to the URL:: +Specifying a different database number when using a unix socket is possible +by adding the ``virtual_host`` parameter to the URL:: redis+socket:///path/to/redis.sock?virtual_host=db_number From 089469843bf82bba4e5d9a762f79e352845303e5 Mon Sep 17 00:00:00 2001 From: raducc Date: Thu, 3 Mar 2016 11:43:46 +0200 Subject: [PATCH 1075/1103] fixed reversed min max values --- celery/worker/control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/worker/control.py b/celery/worker/control.py index 74ac0c33f..f223ff154 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -340,7 +340,7 @@ def autoscale(state, max=None, min=None): autoscaler = state.consumer.controller.autoscaler if autoscaler: max_, min_ = autoscaler.update(max, min) - return ok('autoscale now min={0} max={1}'.format(max_, min_)) + return ok('autoscale now max={0} min={1}'.format(max_, min_)) raise ValueError('Autoscale not enabled') From c636e19c2e36124704ba19b6fb897b88ab48b03e Mon Sep 17 00:00:00 2001 From: dessant Date: Sat, 27 Feb 2016 13:34:04 +0200 Subject: [PATCH 1076/1103] mention that event capturing can be stopped by setting should_stop to True --- celery/events/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/celery/events/__init__.py b/celery/events/__init__.py index 23b3ea0da..8c77a9751 100644 --- a/celery/events/__init__.py +++ b/celery/events/__init__.py @@ -358,8 +358,9 @@ def itercapture(self, limit=None, timeout=None, wakeup=True): def capture(self, limit=None, timeout=None, wakeup=True): """Open up a consumer capturing events. - This has to run in the main process, and it will never - stop unless forced via :exc:`KeyboardInterrupt` or :exc:`SystemExit`. + This has to run in the main process, and it will never stop + unless :attr:`EventDispatcher.should_stop` is set to True, or + forced via :exc:`KeyboardInterrupt` or :exc:`SystemExit`. """ return list(self.consume(limit=limit, timeout=timeout, wakeup=wakeup)) From 040a315b6009e2fe1a842470aabc7309e53b9c6e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Mar 2016 11:01:02 -0800 Subject: [PATCH 1077/1103] [deployment][generic-init.d] Adds the ability to set `su` options. Closes #3055 This adds the following configuration options that can be modified in ``/etc/init.d/celeryd``/``/etc/init.d/celerybeat``. - ``CELERYD_SU`` path to su utility. - ``CELERYD_SU_OPTIONS`` arguments to su. - ``CELERYBEAT_SU`` - ``CELERYBEAT_SU_OPTIONS`` --- docs/tutorials/daemonizing.rst | 13 +++++++++++++ extra/generic-init.d/celerybeat | 6 +++++- extra/generic-init.d/celeryd | 8 +++++++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/daemonizing.rst b/docs/tutorials/daemonizing.rst index be8a5b8a8..feb51afab 100644 --- a/docs/tutorials/daemonizing.rst +++ b/docs/tutorials/daemonizing.rst @@ -119,6 +119,19 @@ This is an example configuration for a Python project. # and owned by the userid/group configured. CELERY_CREATE_DIRS=1 +Using a login shell +~~~~~~~~~~~~~~~~~~~ + +You can inherit the environment of the ``CELERYD_USER`` by using a login +shell: + +.. code-block:: bash + + CELERYD_SU_OPTIONS="-l" + +Note that this is not recommended, and that you should only use this option +when absolutely necessary. + .. _generic-initd-celeryd-django-example: Example Django configuration diff --git a/extra/generic-init.d/celerybeat b/extra/generic-init.d/celerybeat index 5d221e630..5cdbd2b64 100755 --- a/extra/generic-init.d/celerybeat +++ b/extra/generic-init.d/celerybeat @@ -110,6 +110,9 @@ DEFAULT_CELERYBEAT="$CELERY_BIN beat" CELERYBEAT=${CELERYBEAT:-$DEFAULT_CELERYBEAT} CELERYBEAT_LOG_LEVEL=${CELERYBEAT_LOG_LEVEL:-${CELERYBEAT_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} +CELERYBEAT_SU=${CELERYBEAT_SU:-"su"} +CELERYBEAT_SU_ARGS=${CELERYBEAT_SU_ARGS:-""} + # Sets --app argument for CELERY_BIN CELERY_APP_ARG="" if [ ! -z "$CELERY_APP" ]; then @@ -240,7 +243,8 @@ stop_beat () { } _chuid () { - su "$CELERYBEAT_USER" -c "$CELERYBEAT $*" + ${CELERYBEAT_SU} ${CELERYBEAT_SU_ARGS} \ + "$CELERYBEAT_USER" -c "$CELERYBEAT $*" } start_beat () { diff --git a/extra/generic-init.d/celeryd b/extra/generic-init.d/celeryd index 9dd43e9b7..a67a40c2b 100755 --- a/extra/generic-init.d/celeryd +++ b/extra/generic-init.d/celeryd @@ -115,6 +115,12 @@ if [ ! -z "$CELERY_APP" ]; then CELERY_APP_ARG="--app=$CELERY_APP" fi +# Options to su +# can be used to enable login shell (CELERYD_SU_ARGS="-l"), +# or even to use start-stop-daemon instead of su. +CELERYD_SU=${CELERY_SU:-"su"} +CELERYD_SU_ARGS=${CELERYD_SU_ARGS:-""} + CELERYD_USER=${CELERYD_USER:-$DEFAULT_USER} # Set CELERY_CREATE_DIRS to always create log/pid dirs. @@ -235,7 +241,7 @@ _get_pids() { _chuid () { - su "$CELERYD_USER" -c "$CELERYD_MULTI $*" + ${CELERYD_SU} ${CELERYD_SU_ARGS} "$CELERYD_USER" -c "$CELERYD_MULTI $*" } From fc4ca886fdfa25078e05576279354ece64df021e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Mar 2016 11:03:28 -0800 Subject: [PATCH 1078/1103] Fixes typo SU_OPTIONS -> SU_ARGS (Issue #3055) --- docs/tutorials/daemonizing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/daemonizing.rst b/docs/tutorials/daemonizing.rst index feb51afab..9895338e0 100644 --- a/docs/tutorials/daemonizing.rst +++ b/docs/tutorials/daemonizing.rst @@ -127,7 +127,7 @@ shell: .. code-block:: bash - CELERYD_SU_OPTIONS="-l" + CELERYD_SU_ARGS="-l" Note that this is not recommended, and that you should only use this option when absolutely necessary. From d978d862d8bf6b2785a73044907e40206677b589 Mon Sep 17 00:00:00 2001 From: Alexander Oblovatniy Date: Thu, 10 Mar 2016 21:31:33 +0200 Subject: [PATCH 1079/1103] add @oblalex to 'CONTRIBUTORS.txt' as proposed in pull request #3108 --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 5ddcf8ca9..63fc7b60a 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -206,3 +206,4 @@ Mike Attwood, 2016/01/22 David Harrigan, 2016/02/01 Ahmet Demir, 2016/02/27 Maxime Verger, 2016/02/29 +Alexander Oblovatniy, 2016/03/10 From 2c2984e045d33d8a06a5293bd629c504356ec9d5 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Mar 2016 13:38:49 -0800 Subject: [PATCH 1080/1103] [utils] Adds %N abbreviation expanding to the full worker node name --- celery/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/utils/__init__.py b/celery/utils/__init__.py index d6053bc65..40f90cddc 100644 --- a/celery/utils/__init__.py +++ b/celery/utils/__init__.py @@ -355,7 +355,7 @@ def default_nodename(hostname): def node_format(s, nodename, **extra): name, host = nodesplit(nodename) return host_format( - s, host, name or NODENAME_DEFAULT, **extra) + s, host, name or NODENAME_DEFAULT, N=nodename, **extra) def _fmt_process_index(prefix='', default='0'): From f56ff68ab597f9b17fd6d211f080ebd466da3d7b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Mar 2016 13:39:36 -0800 Subject: [PATCH 1081/1103] [docs] Use nodename instead of hostname. Closes #3104 --- docs/userguide/monitoring.rst | 4 ++-- docs/userguide/signals.rst | 2 +- docs/userguide/tasks.rst | 2 +- docs/userguide/workers.rst | 12 +++++++----- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index eb5f42160..0009f1946 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -745,7 +745,7 @@ worker-online The worker has connected to the broker and is online. -- `hostname`: Hostname of the worker. +- `hostname`: Nodename of the worker. - `timestamp`: Event timestamp. - `freq`: Heartbeat frequency in seconds (float). - `sw_ident`: Name of worker software (e.g. ``py-celery``). @@ -763,7 +763,7 @@ worker-heartbeat Sent every minute, if the worker has not sent a heartbeat in 2 minutes, it is considered to be offline. -- `hostname`: Hostname of the worker. +- `hostname`: Nodename of the worker. - `timestamp`: Event timestamp. - `freq`: Heartbeat frequency in seconds (float). - `sw_ident`: Name of worker software (e.g. ``py-celery``). diff --git a/docs/userguide/signals.rst b/docs/userguide/signals.rst index db5c1eb65..a22a4bcb8 100644 --- a/docs/userguide/signals.rst +++ b/docs/userguide/signals.rst @@ -390,7 +390,7 @@ used to route a task to any specific worker: Provides arguments: * sender - Hostname of the worker. + Nodename of the worker. * instance This is the :class:`celery.apps.worker.Worker` instance to be initialized. diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index dc8e79ce6..0579aca0b 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -342,7 +342,7 @@ The request defines the following attributes: :loglevel: The current log level used. -:hostname: Hostname of the worker instance executing the task. +:hostname: Node name of the worker instance executing the task. :delivery_info: Additional message delivery information. This is a mapping containing the exchange and routing key used to deliver this diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index 0b8d0d949..b3ea95de1 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -34,7 +34,7 @@ For a full list of available command-line options see You can also start multiple workers on the same machine. If you do so be sure to give a unique name to each individual worker by specifying a -host name with the :option:`--hostname|-n` argument: +node name with the :option:`--hostname|-n` argument: .. code-block:: console @@ -42,7 +42,7 @@ host name with the :option:`--hostname|-n` argument: $ celery -A proj worker --loglevel=INFO --concurrency=10 -n worker2.%h $ celery -A proj worker --loglevel=INFO --concurrency=10 -n worker3.%h -The hostname argument can expand the following variables: +The ``hostname`` argument can expand the following variables: - ``%h``: Hostname including domain name. - ``%n``: Hostname only. @@ -149,16 +149,18 @@ can contain variables that the worker will expand: Node name replacements ---------------------- +- ``%N``: Full node name. - ``%h``: Hostname including domain name. - ``%n``: Hostname only. - ``%d``: Domain name only. - ``%i``: Prefork pool process index or 0 if MainProcess. - ``%I``: Prefork pool process index with separator. -E.g. if the current hostname is ``george.example.com`` then +E.g. if the current hostname is ``george@foo.example.com`` then these will expand to: -- ``--logfile=%h.log`` -> :file:`george.example.com.log` +- ``--logfile-%N.log`` -> :file:`george@foo.example.com.log` +- ``--logfile=%h.log`` -> :file:`foo.example.com.log` - ``--logfile=%n.log`` -> :file:`george.log` - ``--logfile=%d`` -> :file:`example.com.log` @@ -968,7 +970,7 @@ The output will include the following fields: * ``hostname`` - Hostname of the remote broker. + Node name of the remote broker. * ``insist`` From 376ee40ecc02a99b4db72b746d1bc8b202691ab9 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 10 Mar 2016 15:51:57 -0800 Subject: [PATCH 1082/1103] Apparently %N already taken by multi, so have to use %p for full nodename --- celery/utils/__init__.py | 2 +- docs/userguide/workers.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/utils/__init__.py b/celery/utils/__init__.py index 40f90cddc..697ee66a8 100644 --- a/celery/utils/__init__.py +++ b/celery/utils/__init__.py @@ -355,7 +355,7 @@ def default_nodename(hostname): def node_format(s, nodename, **extra): name, host = nodesplit(nodename) return host_format( - s, host, name or NODENAME_DEFAULT, N=nodename, **extra) + s, host, name or NODENAME_DEFAULT, p=nodename, **extra) def _fmt_process_index(prefix='', default='0'): diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index b3ea95de1..ffff5be32 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -149,7 +149,7 @@ can contain variables that the worker will expand: Node name replacements ---------------------- -- ``%N``: Full node name. +- ``%p``: Full node name. - ``%h``: Hostname including domain name. - ``%n``: Hostname only. - ``%d``: Domain name only. @@ -159,7 +159,7 @@ Node name replacements E.g. if the current hostname is ``george@foo.example.com`` then these will expand to: -- ``--logfile-%N.log`` -> :file:`george@foo.example.com.log` +- ``--logfile-%p.log`` -> :file:`george@foo.example.com.log` - ``--logfile=%h.log`` -> :file:`foo.example.com.log` - ``--logfile=%n.log`` -> :file:`george.log` - ``--logfile=%d`` -> :file:`example.com.log` From c98aa2d41b1956322bcbc569358cf9b7cb12e666 Mon Sep 17 00:00:00 2001 From: Zoran Pavlovic Date: Fri, 11 Mar 2016 10:38:48 +0200 Subject: [PATCH 1083/1103] Database backend url passing Fixed bug where database backend not passing through URL to BaseBackend __init__. --- celery/backends/database/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/celery/backends/database/__init__.py b/celery/backends/database/__init__.py index b63adb816..2a88687a0 100644 --- a/celery/backends/database/__init__.py +++ b/celery/backends/database/__init__.py @@ -77,7 +77,9 @@ def __init__(self, dburi=None, engine_options=None, url=None, **kwargs): # The `url` argument was added later and is used by # the app to set backend by url (celery.backends.get_backend_by_url) super(DatabaseBackend, self).__init__( - expires_type=maybe_timedelta, **kwargs + expires_type=maybe_timedelta, + url=url, + **kwargs ) conf = self.app.conf self.url = url or dburi or conf.sqlalchemy_dburi From e0221e9f69a08674c8a019a90b224223560161f7 Mon Sep 17 00:00:00 2001 From: David Pravec Date: Wed, 9 Mar 2016 11:06:16 +0100 Subject: [PATCH 1084/1103] Improvements and fixes for LimitedSet Getting rid of leaking memory + adding minlen size of the set minlen is minimal residual size of set after operating for long. Minlen items are kept, even if they should be expired by time, until we get newer items. Problems with older and even more old code: 1) Heap would tend to grow in some scenarios (like adding an item multiple times). 2) Adding many items fast would not clean them soon enough (if ever). 3) When talking to other workers, revoked._data was sent, but it was processed on the other side as iterable. That means giving those keys new (current) timestamp. By doing this workers could recycle items forever. Combined with 1) and 2), this means that in large set of workers, you are getting out of memory soon. All those problems should be fixed now, also some new unittests are added. This should fix issues #3095, #3086. --- celery/datastructures.py | 237 +++++++++++++++------- celery/tests/utils/test_datastructures.py | 103 +++++++--- 2 files changed, 245 insertions(+), 95 deletions(-) diff --git a/celery/datastructures.py b/celery/datastructures.py index a4258657d..adab27bf6 100644 --- a/celery/datastructures.py +++ b/celery/datastructures.py @@ -583,120 +583,217 @@ def values(self): class LimitedSet(object): - """Kind-of Set with limitations. + """Kind-of Set (or priority queue) with limitations. Good for when you need to test for membership (`a in set`), but the set should not grow unbounded. - :keyword maxlen: Maximum number of members before we start - evicting expired members. - :keyword expires: Time in seconds, before a membership expires. - + This version is now changed to be more enforcing those limits. + Maxlen is enforced all the time. But you can also configure + minlen now, which is minimal residual size of set. + + + Example:: + + >>> s = LimitedSet(maxlen=50000, expires=3600, minlen=4000) + >>> for i in range(60000): + ... s.add(i) + ... s.add(str(i)) + ... + >>> 57000 in s # last 50k inserted values are kept + True + >>> '10' in s # '10' did expire and was purged from set. + False + >>> len(s) # maxlen is reached + 50000 + >>> s.purge(now=time.time() + 7200) # clock + 2 hours + >>> len(s) # now only minlen items are cached + 4000 + >>>> 57000 in s # even this item is gone now + False """ - def __init__(self, maxlen=None, expires=None, data=None, heap=None): - # heap is ignored + REMOVED = object() # just a placeholder for removed items + _MAX_HEAP_PERCENTS_OVERLOAD = 15 # + + def __init__(self, maxlen=0, expires=0, minlen=0, data=None): + """Initialize LimitedSet. + + All arguments are optional, with exception of minlen, which must + be smaller than maxlen. Unconfigured limits will not be enforced. + + :keyword maxlen: max size of this set. Adding more items than maxlen + results in immediate removing of older items. + :keyword expires: TTL for an item. + Items aging over expiration are purged. + :keyword minlen: minimal residual size of this set. + Oldest expired items will be delete + only until minlen size is reached. + :keyword data: data to initialize set with. Can be iterable of keys, + dict {key:inserted_time} or another LimitedSet. + + """ + if maxlen is None: + maxlen = 0 + if minlen is None: + minlen = 0 + if expires is None: + expires = 0 self.maxlen = maxlen + self.minlen = minlen self.expires = expires - self._data = {} if data is None else data + self._data = {} self._heap = [] - # make shortcuts - self.__len__ = self._heap.__len__ + self.__len__ = self._data.__len__ self.__contains__ = self._data.__contains__ - self._refresh_heap() + if data: + # import items from data + self.update(data) + + if not self.maxlen >= self.minlen >= 0: + raise ValueError('Minlen should be positive number, ' + 'smaller or equal to maxlen.') + if self.expires < 0: + raise ValueError('Expires should not be negative!') def _refresh_heap(self): - self._heap[:] = [(t, key) for key, t in items(self._data)] + """Time consuming recreating of heap. Do not run this too often.""" + self._heap[:] = [entry for entry in self._data.values()] heapify(self._heap) - def add(self, key, now=time.time, heappush=heappush): - """Add a new member.""" - # offset is there to modify the length of the list, - # this way we can expire an item before inserting the value, - # and it will end up in the correct order. - self.purge(1, offset=1) - inserted = now() - self._data[key] = inserted - heappush(self._heap, (inserted, key)) - def clear(self): - """Remove all members""" + """Clear all data, start from scratch again.""" self._data.clear() self._heap[:] = [] - def discard(self, value): - """Remove membership by finding value.""" - try: - itime = self._data[value] - except KeyError: - return - try: - self._heap.remove((itime, value)) - except ValueError: - pass - self._data.pop(value, None) - pop_value = discard # XXX compat - - def purge(self, limit=None, offset=0, now=time.time): - """Purge expired items.""" - H, maxlen = self._heap, self.maxlen - if not maxlen: - return - - # If the data/heap gets corrupted and limit is None - # this will go into an infinite loop, so limit must - # have a value to guard the loop. - limit = len(self) + offset if limit is None else limit - - i = 0 - while len(self) + offset > maxlen: - if i >= limit: - break - try: - item = heappop(H) - except IndexError: - break - if self.expires: - if now() < item[0] + self.expires: - heappush(H, item) - break - try: - self._data.pop(item[1]) - except KeyError: # out of sync with heap - pass - i += 1 + def add(self, item, now=None): + 'Add a new item or update the time of an existing item' + if not now: + now = time.time() + if item in self._data: + self.discard(item) + entry = [now, item] + self._data[item] = entry + heappush(self._heap, entry) + if self.maxlen and len(self._data) >= self.maxlen: + self.purge() def update(self, other): + """Update this LimitedSet from other LimitedSet, dict or iterable.""" if isinstance(other, LimitedSet): self._data.update(other._data) self._refresh_heap() + self.purge() + elif isinstance(other, dict): + # revokes are sent like dict! + for key, inserted in other.items(): + if isinstance(inserted, list): + # in case someone uses ._data directly for sending update + inserted = inserted[0] + if not isinstance(inserted, float): + raise ValueError('Expecting float timestamp, got type ' + '"{0}" with value: {1}'.format( + type(inserted), inserted)) + self.add(key, inserted) else: + # AVOID THIS, it could keep old data if more parties + # exchange them all over and over again for obj in other: self.add(obj) + def discard(self, item): + 'Mark an existing item as REMOVED. If KeyError is not found, pass.' + entry = self._data.pop(item, self.REMOVED) + if entry is self.REMOVED: + return + entry[-1] = self.REMOVED + if self._heap_overload > self._MAX_HEAP_PERCENTS_OVERLOAD: + self._refresh_heap() + + pop_value = discard + + def purge(self, now=None): + """Check oldest items and remove them if needed. + + :keyword now: Time of purging -- by default right now. + This can be usefull for unittesting. + """ + if not now: + now = time.time() + if hasattr(now, '__call__'): + now = now() # if we got this now as function, evaluate it + if self.maxlen: + while len(self._data) > self.maxlen: + self.pop() + # time based expiring: + if self.expires: + while len(self._data) > self.minlen >= 0: + inserted_time, _ = self._heap[0] + if inserted_time + self.expires > now: + break # end this right now, oldest item is not expired yet + self.pop() + + def pop(self): + 'Remove and return the lowest time item. Return None if empty.' + while self._heap: + _, item = heappop(self._heap) + if item is not self.REMOVED: + del self._data[item] + return item + return None + def as_dict(self): - return self._data + """Whole set as serializable dictionary. + Example:: + + >>> s=LimitedSet(maxlen=200) + >>> r=LimitedSet(maxlen=200) + >>> for i in range(500): + ... s.add(i) + ... + >>> r.update(s.as_dict()) + >>> r == s + True + """ + return {key: inserted for inserted, key in self._data.values()} def __eq__(self, other): - return self._heap == other._heap + return self._data == other._data def __ne__(self, other): return not self.__eq__(other) def __repr__(self): - return 'LimitedSet({0})'.format(len(self)) + return 'LimitedSet(maxlen={0}, expires={1}, minlen={2})' \ + ' Current size:{3}'.format( + self.maxlen, self.expires, self.minlen, len(self._data)) def __iter__(self): - return (item[1] for item in self._heap) + # return (item[1] for item in + # self._heap if item[-1] is not self.REMOVED) + # ^ not ordered, slow + return (i for _, i in sorted(self._data.values())) def __len__(self): - return len(self._heap) + return len(self._data) def __contains__(self, key): return key in self._data def __reduce__(self): - return self.__class__, (self.maxlen, self.expires, self._data) + """Pickle helper class. + + This object can be pickled and upickled.""" + return self.__class__, ( + self.maxlen, self.expires, self.minlen, self.as_dict()) + + @property + def _heap_overload(self): + """Compute how much is heap bigger than data [percents].""" + if len(self._data) == 0: + return len(self._heap) + return len(self._heap)*100/len(self._data) - 100 + MutableSet.register(LimitedSet) diff --git a/celery/tests/utils/test_datastructures.py b/celery/tests/utils/test_datastructures.py index 49be7a90e..0c294ff66 100644 --- a/celery/tests/utils/test_datastructures.py +++ b/celery/tests/utils/test_datastructures.py @@ -188,45 +188,58 @@ def test_add(self): for n in 'bar', 'baz': self.assertIn(n, s) self.assertNotIn('foo', s) + s = LimitedSet(maxlen=10) + for i in range(150): + s.add(i) + self.assertLessEqual(len(s), 10) + # make sure heap is not leaking: + self.assertLessEqual(len(s._heap), + len(s) * (100. + + s._MAX_HEAP_PERCENTS_OVERLOAD) / 100) def test_purge(self): - s = LimitedSet(maxlen=None) + # purge now enforces rules + # cant purge(1) now. but .purge(now=...) still works + s = LimitedSet(maxlen=10) [s.add(i) for i in range(10)] s.maxlen = 2 - s.purge(1) - self.assertEqual(len(s), 9) - s.purge(None) + s.purge() self.assertEqual(len(s), 2) # expired - s = LimitedSet(maxlen=None, expires=1) + s = LimitedSet(maxlen=10, expires=1) [s.add(i) for i in range(10)] s.maxlen = 2 - s.purge(1, now=lambda: time() + 100) - self.assertEqual(len(s), 9) - s.purge(None, now=lambda: time() + 100) - self.assertEqual(len(s), 2) + s.purge(now=time() + 100) + self.assertEqual(len(s), 0) # not expired s = LimitedSet(maxlen=None, expires=1) [s.add(i) for i in range(10)] s.maxlen = 2 - s.purge(1, now=lambda: time() - 100) - self.assertEqual(len(s), 10) - s.purge(None, now=lambda: time() - 100) - self.assertEqual(len(s), 10) + s.purge(now=lambda: time() - 100) + self.assertEqual(len(s), 2) - s = LimitedSet(maxlen=None) - [s.add(i) for i in range(10)] - s.maxlen = 2 - with patch('celery.datastructures.heappop') as hp: - hp.side_effect = IndexError() - s.purge() - hp.assert_called_with(s._heap) - with patch('celery.datastructures.heappop') as hp: - s._data = {i * 2: i * 2 for i in range(10)} - s.purge() - self.assertEqual(hp.call_count, 10) + # expired -> minsize + s = LimitedSet(maxlen=10, minlen=10, expires=1) + [s.add(i) for i in range(20)] + s.minlen = 3 + s.purge(now=time() + 3) + self.assertEqual(s.minlen, len(s)) + self.assertLessEqual(len(s._heap), + s.maxlen * + (100. + s._MAX_HEAP_PERCENTS_OVERLOAD)/100) + # s = LimitedSet(maxlen=None) + # [s.add(i) for i in range(10)] + # s.maxlen = 2 + # with patch('celery.datastructures.heappop') as hp: + # hp.side_effect = IndexError() + # s.purge() + # hp.assert_called_with(s._heap) + # with patch('celery.datastructures.heappop') as hp: + # s._data = {i * 2: i * 2 for i in range(10)} + # s.purge() + # self.assertEqual(hp.call_count, 10) def test_pickleable(self): s = LimitedSet(maxlen=2) @@ -260,7 +273,7 @@ def test_discard(self): s.discard('foo') self.assertNotIn('foo', s) self.assertEqual(len(s._data), 0) - self.assertEqual(len(s._heap), 0) + # self.assertLessEqual(len(s._heap), 0 + s.heap_overload) s.discard('foo') def test_clear(self): @@ -285,6 +298,46 @@ def test_update(self): s2.update(['do', 're']) self.assertItemsEqual(list(s2), ['do', 're']) + s1 = LimitedSet(maxlen=10, expires=None) + s2 = LimitedSet(maxlen=10, expires=None) + s3 = LimitedSet(maxlen=10, expires=None) + s4 = LimitedSet(maxlen=10, expires=None) + s5 = LimitedSet(maxlen=10, expires=None) + for i in range(12): + s1.add(i) + s2.add(i*i) + s3.update(s1) + s3.update(s2) + s4.update(s1.as_dict()) + s4.update(s2.as_dict()) + s5.update(s1._data) # revoke is using this + s5.update(s2._data) # + self.assertEqual(s3, s4) + self.assertEqual(s3, s5) + s2.update(s4) + s4.update(s2) + self.assertEqual(s2, s4) + + def test_iterable_and_ordering(self): + s = LimitedSet(maxlen=35, expires=None) + for i in reversed(range(15)): + s.add(i) + j = 40 + for i in s: + self.assertLess(i, j) # each item is smaller and smaller + j = i + self.assertEqual(i, 0) # last item = 0 + + def test_pop_and_ordering_again(self): + s = LimitedSet(maxlen=5) + for i in range(10): + s.add(i) + j = -1 + for _ in range(5): + i = s.pop() + self.assertLess(j, i) + i = s.pop() + self.assertEqual(i, None) def test_as_dict(self): s = LimitedSet(maxlen=2) From 7c4c6eb22e56c129eb2f72452fd8c3e1f22eba6e Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Mar 2016 18:26:01 -0800 Subject: [PATCH 1085/1103] Cosmetics for #3102 --- celery/datastructures.py | 178 +++++++++++----------- celery/tests/utils/test_datastructures.py | 34 ++--- 2 files changed, 105 insertions(+), 107 deletions(-) diff --git a/celery/datastructures.py b/celery/datastructures.py index adab27bf6..e897f0741 100644 --- a/celery/datastructures.py +++ b/celery/datastructures.py @@ -11,7 +11,9 @@ import sys import time -from collections import defaultdict, Mapping, MutableMapping, MutableSet +from collections import ( + Callable, Mapping, MutableMapping, MutableSet, defaultdict, +) from heapq import heapify, heappush, heappop from itertools import chain @@ -19,7 +21,7 @@ from kombu.utils.encoding import safe_str, bytes_to_str from kombu.utils.limits import TokenBucket # noqa -from celery.five import items +from celery.five import items, values from celery.utils.functional import LRUCache, first, uniq # noqa from celery.utils.text import match_case @@ -30,6 +32,10 @@ class LazyObject(object): # noqa pass LazySettings = LazyObject # noqa +__all__ = ['GraphFormatter', 'CycleError', 'DependencyGraph', + 'AttributeDictMixin', 'AttributeDict', 'DictAttribute', + 'ConfigurationView', 'LimitedSet'] + DOT_HEAD = """ {IN}{type} {id} {{ {INp}graph [{attrs}] @@ -41,9 +47,11 @@ class LazyObject(object): # noqa DOT_DIRS = {'graph': '--', 'digraph': '->'} DOT_TAIL = '{IN}}}' -__all__ = ['GraphFormatter', 'CycleError', 'DependencyGraph', - 'AttributeDictMixin', 'AttributeDict', 'DictAttribute', - 'ConfigurationView', 'LimitedSet'] +REPR_LIMITED_SET = """\ +<{name}({size}): maxlen={0.maxlen}, expires={0.expires}, minlen={0.minlen}>\ +""" + +sentinel = object() def force_mapping(m): @@ -578,7 +586,6 @@ def items(self): def values(self): return list(self._iterate_values()) - MutableMapping.register(ConfigurationView) @@ -588,10 +595,34 @@ class LimitedSet(object): Good for when you need to test for membership (`a in set`), but the set should not grow unbounded. - This version is now changed to be more enforcing those limits. - Maxlen is enforced all the time. But you can also configure - minlen now, which is minimal residual size of set. + Maxlen is enforced at all times, so if the limit is reached + we will also remove non-expired items. + + You can also configure minlen, which is the minimal residual size + of the set. + + All arguments are optional, with exception of minlen, which must + be smaller than maxlen. Unconfigured limits will not be enforced. + + :keyword maxlen: Optional max number of items. + + Adding more items than maxlen will result in immediate + removal of items sorted by oldest insertion time. + + :keyword expires: TTL for all items. + + Items aging over expiration are purged as keys are inserted. + :keyword minlen: Minimal residual size of this set. + .. versionadded:: 4.0 + + Older expired items will be deleted, only after the set + exceeds minlen number of items. + + :keyword data: Initial data to initialize set with. + Can be an iterable of ``(key, value)`` pairs, + a dict (``{key: insertion_time}``), or another instance + of :class:`LimitedSet`. Example:: @@ -611,39 +642,18 @@ class LimitedSet(object): 4000 >>>> 57000 in s # even this item is gone now False - """ - - REMOVED = object() # just a placeholder for removed items - _MAX_HEAP_PERCENTS_OVERLOAD = 15 # - def __init__(self, maxlen=0, expires=0, minlen=0, data=None): - """Initialize LimitedSet. - - All arguments are optional, with exception of minlen, which must - be smaller than maxlen. Unconfigured limits will not be enforced. + """ - :keyword maxlen: max size of this set. Adding more items than maxlen - results in immediate removing of older items. - :keyword expires: TTL for an item. - Items aging over expiration are purged. - :keyword minlen: minimal residual size of this set. - Oldest expired items will be delete - only until minlen size is reached. - :keyword data: data to initialize set with. Can be iterable of keys, - dict {key:inserted_time} or another LimitedSet. + max_heap_percent_overload = 15 - """ - if maxlen is None: - maxlen = 0 - if minlen is None: - minlen = 0 - if expires is None: - expires = 0 - self.maxlen = maxlen - self.minlen = minlen - self.expires = expires + def __init__(self, maxlen=0, expires=0, data=None, minlen=0): + self.maxlen = 0 if maxlen is None else maxlen + self.minlen = 0 if minlen is None else minlen + self.expires = 0 if expires is None else expires self._data = {} self._heap = [] + # make shortcuts self.__len__ = self._data.__len__ self.__contains__ = self._data.__contains__ @@ -653,14 +663,17 @@ def __init__(self, maxlen=0, expires=0, minlen=0, data=None): self.update(data) if not self.maxlen >= self.minlen >= 0: - raise ValueError('Minlen should be positive number, ' - 'smaller or equal to maxlen.') + raise ValueError( + 'minlen must be a positive number, less or equal to maxlen.') if self.expires < 0: - raise ValueError('Expires should not be negative!') + raise ValueError('expires cannot be negative!') def _refresh_heap(self): """Time consuming recreating of heap. Do not run this too often.""" - self._heap[:] = [entry for entry in self._data.values()] + self._heap[:] = [ + entry for entry in values(self._data) + if entry is not sentinel + ] heapify(self._heap) def clear(self): @@ -669,12 +682,11 @@ def clear(self): self._heap[:] = [] def add(self, item, now=None): - 'Add a new item or update the time of an existing item' - if not now: - now = time.time() + """Add a new item, or reset the expiry time of an existing item.""" + now = now or time.time() if item in self._data: self.discard(item) - entry = [now, item] + entry = (now, item) self._data[item] = entry heappush(self._heap, entry) if self.maxlen and len(self._data) >= self.maxlen: @@ -687,43 +699,41 @@ def update(self, other): self._refresh_heap() self.purge() elif isinstance(other, dict): - # revokes are sent like dict! - for key, inserted in other.items(): + # revokes are sent as a dict + for key, inserted in items(other): if isinstance(inserted, list): # in case someone uses ._data directly for sending update inserted = inserted[0] if not isinstance(inserted, float): - raise ValueError('Expecting float timestamp, got type ' - '"{0}" with value: {1}'.format( - type(inserted), inserted)) + raise ValueError( + 'Expecting float timestamp, got type ' + '{0!r} with value: {1}'.format( + type(inserted), inserted)) self.add(key, inserted) else: - # AVOID THIS, it could keep old data if more parties + # XXX AVOID THIS, it could keep old data if more parties # exchange them all over and over again for obj in other: self.add(obj) def discard(self, item): - 'Mark an existing item as REMOVED. If KeyError is not found, pass.' - entry = self._data.pop(item, self.REMOVED) - if entry is self.REMOVED: - return - entry[-1] = self.REMOVED - if self._heap_overload > self._MAX_HEAP_PERCENTS_OVERLOAD: - self._refresh_heap() - + # mark an existing item as removed. If KeyError is not found, pass. + entry = self._data.pop(item, sentinel) + if entry is not sentinel: + entry[-1] = sentinel + if self._heap_overload > self.max_heap_percent_overload: + self._refresh_heap() pop_value = discard def purge(self, now=None): """Check oldest items and remove them if needed. :keyword now: Time of purging -- by default right now. - This can be usefull for unittesting. + This can be useful for unit testing. + """ - if not now: - now = time.time() - if hasattr(now, '__call__'): - now = now() # if we got this now as function, evaluate it + now = now or time.time() + now = now() if isinstance(now, Callable) else now if self.maxlen: while len(self._data) > self.maxlen: self.pop() @@ -732,32 +742,33 @@ def purge(self, now=None): while len(self._data) > self.minlen >= 0: inserted_time, _ = self._heap[0] if inserted_time + self.expires > now: - break # end this right now, oldest item is not expired yet + break # oldest item has not expired yet self.pop() - def pop(self): - 'Remove and return the lowest time item. Return None if empty.' + def pop(self, default=None): + """Remove and return the oldest item, or :const:`None` when empty.""" while self._heap: _, item = heappop(self._heap) - if item is not self.REMOVED: - del self._data[item] + if self._data.pop(item, None) is not sentinel: return item - return None + return default def as_dict(self): """Whole set as serializable dictionary. + Example:: - >>> s=LimitedSet(maxlen=200) - >>> r=LimitedSet(maxlen=200) + >>> s = LimitedSet(maxlen=200) + >>> r = LimitedSet(maxlen=200) >>> for i in range(500): ... s.add(i) ... >>> r.update(s.as_dict()) >>> r == s True + """ - return {key: inserted for inserted, key in self._data.values()} + return {key: inserted for inserted, key in values(self._data)} def __eq__(self, other): return self._data == other._data @@ -766,15 +777,12 @@ def __ne__(self, other): return not self.__eq__(other) def __repr__(self): - return 'LimitedSet(maxlen={0}, expires={1}, minlen={2})' \ - ' Current size:{3}'.format( - self.maxlen, self.expires, self.minlen, len(self._data)) + return REPR_LIMITED_SET.format( + self, name=type(self).__name__, size=len(self), + ) def __iter__(self): - # return (item[1] for item in - # self._heap if item[-1] is not self.REMOVED) - # ^ not ordered, slow - return (i for _, i in sorted(self._data.values())) + return (i for _, i in sorted(values(self._data))) def __len__(self): return len(self._data) @@ -783,17 +791,13 @@ def __contains__(self, key): return key in self._data def __reduce__(self): - """Pickle helper class. - - This object can be pickled and upickled.""" return self.__class__, ( - self.maxlen, self.expires, self.minlen, self.as_dict()) + self.maxlen, self.expires, self.as_dict(), self.minlen) @property def _heap_overload(self): """Compute how much is heap bigger than data [percents].""" - if len(self._data) == 0: + if not self._data: return len(self._heap) - return len(self._heap)*100/len(self._data) - 100 - + return len(self._heap) * 100 / len(self._data) - 100 MutableSet.register(LimitedSet) diff --git a/celery/tests/utils/test_datastructures.py b/celery/tests/utils/test_datastructures.py index 0c294ff66..a536acf72 100644 --- a/celery/tests/utils/test_datastructures.py +++ b/celery/tests/utils/test_datastructures.py @@ -3,6 +3,8 @@ import pickle import sys +from collections import Mapping + from billiard.einfo import ExceptionInfo from time import time @@ -188,14 +190,17 @@ def test_add(self): for n in 'bar', 'baz': self.assertIn(n, s) self.assertNotIn('foo', s) + s = LimitedSet(maxlen=10) for i in range(150): s.add(i) self.assertLessEqual(len(s), 10) + # make sure heap is not leaking: - self.assertLessEqual(len(s._heap), - len(s) * (100. + - s._MAX_HEAP_PERCENTS_OVERLOAD) / 100) + self.assertLessEqual( + len(s._heap), + len(s) * (100. + s.max_heap_percent_overload) / 100, + ) def test_purge(self): # purge now enforces rules @@ -226,20 +231,10 @@ def test_purge(self): s.minlen = 3 s.purge(now=time() + 3) self.assertEqual(s.minlen, len(s)) - self.assertLessEqual(len(s._heap), - s.maxlen * - (100. + s._MAX_HEAP_PERCENTS_OVERLOAD)/100) - # s = LimitedSet(maxlen=None) - # [s.add(i) for i in range(10)] - # s.maxlen = 2 - # with patch('celery.datastructures.heappop') as hp: - # hp.side_effect = IndexError() - # s.purge() - # hp.assert_called_with(s._heap) - # with patch('celery.datastructures.heappop') as hp: - # s._data = {i * 2: i * 2 for i in range(10)} - # s.purge() - # self.assertEqual(hp.call_count, 10) + self.assertLessEqual( + len(s._heap), + s.maxlen * (100. + s._MAX_HEAP_PERCENTS_OVERLOAD) / 100, + ) def test_pickleable(self): s = LimitedSet(maxlen=2) @@ -273,7 +268,6 @@ def test_discard(self): s.discard('foo') self.assertNotIn('foo', s) self.assertEqual(len(s._data), 0) - # self.assertLessEqual(len(s._heap), 0 + s.heap_overload) s.discard('foo') def test_clear(self): @@ -311,7 +305,7 @@ def test_update(self): s4.update(s1.as_dict()) s4.update(s2.as_dict()) s5.update(s1._data) # revoke is using this - s5.update(s2._data) # + s5.update(s2._data) self.assertEqual(s3, s4) self.assertEqual(s3, s5) s2.update(s4) @@ -342,7 +336,7 @@ def test_pop_and_ordering_again(self): def test_as_dict(self): s = LimitedSet(maxlen=2) s.add('foo') - self.assertIsInstance(s.as_dict(), dict) + self.assertIsInstance(s.as_dict(), Mapping) class test_AttributeDict(Case): From 132a088e3a69d3e2131e32e3ba1179cdb8426bef Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Mar 2016 18:55:07 -0800 Subject: [PATCH 1086/1103] More fixes for #3102 --- celery/datastructures.py | 18 ++++++++++++------ celery/tests/utils/test_datastructures.py | 13 ++++++------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/celery/datastructures.py b/celery/datastructures.py index e897f0741..5b359d59c 100644 --- a/celery/datastructures.py +++ b/celery/datastructures.py @@ -601,8 +601,7 @@ class LimitedSet(object): You can also configure minlen, which is the minimal residual size of the set. - All arguments are optional, with exception of minlen, which must - be smaller than maxlen. Unconfigured limits will not be enforced. + All arguments are optional, and no limits are enabled by default. :keyword maxlen: Optional max number of items. @@ -611,11 +610,13 @@ class LimitedSet(object): :keyword expires: TTL for all items. - Items aging over expiration are purged as keys are inserted. + Expired items are purged as keys are inserted. :keyword minlen: Minimal residual size of this set. .. versionadded:: 4.0 + Value must be less than ``maxlen`` if both are configured. + Older expired items will be deleted, only after the set exceeds minlen number of items. @@ -693,7 +694,9 @@ def add(self, item, now=None): self.purge() def update(self, other): - """Update this LimitedSet from other LimitedSet, dict or iterable.""" + """Update this set from other LimitedSet, dict or iterable.""" + if not other: + return if isinstance(other, LimitedSet): self._data.update(other._data) self._refresh_heap() @@ -701,7 +704,7 @@ def update(self, other): elif isinstance(other, dict): # revokes are sent as a dict for key, inserted in items(other): - if isinstance(inserted, list): + if isinstance(inserted, (tuple, list)): # in case someone uses ._data directly for sending update inserted = inserted[0] if not isinstance(inserted, float): @@ -720,7 +723,6 @@ def discard(self, item): # mark an existing item as removed. If KeyError is not found, pass. entry = self._data.pop(item, sentinel) if entry is not sentinel: - entry[-1] = sentinel if self._heap_overload > self.max_heap_percent_overload: self._refresh_heap() pop_value = discard @@ -794,6 +796,10 @@ def __reduce__(self): return self.__class__, ( self.maxlen, self.expires, self.as_dict(), self.minlen) + def __bool__(self): + return bool(self._data) + __nonzero__ = __bool__ # Py2 + @property def _heap_overload(self): """Compute how much is heap bigger than data [percents].""" diff --git a/celery/tests/utils/test_datastructures.py b/celery/tests/utils/test_datastructures.py index a536acf72..fb07bc4aa 100644 --- a/celery/tests/utils/test_datastructures.py +++ b/celery/tests/utils/test_datastructures.py @@ -233,7 +233,7 @@ def test_purge(self): self.assertEqual(s.minlen, len(s)) self.assertLessEqual( len(s._heap), - s.maxlen * (100. + s._MAX_HEAP_PERCENTS_OVERLOAD) / 100, + s.maxlen * (100. + s.max_heap_percent_overload) / 100, ) def test_pickleable(self): @@ -314,13 +314,12 @@ def test_update(self): def test_iterable_and_ordering(self): s = LimitedSet(maxlen=35, expires=None) - for i in reversed(range(15)): + for i in range(15): s.add(i) - j = 40 - for i in s: - self.assertLess(i, j) # each item is smaller and smaller - j = i - self.assertEqual(i, 0) # last item = 0 + # NOTE: This test used to reverse the input numbers, but + # timestamps do not have enough precision to keep the data + # ordered when inserted quickly. + self.assertEqual(list(s), list(range(15))) def test_pop_and_ordering_again(self): s = LimitedSet(maxlen=5) From ddcb59b059990e37a8c4877c7957c526c65dee72 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Mar 2016 19:00:10 -0800 Subject: [PATCH 1087/1103] Found a way to actually test for ordering (Issue #3102) --- celery/tests/utils/test_datastructures.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/celery/tests/utils/test_datastructures.py b/celery/tests/utils/test_datastructures.py index fb07bc4aa..24879c1aa 100644 --- a/celery/tests/utils/test_datastructures.py +++ b/celery/tests/utils/test_datastructures.py @@ -4,6 +4,7 @@ import sys from collections import Mapping +from itertools import count from billiard.einfo import ExceptionInfo from time import time @@ -314,12 +315,16 @@ def test_update(self): def test_iterable_and_ordering(self): s = LimitedSet(maxlen=35, expires=None) - for i in range(15): - s.add(i) - # NOTE: This test used to reverse the input numbers, but - # timestamps do not have enough precision to keep the data - # ordered when inserted quickly. - self.assertEqual(list(s), list(range(15))) + # we use a custom clock here, as time.time() does not have enough + # precision when called quickly (can return the same value twice). + clock = count(1) + for i in reversed(range(15)): + s.add(i, now=next(clock)) + j = 40 + for i in s: + self.assertLess(i, j) # each item is smaller and smaller + j = i + self.assertEqual(i, 0) # last item is zero def test_pop_and_ordering_again(self): s = LimitedSet(maxlen=5) From fd7e48bfd1c1c2b7a0ea9c4b8b6610b329d0801d Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Mar 2016 19:03:14 -0800 Subject: [PATCH 1088/1103] [docs][calling] clarification in help box --- docs/userguide/calling.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index bd0e8e0c3..f7ce4352e 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -39,7 +39,8 @@ The API defines a standard set of execution options, as well as three methods: .. topic:: Quick Cheat Sheet - ``T.delay(arg, kwarg=value)`` - always a shortcut to ``.apply_async``. + Star arguments shortcut to ``.apply_async``. + (``.delay(*args, **kwargs)`` calls ``.apply_async(args, kwargs)``). - ``T.apply_async((arg,), {'kwarg': value})`` From b24fadcec6b530342ab533e47b05ed5dfb2d642c Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Mar 2016 19:04:59 -0800 Subject: [PATCH 1089/1103] [examples][app] Fixes restructuredtext error --- examples/app/myapp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/app/myapp.py b/examples/app/myapp.py index b72e9baab..d2939b567 100644 --- a/examples/app/myapp.py +++ b/examples/app/myapp.py @@ -1,6 +1,6 @@ """myapp.py -Usage: +Usage:: (window1)$ python myapp.py worker -l info From 1497a5487171ce38a23c40240a19e5e377179ad9 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 11 Mar 2016 19:22:57 -0800 Subject: [PATCH 1090/1103] [tests] Use moar .utils.objects.Bunch --- celery/tests/app/test_app.py | 14 ++++---------- celery/tests/app/test_beat.py | 11 +++++------ celery/tests/backends/test_cassandra.py | 10 ++++------ celery/tests/bin/test_base.py | 10 +++------- celery/tests/concurrency/test_prefork.py | 15 ++++++--------- celery/tests/fixups/test_django.py | 6 ++---- celery/tests/utils/test_datastructures.py | 19 +++++++------------ celery/tests/worker/test_autoscale.py | 8 ++------ 8 files changed, 33 insertions(+), 60 deletions(-) diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index 70fe7351c..2455eb6d6 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -16,7 +16,7 @@ from celery.app import base as _appbase from celery.app import defaults from celery.exceptions import ImproperlyConfigured -from celery.five import items, keys +from celery.five import keys from celery.loaders.base import BaseLoader, unconfigured from celery.platforms import pyimplementation from celery.utils.serialization import pickle @@ -38,6 +38,7 @@ ) from celery.utils import uuid from celery.utils.mail import ErrorMail +from celery.utils.objects import Bunch THIS_IS_A_KEY = 'this is a value' @@ -58,13 +59,6 @@ class ObjectConfig2(object): UNDERSTAND_ME = True -class Object(object): - - def __init__(self, **kwargs): - for key, value in items(kwargs): - setattr(self, key, value) - - def _get_test_config(): return deepcopy(CELERY_TEST_CONFIG) test_config = _get_test_config() @@ -647,10 +641,10 @@ def test_setting__broker_transport_options(self): _args = {'foo': 'bar', 'spam': 'baz'} - self.app.config_from_object(Object()) + self.app.config_from_object(Bunch()) self.assertEqual(self.app.conf.broker_transport_options, {}) - self.app.config_from_object(Object(broker_transport_options=_args)) + self.app.config_from_object(Bunch(broker_transport_options=_args)) self.assertEqual(self.app.conf.broker_transport_options, _args) def test_Windows_log_color_disabled(self): diff --git a/celery/tests/app/test_beat.py b/celery/tests/app/test_beat.py index 6ce5a8d2e..05edae42f 100644 --- a/celery/tests/app/test_beat.py +++ b/celery/tests/app/test_beat.py @@ -9,11 +9,9 @@ from celery.five import keys, string_t from celery.schedules import schedule from celery.utils import uuid -from celery.tests.case import AppCase, Mock, SkipTest, call, patch - +from celery.utils.objects import Bunch -class Object(object): - pass +from celery.tests.case import AppCase, Mock, SkipTest, call, patch class MockShelve(dict): @@ -353,8 +351,9 @@ def create_persistent_scheduler(shelv=None): class MockPersistentScheduler(beat.PersistentScheduler): sh = shelv - persistence = Object() - persistence.open = lambda *a, **kw: shelv + persistence = Bunch( + open=lambda *a, **kw: shelv, + ) tick_raises_exit = False shutdown_service = None diff --git a/celery/tests/backends/test_cassandra.py b/celery/tests/backends/test_cassandra.py index d97e584f4..848ac97fa 100644 --- a/celery/tests/backends/test_cassandra.py +++ b/celery/tests/backends/test_cassandra.py @@ -5,6 +5,7 @@ from celery import states from celery.exceptions import ImproperlyConfigured +from celery.utils.objects import Bunch from celery.tests.case import ( AppCase, Mock, mock_module, depends_on_current_app ) @@ -12,10 +13,6 @@ CASSANDRA_MODULES = ['cassandra', 'cassandra.auth', 'cassandra.cluster'] -class Object(object): - pass - - class test_CassandraBackend(AppCase): def setup(self): @@ -42,8 +39,9 @@ def test_init_with_and_without_LOCAL_QUROM(self): from celery.backends import cassandra as mod mod.cassandra = Mock() - cons = mod.cassandra.ConsistencyLevel = Object() - cons.LOCAL_QUORUM = 'foo' + cons = mod.cassandra.ConsistencyLevel = Bunch( + LOCAL_QUORUM='foo', + ) self.app.conf.cassandra_read_consistency = 'LOCAL_FOO' self.app.conf.cassandra_write_consistency = 'LOCAL_FOO' diff --git a/celery/tests/bin/test_base.py b/celery/tests/bin/test_base.py index f8a8b5e58..3c02ca8ef 100644 --- a/celery/tests/bin/test_base.py +++ b/celery/tests/bin/test_base.py @@ -8,15 +8,13 @@ Extensions, HelpFormatter, ) +from celery.utils.objects import Bunch + from celery.tests.case import ( AppCase, Mock, depends_on_current_app, override_stdouts, patch, ) -class Object(object): - pass - - class MyApp(object): user_options = {'preload': None} @@ -27,9 +25,7 @@ class MockCommand(Command): mock_args = ('arg1', 'arg2', 'arg3') def parse_options(self, prog_name, arguments, command=None): - options = Object() - options.foo = 'bar' - options.prog_name = prog_name + options = Bunch(foo='bar', prog_name=prog_name) return options, self.mock_args def run(self, *args, **kwargs): diff --git a/celery/tests/concurrency/test_prefork.py b/celery/tests/concurrency/test_prefork.py index b317d6821..c829cd596 100644 --- a/celery/tests/concurrency/test_prefork.py +++ b/celery/tests/concurrency/test_prefork.py @@ -9,9 +9,12 @@ from celery.app.defaults import DEFAULTS from celery.datastructures import AttributeDict -from celery.five import items, range +from celery.five import range from celery.utils.functional import noop +from celery.utils.objects import Bunch + from celery.tests.case import AppCase, Mock, SkipTest, patch, restore_logging + try: from celery.concurrency import prefork as mp from celery.concurrency import asynpool @@ -38,12 +41,6 @@ def apply_async(self, *args, **kwargs): asynpool = None # noqa -class Object(object): # for writeable attributes. - - def __init__(self, **kwargs): - [setattr(self, k, v) for k, v in items(kwargs)] - - class MockResult(object): def __init__(self, value, pid): @@ -132,7 +129,7 @@ def __init__(self, *args, **kwargs): self.maintain_pool = Mock() self._state = mp.RUN self._processes = kwargs.get('processes') - self._pool = [Object(pid=i, inqW_fd=1, outqR_fd=2) + self._pool = [Bunch(pid=i, inqW_fd=1, outqR_fd=2) for i in range(self._processes)] self._current_proc = cycle(range(self._processes)) @@ -405,7 +402,7 @@ def test_grow_shrink(self): def test_info(self): pool = TaskPool(10) - procs = [Object(pid=i) for i in range(pool.limit)] + procs = [Bunch(pid=i) for i in range(pool.limit)] class _Pool(object): _pool = procs diff --git a/celery/tests/fixups/test_django.py b/celery/tests/fixups/test_django.py index 45ae675df..f99d73f0c 100644 --- a/celery/tests/fixups/test_django.py +++ b/celery/tests/fixups/test_django.py @@ -10,6 +10,7 @@ DjangoFixup, DjangoWorkerFixup, ) +from celery.utils.objects import Bunch from celery.tests.case import ( AppCase, Mock, patch, patch_modules, mask_modules, @@ -275,10 +276,7 @@ def test__close_database(self): with self.assertRaises(KeyError): f._close_database() - class Object(object): - pass - o = Object() - o.close_connection = Mock() + o = Bunch(close_connection=Mock()) f._db = o f._close_database() o.close_connection.assert_called_with() diff --git a/celery/tests/utils/test_datastructures.py b/celery/tests/utils/test_datastructures.py index 24879c1aa..f8ff56cda 100644 --- a/celery/tests/utils/test_datastructures.py +++ b/celery/tests/utils/test_datastructures.py @@ -17,18 +17,15 @@ DependencyGraph, ) from celery.five import items +from celery.utils.objects import Bunch -from celery.tests.case import Case, Mock, WhateverIO, SkipTest, patch - - -class Object(object): - pass +from celery.tests.case import Case, Mock, WhateverIO, SkipTest class test_DictAttribute(Case): def test_get_set_keys_values_items(self): - x = DictAttribute(Object()) + x = DictAttribute(Bunch()) x['foo'] = 'The quick brown fox' self.assertEqual(x['foo'], 'The quick brown fox') self.assertEqual(x['foo'], x.obj.foo) @@ -46,21 +43,20 @@ def test_get_set_keys_values_items(self): self.assertIn('The quick yellow fox', list(x.values())) def test_setdefault(self): - x = DictAttribute(Object()) + x = DictAttribute(Bunch()) x.setdefault('foo', 'NEW') self.assertEqual(x['foo'], 'NEW') x.setdefault('foo', 'XYZ') self.assertEqual(x['foo'], 'NEW') def test_contains(self): - x = DictAttribute(Object()) + x = DictAttribute(Bunch()) x['foo'] = 1 self.assertIn('foo', x) self.assertNotIn('bar', x) def test_items(self): - obj = Object() - obj.attr1 = 1 + obj = Bunch(attr1=1) x = DictAttribute(obj) x['attr2'] = 2 self.assertEqual(x['attr1'], 1) @@ -123,8 +119,7 @@ def test_add_defaults_dict(self): self.assertEqual(self.view.foo, 10) def test_add_defaults_object(self): - defaults = Object() - defaults.foo = 10 + defaults = Bunch(foo=10) self.view.add_defaults(defaults) self.assertEqual(self.view.foo, 10) diff --git a/celery/tests/worker/test_autoscale.py b/celery/tests/worker/test_autoscale.py index 774d89b61..b0c15f9e8 100644 --- a/celery/tests/worker/test_autoscale.py +++ b/celery/tests/worker/test_autoscale.py @@ -7,10 +7,7 @@ from celery.worker import state from celery.worker import autoscale from celery.tests.case import AppCase, Mock, patch, sleepdeprived - - -class Object(object): - pass +from celery.utils.objects import Bunch class MockPool(BasePool): @@ -19,8 +16,7 @@ class MockPool(BasePool): def __init__(self, *args, **kwargs): super(MockPool, self).__init__(*args, **kwargs) - self._pool = Object() - self._pool._processes = self.limit + self._pool = Bunch(_processes=self.limit) def grow(self, n=1): self._pool._processes += n From 8de8bb99b29119135fdc932cf5c5a213d7367bf0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Sat, 12 Mar 2016 16:02:09 -0800 Subject: [PATCH 1091/1103] Cosmetics --- celery/app/base.py | 30 +++++++++++++++++------------- celery/app/control.py | 13 ++++++------- celery/tests/app/test_app.py | 8 ++++---- celery/utils/encoding.py | 2 +- celery/utils/functional.py | 3 ++- celery/utils/imports.py | 2 +- celery/utils/iso8601.py | 5 ++--- celery/utils/log.py | 2 +- celery/utils/mail.py | 2 +- celery/utils/objects.py | 2 +- celery/utils/serialization.py | 2 +- celery/utils/sysinfo.py | 2 +- celery/utils/text.py | 2 +- celery/utils/threads.py | 2 +- celery/utils/timer2.py | 2 +- celery/utils/timeutils.py | 2 +- 16 files changed, 42 insertions(+), 39 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index cd8c250ac..4127ea465 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -39,6 +39,7 @@ from celery.utils import gen_task_name from celery.utils.dispatch import Signal from celery.utils.functional import first, maybe_list, head_from_fun +from celery.utils.timeutils import timezone from celery.utils.imports import instantiate, symbol_by_name from celery.utils.log import get_logger from celery.utils.objects import FallbackContext, mro_lookup @@ -58,7 +59,7 @@ logger = get_logger(__name__) -_EXECV = os.environ.get('FORKED_BY_MULTIPROCESSING') +USING_EXECV = os.environ.get('FORKED_BY_MULTIPROCESSING') BUILTIN_FIXUPS = { 'celery.fixups.django:fixup', } @@ -89,6 +90,7 @@ def _after_fork_cleanup_app(app): class PendingConfiguration(UserDict, AttributeDictMixin): + callback = None data = None @@ -104,7 +106,8 @@ class Celery(object): """Celery application. :param main: Name of the main module if running as `__main__`. - This is used as a prefix for task names. + This is used as the prefix for autogenerated task names. + :keyword broker: URL of the default broker used. :keyword loader: The loader class, or the name of the loader class to use. Default is :class:`celery.loaders.app.AppLoader`. @@ -145,6 +148,8 @@ class Celery(object): #: See :ref:`extending-bootsteps`. steps = None + builtin_fixups = BUILTIN_FIXUPS + amqp_cls = 'celery.app.amqp:AMQP' backend_cls = None events_cls = 'celery.events:Events' @@ -153,10 +158,10 @@ class Celery(object): control_cls = 'celery.app.control:Control' task_cls = 'celery.app.task:Task' registry_cls = TaskRegistry + _fixups = None _pool = None _conf = None - builtin_fixups = BUILTIN_FIXUPS _after_fork_registered = False #: Signal sent when app is loading configuration. @@ -240,6 +245,10 @@ def __init__(self, main=None, loader=None, backend=None, self.on_init() _register_app(self) + def on_init(self): + """Optional callback called at init.""" + pass + def __autoset(self, key, value): if value: self._preconf[key] = value @@ -278,10 +287,6 @@ def close(self): self._pool = None _deregister_app(self) - def on_init(self): - """Optional callback called at init.""" - pass - def start(self, argv=None): """Run :program:`celery` using `argv`. @@ -289,8 +294,8 @@ def start(self, argv=None): """ return instantiate( - 'celery.bin.celery:CeleryCommand', - app=self).execute_from_commandline(argv) + 'celery.bin.celery:CeleryCommand', app=self + ).execute_from_commandline(argv) def worker_main(self, argv=None): """Run :program:`celery worker` using `argv`. @@ -299,8 +304,8 @@ def worker_main(self, argv=None): """ return instantiate( - 'celery.bin.worker:worker', - app=self).execute_from_commandline(argv) + 'celery.bin.worker:worker', app=self + ).execute_from_commandline(argv) def task(self, *args, **opts): """Decorator to create a task class out of any callable. @@ -332,7 +337,7 @@ def refresh_feed(url): application is fully set up (finalized). """ - if _EXECV and opts.get('lazy', True): + if USING_EXECV and opts.get('lazy', True): # When using execv the task in the original module will point to a # different app, so doing things like 'add.request' will point to # a different task instance. This makes sure it will always use @@ -1127,7 +1132,6 @@ def timezone(self): :setting:`timezone` setting. """ - from celery.utils.timeutils import timezone conf = self.conf tz = conf.timezone if not tz: diff --git a/celery/app/control.py b/celery/app/control.py index 4b68f4b99..0c4446906 100644 --- a/celery/app/control.py +++ b/celery/app/control.py @@ -55,13 +55,12 @@ def __init__(self, destination=None, timeout=1, callback=None, self.limit = limit def _prepare(self, reply): - if not reply: - return - by_node = flatten_reply(reply) - if self.destination and \ - not isinstance(self.destination, (list, tuple)): - return by_node.get(self.destination) - return by_node + if reply: + by_node = flatten_reply(reply) + if (self.destination and + not isinstance(self.destination, (list, tuple))): + return by_node.get(self.destination) + return by_node def _request(self, command, **kwargs): return self._prepare(self.app.control.broadcast( diff --git a/celery/tests/app/test_app.py b/celery/tests/app/test_app.py index 2455eb6d6..546ef6a80 100644 --- a/celery/tests/app/test_app.py +++ b/celery/tests/app/test_app.py @@ -138,7 +138,7 @@ def test_with_config_source(self): @depends_on_current_app def test_task_windows_execv(self): - prev, _appbase._EXECV = _appbase._EXECV, True + prev, _appbase.USING_EXECV = _appbase.USING_EXECV, True try: @self.app.task(shared=False) def foo(): @@ -147,8 +147,8 @@ def foo(): self.assertTrue(foo._get_current_object()) # is proxy finally: - _appbase._EXECV = prev - assert not _appbase._EXECV + _appbase.USING_EXECV = prev + assert not _appbase.USING_EXECV def test_task_takes_no_args(self): with self.assertRaises(TypeError): @@ -405,7 +405,7 @@ def filter(task): check(task) return task - assert not _appbase._EXECV + assert not _appbase.USING_EXECV @app.task(filter=filter, shared=False) def foo(): diff --git a/celery/utils/encoding.py b/celery/utils/encoding.py index 3ddcd35eb..03da6d9eb 100644 --- a/celery/utils/encoding.py +++ b/celery/utils/encoding.py @@ -6,7 +6,7 @@ This module has moved to :mod:`kombu.utils.encoding`. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals from kombu.utils.encoding import ( # noqa default_encode, default_encoding, bytes_t, bytes_to_str, str_t, diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 2715743f2..4f7e6b147 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -6,7 +6,7 @@ Utilities for functions. """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import sys import threading @@ -321,6 +321,7 @@ def regen(it): class _regen(UserList, list): # must be subclass of list so that json can encode. + def __init__(self, it): self.__it = it self.__index = 0 diff --git a/celery/utils/imports.py b/celery/utils/imports.py index 22a2fdcd3..e82db0c6e 100644 --- a/celery/utils/imports.py +++ b/celery/utils/imports.py @@ -6,7 +6,7 @@ Utilities related to importing modules and symbols by name. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import imp as _imp import importlib diff --git a/celery/utils/iso8601.py b/celery/utils/iso8601.py index 9f9ba9a3a..98a336170 100644 --- a/celery/utils/iso8601.py +++ b/celery/utils/iso8601.py @@ -1,5 +1,4 @@ -""" -Originally taken from pyiso8601 (http://code.google.com/p/pyiso8601/) +"""Originally taken from pyiso8601 (http://code.google.com/p/pyiso8601/) Modified to match the behavior of dateutil.parser: @@ -31,7 +30,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import re diff --git a/celery/utils/log.py b/celery/utils/log.py index 5907ca7c3..743a9a663 100644 --- a/celery/utils/log.py +++ b/celery/utils/log.py @@ -6,7 +6,7 @@ Logging utilities. """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import logging import numbers diff --git a/celery/utils/mail.py b/celery/utils/mail.py index 585a7abcb..0f0ec2082 100644 --- a/celery/utils/mail.py +++ b/celery/utils/mail.py @@ -6,7 +6,7 @@ How task error emails are formatted and sent. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import smtplib import socket diff --git a/celery/utils/objects.py b/celery/utils/objects.py index f6bd0ba28..1fac84ddc 100644 --- a/celery/utils/objects.py +++ b/celery/utils/objects.py @@ -6,7 +6,7 @@ Object related utilities including introspection, etc. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals __all__ = ['mro_lookup'] diff --git a/celery/utils/serialization.py b/celery/utils/serialization.py index 91a79fc88..6a2c28c8e 100644 --- a/celery/utils/serialization.py +++ b/celery/utils/serialization.py @@ -6,7 +6,7 @@ Utilities for safely pickling exceptions. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals from base64 import b64encode as base64encode, b64decode as base64decode from inspect import getmro diff --git a/celery/utils/sysinfo.py b/celery/utils/sysinfo.py index 65073a6f9..19264baa5 100644 --- a/celery/utils/sysinfo.py +++ b/celery/utils/sysinfo.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import os diff --git a/celery/utils/text.py b/celery/utils/text.py index 2920ad782..851c9f86e 100644 --- a/celery/utils/text.py +++ b/celery/utils/text.py @@ -6,7 +6,7 @@ Text formatting utilities """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals from textwrap import fill diff --git a/celery/utils/threads.py b/celery/utils/threads.py index 5d4237329..1016496f9 100644 --- a/celery/utils/threads.py +++ b/celery/utils/threads.py @@ -6,7 +6,7 @@ Threading utilities. """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import os import socket diff --git a/celery/utils/timer2.py b/celery/utils/timer2.py index fdac90803..cfeb034e2 100644 --- a/celery/utils/timer2.py +++ b/celery/utils/timer2.py @@ -6,7 +6,7 @@ Scheduler for Python functions. """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import os import sys diff --git a/celery/utils/timeutils.py b/celery/utils/timeutils.py index e9a52dfac..76a01020d 100644 --- a/celery/utils/timeutils.py +++ b/celery/utils/timeutils.py @@ -6,7 +6,7 @@ This module contains various utilities related to dates and times. """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import numbers import os From 3be6bb6d25546952611ca073b9afac6ef5121192 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Sat, 12 Mar 2016 16:03:19 -0800 Subject: [PATCH 1092/1103] Removes unused module celery.utils.compat (import from celery.five instead) --- celery/utils/compat.py | 1 - docs/internals/reference/celery.utils.compat.rst | 11 ----------- docs/internals/reference/index.rst | 1 - 3 files changed, 13 deletions(-) delete mode 100644 celery/utils/compat.py delete mode 100644 docs/internals/reference/celery.utils.compat.rst diff --git a/celery/utils/compat.py b/celery/utils/compat.py deleted file mode 100644 index 6f6296489..000000000 --- a/celery/utils/compat.py +++ /dev/null @@ -1 +0,0 @@ -from celery.five import * # noqa diff --git a/docs/internals/reference/celery.utils.compat.rst b/docs/internals/reference/celery.utils.compat.rst deleted file mode 100644 index 851851f09..000000000 --- a/docs/internals/reference/celery.utils.compat.rst +++ /dev/null @@ -1,11 +0,0 @@ -============================================ - celery.utils.compat -============================================ - -.. contents:: - :local: -.. currentmodule:: celery.utils.compat - -.. automodule:: celery.utils.compat - :members: - :undoc-members: diff --git a/docs/internals/reference/index.rst b/docs/internals/reference/index.rst index d7329cd2e..864f7fd70 100644 --- a/docs/internals/reference/index.rst +++ b/docs/internals/reference/index.rst @@ -56,7 +56,6 @@ celery.utils.term celery.utils.timeutils celery.utils.iso8601 - celery.utils.compat celery.utils.saferepr celery.utils.serialization celery.utils.sysinfo From 51fca36f19c00dc98ac1fa34fb24b2c8dd9c3c16 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 14 Mar 2016 14:36:24 -0700 Subject: [PATCH 1093/1103] Cosmetics #2 --- celery/bootsteps.py | 8 ++--- celery/platforms.py | 2 +- celery/result.py | 12 +++---- celery/schedules.py | 76 +++++++++++++++++++++++----------------- celery/signals.py | 3 +- celery/states.py | 2 +- celery/utils/__init__.py | 4 +-- celery/utils/abstract.py | 2 +- 8 files changed, 60 insertions(+), 49 deletions(-) diff --git a/celery/bootsteps.py b/celery/bootsteps.py index edc7d563f..85a351cf3 100644 --- a/celery/bootsteps.py +++ b/celery/bootsteps.py @@ -22,9 +22,10 @@ try: from greenlet import GreenletExit - IGNORE_ERRORS = (GreenletExit,) except ImportError: # pragma: no cover IGNORE_ERRORS = () +else: + IGNORE_ERRORS = (GreenletExit,) __all__ = ['Blueprint', 'Step', 'StartStopStep', 'ConsumerStep'] @@ -34,7 +35,6 @@ TERMINATE = 0x3 logger = get_logger(__name__) -debug = logger.debug def _pre(ns, fmt): @@ -123,7 +123,7 @@ def start(self, parent): self._debug('Starting %s', step.alias) self.started = i + 1 step.start(parent) - debug('^-- substep ok') + logger.debug('^-- substep ok') def human_state(self): return self.state_to_name[self.state or 0] @@ -271,7 +271,7 @@ def load_step(self, step): return step.name, step def _debug(self, msg, *args): - return debug(_pre(self, msg), *args) + return logger.debug(_pre(self, msg), *args) @property def alias(self): diff --git a/celery/platforms.py b/celery/platforms.py index fd4410df3..b86173554 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -7,7 +7,7 @@ users, groups, and so on. """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import atexit import errno diff --git a/celery/result.py b/celery/result.py index 1efde9ddd..c6fe26ee2 100644 --- a/celery/result.py +++ b/celery/result.py @@ -6,7 +6,7 @@ Task results/state and groups of results. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import time @@ -873,11 +873,6 @@ def then(self, callback, on_error=None): def _get_task_meta(self): return self._cache - @property - def _cache(self): - return {'task_id': self.id, 'result': self._result, 'status': - self._state, 'traceback': self._traceback} - def __del__(self): pass @@ -912,6 +907,11 @@ def revoke(self, *args, **kwargs): def __repr__(self): return ''.format(self) + @property + def _cache(self): + return {'task_id': self.id, 'result': self._result, 'status': + self._state, 'traceback': self._traceback} + @property def result(self): """The tasks return value""" diff --git a/celery/schedules.py b/celery/schedules.py index 52c366128..657d6f787 100644 --- a/celery/schedules.py +++ b/celery/schedules.py @@ -7,11 +7,12 @@ should run. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import numbers import re +from bisect import bisect, bisect_left from collections import namedtuple from datetime import datetime, timedelta @@ -72,11 +73,11 @@ class schedule(object): """Schedule for periodic task. :param run_every: Interval in seconds (or a :class:`~datetime.timedelta`). - :param relative: If set to True the run time will be rounded to the + :keyword relative: If set to True the run time will be rounded to the resolution of the interval. - :param nowfun: Function returning the current date and time + :keyword nowfun: Function returning the current date and time (class:`~datetime.datetime`). - :param app: Celery app instance. + :keyword app: Celery app instance. """ relative = False @@ -431,14 +432,13 @@ def _expand_cronspec(cronspec, max_, min_=0): return result def _delta_to_next(self, last_run_at, next_hour, next_minute): - """ - Takes a datetime of last run, next minute and hour, and + """Takes a datetime of last run, next minute and hour, and returns a relativedelta for the next scheduled day and time. + Only called when day_of_month and/or month_of_year cronspec is specified to further limit scheduled task execution. - """ - from bisect import bisect, bisect_left + """ datedata = AttributeDict(year=last_run_at.year) days_of_month = sorted(self.day_of_month) months_of_year = sorted(self.month_of_year) @@ -515,16 +515,20 @@ def remaining_delta(self, last_run_at, tz=None, ffwd=ffwd): now = self.maybe_make_aware(self.now()) dow_num = last_run_at.isoweekday() % 7 # Sunday is day 0, not day 7 - execute_this_date = (last_run_at.month in self.month_of_year and - last_run_at.day in self.day_of_month and - dow_num in self.day_of_week) + execute_this_date = ( + last_run_at.month in self.month_of_year and + last_run_at.day in self.day_of_month and + dow_num in self.day_of_week + ) - execute_this_hour = (execute_this_date and - last_run_at.day == now.day and - last_run_at.month == now.month and - last_run_at.year == now.year and - last_run_at.hour in self.hour and - last_run_at.minute < max(self.minute)) + execute_this_hour = ( + execute_this_date and + last_run_at.day == now.day and + last_run_at.month == now.month and + last_run_at.year == now.year and + last_run_at.hour in self.hour and + last_run_at.minute < max(self.minute) + ) if execute_this_hour: next_minute = min(minute for minute in self.minute @@ -549,12 +553,14 @@ def remaining_delta(self, last_run_at, tz=None, ffwd=ffwd): if day > dow_num] or self.day_of_week) add_week = next_day == dow_num - delta = ffwd(weeks=add_week and 1 or 0, - weekday=(next_day - 1) % 7, - hour=next_hour, - minute=next_minute, - second=0, - microsecond=0) + delta = ffwd( + weeks=add_week and 1 or 0, + weekday=(next_day - 1) % 7, + hour=next_hour, + minute=next_minute, + second=0, + microsecond=0, + ) else: delta = self._delta_to_next(last_run_at, next_hour, next_minute) @@ -581,11 +587,13 @@ def is_due(self, last_run_at): def __eq__(self, other): if isinstance(other, crontab): - return (other.month_of_year == self.month_of_year and - other.day_of_month == self.day_of_month and - other.day_of_week == self.day_of_week and - other.hour == self.hour and - other.minute == self.minute) + return ( + other.month_of_year == self.month_of_year and + other.day_of_month == self.day_of_month and + other.day_of_week == self.day_of_week and + other.hour == self.hour and + other.minute == self.minute + ) return NotImplemented def __ne__(self, other): @@ -715,8 +723,8 @@ def remaining_estimate(self, last_run_at): start=last_run_at_utc, use_center=self.use_center, ) except self.ephem.CircumpolarError: # pragma: no cover - """Sun will not rise/set today. Check again tomorrow - (specifically, after the next anti-transit).""" + # Sun will not rise/set today. Check again tomorrow + # (specifically, after the next anti-transit). next_utc = ( self.cal.next_antitransit(self.ephem.Sun()) + timedelta(minutes=1) @@ -743,9 +751,11 @@ def is_due(self, last_run_at): def __eq__(self, other): if isinstance(other, solar): - return (other.event == self.event and - other.lat == self.lat and - other.lon == self.lon) + return ( + other.event == self.event and + other.lat == self.lat and + other.lon == self.lon + ) return NotImplemented def __ne__(self, other): diff --git a/celery/signals.py b/celery/signals.py index c864a1b64..ba2c1a213 100644 --- a/celery/signals.py +++ b/celery/signals.py @@ -12,7 +12,8 @@ See :ref:`signals` for more information. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals + from .utils.dispatch import Signal __all__ = ['before_task_publish', 'after_task_publish', diff --git a/celery/states.py b/celery/states.py index 0525375b2..697bc1868 100644 --- a/celery/states.py +++ b/celery/states.py @@ -57,7 +57,7 @@ ----- """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals __all__ = ['PENDING', 'RECEIVED', 'STARTED', 'SUCCESS', 'FAILURE', 'REVOKED', 'RETRY', 'IGNORED', 'READY_STATES', 'UNREADY_STATES', diff --git a/celery/utils/__init__.py b/celery/utils/__init__.py index 697ee66a8..44d553887 100644 --- a/celery/utils/__init__.py +++ b/celery/utils/__init__.py @@ -6,7 +6,7 @@ Utility functions. """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import numbers import os @@ -56,7 +56,7 @@ #: We use it to find out the name of the original ``__main__`` #: module, so that we can properly rewrite the name of the #: task to be that of ``App.main``. -MP_MAIN_FILE = os.environ.get('MP_MAIN_FILE') or None +MP_MAIN_FILE = os.environ.get('MP_MAIN_FILE') #: Exchange for worker direct queues. WORKER_DIRECT_EXCHANGE = Exchange('C.dq2') diff --git a/celery/utils/abstract.py b/celery/utils/abstract.py index f2a7e1504..f8357393d 100644 --- a/celery/utils/abstract.py +++ b/celery/utils/abstract.py @@ -6,7 +6,7 @@ Abstract classes. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals from abc import ABCMeta, abstractmethod, abstractproperty from collections import Callable From 60e4bfcfe4241f5886464e5150b2b74e18fbdb71 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 14 Mar 2016 18:40:35 -0700 Subject: [PATCH 1094/1103] [stress] Fixes seconds display for subsecond results --- funtests/stress/stress/suite.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/funtests/stress/stress/suite.py b/funtests/stress/stress/suite.py index e6e1d4d65..fa237b790 100755 --- a/funtests/stress/stress/suite.py +++ b/funtests/stress/stress/suite.py @@ -60,10 +60,12 @@ class StopSuite(Exception): def pstatus(p): + runtime = monotonic() - p.runtime + elapsed = monotonic() - p.elapsed return F_PROGRESS.format( p, - runtime=humanize_seconds(monotonic() - p.runtime, now='0 seconds'), - elapsed=humanize_seconds(monotonic() - p.elapsed, now='0 seconds'), + runtime=humanize_seconds(runtime, now=runtime), + elapsed=humanize_seconds(elapsed, now=elapsed), ) From 5c8f03b2a1b41bc9ec242b854f31eb59e35b3807 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Mon, 14 Mar 2016 18:41:18 -0700 Subject: [PATCH 1095/1103] [worker] Consumer.on_unknown_task must construct fake request to store result --- celery/worker/consumer/consumer.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/celery/worker/consumer/consumer.py b/celery/worker/consumer/consumer.py index ea2d93e96..14b6d56f5 100644 --- a/celery/worker/consumer/consumer.py +++ b/celery/worker/consumer/consumer.py @@ -33,6 +33,7 @@ from celery.utils import gethostname from celery.utils.functional import noop from celery.utils.log import get_logger +from celery.utils.objects import Bunch from celery.utils.text import truncate from celery.utils.timeutils import humanize_seconds, rate @@ -451,10 +452,20 @@ def on_unknown_task(self, body, message, exc): error(UNKNOWN_TASK_ERROR, exc, dump_body(message, body), exc_info=True) try: id_, name = message.headers['id'], message.headers['task'] + root_id = message.headers['root_id'] except KeyError: # proto1 id_, name = body['id'], body['task'] + root_id = None + request = Bunch( + name=name, chord=None, root_id=root_id, + correlation_id=message.properties.get('correlation_id'), + reply_to=message.properties.get('reply_to'), + errbacks=None, + ) message.reject_log_error(logger, self.connection_errors) - self.app.backend.mark_as_failure(id_, NotRegistered(name)) + self.app.backend.mark_as_failure( + id_, NotRegistered(name), request=request, + ) if self.event_dispatcher: self.event_dispatcher.send( 'task-failed', uuid=id_, From bf491045176646af7e92d709cdb19445790cc433 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 15 Mar 2016 12:05:05 -0700 Subject: [PATCH 1096/1103] [docs] fixes rst markup error --- docs/userguide/signals.rst | 2 +- funtests/stress/stress/app.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/userguide/signals.rst b/docs/userguide/signals.rst index a22a4bcb8..40d9f7096 100644 --- a/docs/userguide/signals.rst +++ b/docs/userguide/signals.rst @@ -109,7 +109,7 @@ Provides arguments: * declare List of entities (:class:`~kombu.Exchange`, - :class:`~kombu.Queue` or :class:~`kombu.binding` to declare before + :class:`~kombu.Queue` or :class:`~kombu.binding` to declare before publishing the message. Can be modified. * retry_policy diff --git a/funtests/stress/stress/app.py b/funtests/stress/stress/app.py index d4541961c..7b5d592d2 100644 --- a/funtests/stress/stress/app.py +++ b/funtests/stress/stress/app.py @@ -73,6 +73,7 @@ def _marker(s, sep='-'): @app.task def add(x, y): + add.delay(x + x, y + y) return x + y From 90544edce1850fecb6744b42534dce2cc81f2bd0 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 16 Mar 2016 19:03:08 -0700 Subject: [PATCH 1097/1103] Fixes build --- celery/worker/consumer/consumer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/worker/consumer/consumer.py b/celery/worker/consumer/consumer.py index 14b6d56f5..c189718fb 100644 --- a/celery/worker/consumer/consumer.py +++ b/celery/worker/consumer/consumer.py @@ -452,7 +452,7 @@ def on_unknown_task(self, body, message, exc): error(UNKNOWN_TASK_ERROR, exc, dump_body(message, body), exc_info=True) try: id_, name = message.headers['id'], message.headers['task'] - root_id = message.headers['root_id'] + root_id = message.headers.get('root_id') except KeyError: # proto1 id_, name = body['id'], body['task'] root_id = None @@ -515,7 +515,7 @@ def on_task_received(message): try: strategy = strategies[type_] except KeyError as exc: - return on_unknown_task(payload, message, exc) + return on_unknown_task(None, message, exc) else: try: strategy( From 9bcf06aa0860863a5eded4bb53a84c2f04664d1a Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Wed, 16 Mar 2016 20:09:30 -0700 Subject: [PATCH 1098/1103] Remove leftover test statement --- funtests/stress/stress/app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/funtests/stress/stress/app.py b/funtests/stress/stress/app.py index 7b5d592d2..d4541961c 100644 --- a/funtests/stress/stress/app.py +++ b/funtests/stress/stress/app.py @@ -73,7 +73,6 @@ def _marker(s, sep='-'): @app.task def add(x, y): - add.delay(x + x, y + y) return x + y From d0a0aafde4f4848893f0a938d4e88420d12fab37 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 17 Mar 2016 16:11:35 -0700 Subject: [PATCH 1099/1103] [docs][contributing] Direct users to install dev requirements. Closes #2083 --- CONTRIBUTING.rst | 12 ++++++++++-- docs/contributing.rst | 13 +++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 91160cefd..cd1948254 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -443,12 +443,20 @@ To run the Celery test suite you need to install a few dependencies. A complete list of the dependencies needed are located in ``requirements/test.txt``. -Installing the test requirements: +If you're working on the development version, then you need to +install the development requirements first: +:: + + $ pip install -U -r requirements/dev.txt + +Both the stable and the development version have testing related +dependencies, so install these next: :: $ pip install -U -r requirements/test.txt + $ pip install -U -r requirements/default.txt -When installation of dependencies is complete you can execute +After installing the dependencies required, you can now execute the test suite by calling ``nosetests``: :: diff --git a/docs/contributing.rst b/docs/contributing.rst index 931b8883a..438bf9891 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -447,13 +447,22 @@ To run the Celery test suite you need to install a few dependencies. A complete list of the dependencies needed are located in :file:`requirements/test.txt`. -Installing the test requirements: +If you're working on the development version, then you need to +install the development requirements first: + +.. code-block:: console + + $ pip install -U -r requirements/dev.txt + +Both the stable and the development version have testing related +dependencies, so install these next: .. code-block:: console $ pip install -U -r requirements/test.txt + $ pip install -U -r requirements/default.txt -When installation of dependencies is complete you can execute +After installing the dependencies required, you can now execute the test suite by calling ``nosetests``: .. code-block:: console From c3ffe689cef80a8ecf178764b05b963f47116a5b Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 17 Mar 2016 16:36:33 -0700 Subject: [PATCH 1100/1103] [utils][LimitedSet] Removes the need for having a `sentinel` (Issue #3102) --- celery/datastructures.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/celery/datastructures.py b/celery/datastructures.py index 5b359d59c..19a1b6398 100644 --- a/celery/datastructures.py +++ b/celery/datastructures.py @@ -51,8 +51,6 @@ class LazyObject(object): # noqa <{name}({size}): maxlen={0.maxlen}, expires={0.expires}, minlen={0.minlen}>\ """ -sentinel = object() - def force_mapping(m): if isinstance(m, (LazyObject, LazySettings)): @@ -671,10 +669,7 @@ def __init__(self, maxlen=0, expires=0, data=None, minlen=0): def _refresh_heap(self): """Time consuming recreating of heap. Do not run this too often.""" - self._heap[:] = [ - entry for entry in values(self._data) - if entry is not sentinel - ] + self._heap[:] = [entry for entry in values(self._data)] heapify(self._heap) def clear(self): @@ -721,8 +716,11 @@ def update(self, other): def discard(self, item): # mark an existing item as removed. If KeyError is not found, pass. - entry = self._data.pop(item, sentinel) - if entry is not sentinel: + try: + entry = self._data.pop(item) + except KeyError: + pass + else: if self._heap_overload > self.max_heap_percent_overload: self._refresh_heap() pop_value = discard @@ -751,7 +749,11 @@ def pop(self, default=None): """Remove and return the oldest item, or :const:`None` when empty.""" while self._heap: _, item = heappop(self._heap) - if self._data.pop(item, None) is not sentinel: + try: + self._data.pop(item) + except KeyError: + pass + else: return item return default From 0bf3ab662bbbe85b1ea4b8073e9eef82596136bf Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Tue, 8 Mar 2016 19:29:43 -0800 Subject: [PATCH 1101/1103] [result][redis] Use pubsub for consuming results, and use the new async backend interface (Issue #2511) --- celery/backends/amqp.py | 21 ++++++++------- celery/backends/async.py | 18 ++++++++++--- celery/backends/base.py | 11 +++++--- celery/backends/redis.py | 57 ++++++++++++++++++++++++++++++++++++++-- celery/backends/rpc.py | 4 ++- 5 files changed, 92 insertions(+), 19 deletions(-) diff --git a/celery/backends/amqp.py b/celery/backends/amqp.py index 0bb925d19..6af14a192 100644 --- a/celery/backends/amqp.py +++ b/celery/backends/amqp.py @@ -49,13 +49,16 @@ class NoCacheQueue(Queue): class ResultConsumer(BaseResultConsumer): Consumer = Consumer + _connection = None + _consumer = None + def __init__(self, *args, **kwargs): super(ResultConsumer, self).__init__(*args, **kwargs) - self._connection = None - self._consumer = None + self._create_binding = self.backend._create_binding - def start(self, initial_queue, no_ack=True): + def start(self, initial_task_id, no_ack=True): self._connection = self.app.connection() + initial_queue = self._create_binding(initial_task_id) self._consumer = self.Consumer( self._connection.default_channel, [initial_queue], callbacks=[self.on_state_change], no_ack=no_ack, @@ -77,16 +80,17 @@ def on_after_fork(self): self._connection.collect() self._connection = None - def consume_from(self, queue): + def consume_from(self, task_id): if self._consumer is None: - return self.start(queue) + return self.start(task_id) + queue = self._create_binding(task_id) if not self._consumer.consuming_from(queue): self._consumer.add_queue(queue) self._consumer.consume() - def cancel_for(self, queue): + def cancel_for(self, task_id): if self._consumer: - self._consumer.cancel_by_queue(queue.name) + self._consumer.cancel_by_queue(self._create_binding(task_id).name) class AMQPBackend(base.Backend, AsyncBackendMixin): @@ -138,9 +142,6 @@ def _after_fork(self): self._pending_results.clear() self.result_consumer._after_fork() - def on_result_fulfilled(self, result): - self.result_consumer.cancel_for(self._create_binding(result.id)) - def _create_exchange(self, name, type='direct', delivery_mode=2): return self.Exchange(name=name, type=type, diff --git a/celery/backends/async.py b/celery/backends/async.py index ddb56287f..0ff5ac045 100644 --- a/celery/backends/async.py +++ b/celery/backends/async.py @@ -135,7 +135,7 @@ def iter_native(self, result, timeout=None, interval=0.5, no_ack=True, def add_pending_result(self, result): if result.id not in self._pending_results: self._pending_results[result.id] = result - self.result_consumer.consume_from(self._create_binding(result.id)) + self.result_consumer.consume_from(result.id) return result def remove_pending_result(self, result): @@ -144,7 +144,7 @@ def remove_pending_result(self, result): return result def on_result_fulfilled(self, result): - pass + self.result_consumer.cancel_for(result.id) def wait_for_pending(self, result, callback=None, propagate=True, **kwargs): @@ -177,8 +177,20 @@ def __init__(self, backend, app, accept, pending_results): self.buckets = WeakKeyDictionary() self.drainer = drainers[detect_environment()](self) + def start(self): + raise NotImplementedError() + + def stop(self): + pass + def drain_events(self, timeout=None): - raise NotImplementedError('subclass responsibility') + raise NotImplementedError() + + def consume_from(self, task_id): + raise NotImplementedError() + + def cancel_for(self, task_id): + raise NotImplementedError() def _after_fork(self): self.bucket.clear() diff --git a/celery/backends/base.py b/celery/backends/base.py index c1793fa83..4077a5ac8 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -448,7 +448,7 @@ class BaseBackend(Backend, SyncBackendMixin): BaseDictBackend = BaseBackend # XXX compat -class KeyValueStoreBackend(BaseBackend): +class BaseKeyValueStoreBackend(Backend): key_t = ensure_bytes task_keyprefix = 'celery-task-meta-' group_keyprefix = 'celery-taskset-meta-' @@ -459,7 +459,7 @@ def __init__(self, *args, **kwargs): if hasattr(self.key_t, '__func__'): # pragma: no cover self.key_t = self.key_t.__func__ # remove binding self._encode_prefixes() - super(KeyValueStoreBackend, self).__init__(*args, **kwargs) + super(BaseKeyValueStoreBackend, self).__init__(*args, **kwargs) if self.implements_incr: self.apply_chord = self._apply_chord_incr @@ -578,7 +578,8 @@ def _forget(self, task_id): def _store_result(self, task_id, result, state, traceback=None, request=None, **kwargs): meta = {'status': state, 'result': result, 'traceback': traceback, - 'children': self.current_task_children(request)} + 'children': self.current_task_children(request), + 'task_id': task_id} self.set(self.get_key_for_task(task_id), self.encode(meta)) return result @@ -683,6 +684,10 @@ def on_chord_part_return(self, request, state, result, **kwargs): self.expire(key, 86400) +class KeyValueStoreBackend(BaseKeyValueStoreBackend, SyncBackendMixin): + pass + + class DisabledBackend(BaseBackend): _cache = {} # need this attribute to reset cache in tests. diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 00bc01227..8cbb8fe27 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -14,6 +14,7 @@ from kombu.utils.url import _parse_url from celery import states +from celery._state import task_join_will_block from celery.canvas import maybe_signature from celery.exceptions import ChordError, ImproperlyConfigured from celery.five import string_t @@ -22,7 +23,8 @@ from celery.utils.log import get_logger from celery.utils.timeutils import humanize_seconds -from .base import KeyValueStoreBackend +from . import async +from . import base try: import redis @@ -47,9 +49,54 @@ error = logger.error -class RedisBackend(KeyValueStoreBackend): +class ResultConsumer(async.BaseResultConsumer): + + _pubsub = None + + def __init__(self, *args, **kwargs): + super(ResultConsumer, self).__init__(*args, **kwargs) + self._get_key_for_task = self.backend.get_key_for_task + self._decode_result = self.backend.decode_result + self.subscribed_to = set() + + def start(self, initial_task_id): + self._pubsub = self.backend.client.pubsub( + ignore_subscribe_messages=True, + ) + self._consume_from(initial_task_id) + + def stop(self): + if self._pubsub is not None: + self._pubsub.close() + + def drain_events(self, timeout=None): + m = self._pubsub.get_message(timeout=timeout) + if m and m['type'] == 'message': + self.on_state_change(self._decode_result(m['data']), m) + + def consume_from(self, task_id): + if self._pubsub is None: + return self.start(task_id) + self._consume_from(task_id) + + def _consume_from(self, task_id): + key = self._get_key_for_task(task_id) + if key not in self.subscribed_to: + self.subscribed_to.add(key) + self._pubsub.subscribe(key) + + def cancel_for(self, task_id): + if self._pubsub: + key = self._get_key_for_task(task_id) + self.subscribed_to.discard(key) + self._pubsub.unsubscribe(key) + + +class RedisBackend(base.BaseKeyValueStoreBackend, async.AsyncBackendMixin): """Redis task result store.""" + ResultConsumer = ResultConsumer + #: redis-py client module. redis = redis @@ -93,6 +140,8 @@ def __init__(self, host=None, port=None, db=None, password=None, self.connection_errors, self.channel_errors = ( get_redis_error_classes() if get_redis_error_classes else ((), ())) + self.result_consumer = self.ResultConsumer( + self, self.app, self.accept, self._pending_results) def _params_from_url(self, url, defaults): scheme, host, port, user, password, path, query = _parse_url(url) @@ -124,6 +173,10 @@ def _params_from_url(self, url, defaults): connparams.update(query) return connparams + def on_task_call(self, producer, task_id): + if not task_join_will_block(): + self.result_consumer.consume_from(task_id) + def get(self, key): return self.client.get(key) diff --git a/celery/backends/rpc.py b/celery/backends/rpc.py index 7c6c68ebb..620055583 100644 --- a/celery/backends/rpc.py +++ b/celery/backends/rpc.py @@ -13,6 +13,7 @@ from kombu.utils import cached_property from celery import current_task +from celery._state import task_join_will_block from celery.backends import amqp __all__ = ['RPCBackend'] @@ -29,7 +30,8 @@ def _create_exchange(self, name, type='direct', delivery_mode=2): return Exchange(None) def on_task_call(self, producer, task_id): - maybe_declare(self.binding(producer.channel), retry=True) + if not task_join_will_block(): + maybe_declare(self.binding(producer.channel), retry=True) def _create_binding(self, task_id): return self.binding From c34a98df51d18af638cbcef07b690bc393d55829 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 17 Mar 2016 22:09:37 -0700 Subject: [PATCH 1102/1103] [Stress] Expose redis group --- funtests/stress/stress/__main__.py | 2 +- funtests/stress/stress/suite.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/funtests/stress/stress/__main__.py b/funtests/stress/stress/__main__.py index f83c8c192..1b5b97579 100644 --- a/funtests/stress/stress/__main__.py +++ b/funtests/stress/stress/__main__.py @@ -31,7 +31,7 @@ def get_options(self): Option('-r', '--repeat', type='float', default=0, help='Number of times to repeat the test suite'), Option('-g', '--group', default='all', - help='Specify test group (all|green)'), + help='Specify test group (all|green|redis)'), Option('--diag', default=False, action='store_true', help='Enable diagnostics (slow)'), Option('-J', '--no-join', default=False, action='store_true', diff --git a/funtests/stress/stress/suite.py b/funtests/stress/stress/suite.py index fa237b790..daff39cd8 100755 --- a/funtests/stress/stress/suite.py +++ b/funtests/stress/stress/suite.py @@ -273,12 +273,12 @@ def _is_descriptor(obj, attr): class Suite(BaseSuite): - @testcase('all', 'green', iterations=1) + @testcase('all', 'green', 'redis', iterations=1) def chain(self): c = add.s(4, 4) | add.s(8) | add.s(16) assert_equal(self.join(c()), 32) - @testcase('all', 'green', iterations=1) + @testcase('all', 'green', 'redis', iterations=1) def chaincomplex(self): c = ( add.s(2, 2) | ( @@ -289,7 +289,7 @@ def chaincomplex(self): res = c() assert_equal(res.get(), [32, 33, 34, 35]) - @testcase('all', 'green', iterations=1) + @testcase('all', 'green', 'redis', iterations=1) def parentids_chain(self, num=248): c = chain(ids.si(i) for i in range(num)) c.freeze() @@ -297,7 +297,7 @@ def parentids_chain(self, num=248): res.get(timeout=5) self.assert_ids(res, num - 1) - @testcase('all', 'green', iterations=1) + @testcase('all', 'green', 'redis', iterations=1) def parentids_group(self): g = ids.si(1) | ids.si(2) | group(ids.si(i) for i in range(2, 50)) res = g() From e6835bdbd266d1a0917f854e6e60b0821a7b53b1 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Thu, 17 Mar 2016 22:15:08 -0700 Subject: [PATCH 1103/1103] [Redis][async] Fixes waiting for groups, and more --- celery/backends/async.py | 45 ++++++++++++++++--------------- celery/backends/base.py | 4 ++- celery/backends/redis.py | 5 ++++ celery/result.py | 12 +++++++-- celery/tests/tasks/test_result.py | 25 +++++++++-------- 5 files changed, 56 insertions(+), 35 deletions(-) diff --git a/celery/backends/async.py b/celery/backends/async.py index 0ff5ac045..aac64bb5d 100644 --- a/celery/backends/async.py +++ b/celery/backends/async.py @@ -8,9 +8,9 @@ from __future__ import absolute_import, unicode_literals import socket -import time from collections import deque +from time import sleep from weakref import WeakKeyDictionary from kombu.syn import detect_environment @@ -82,7 +82,7 @@ def wait_for(self, p, wait, timeout=None): if self._g is None: self.start() if not p.ready: - time.sleep(0) + sleep(0) @register_drainer('eventlet') @@ -115,22 +115,22 @@ def iter_native(self, result, timeout=None, interval=0.5, no_ack=True, raise StopIteration() bucket = deque() - for result in results: - if result._cache: - bucket.append(result) + for node in results: + if node._cache: + bucket.append(node) else: - self._collect_into(result, bucket) + self._collect_into(node, bucket) for _ in self._wait_for_pending( result, timeout=timeout, interval=interval, no_ack=no_ack, on_message=on_message, on_interval=on_interval): while bucket: - result = bucket.popleft() - yield result.id, result._cache + node = bucket.popleft() + yield result.id, node._cache while bucket: - result = bucket.popleft() - yield result.id, result._cache + node = bucket.popleft() + yield result.id, node._cache def add_pending_result(self, result): if result.id not in self._pending_results: @@ -152,13 +152,12 @@ def wait_for_pending(self, result, pass return result.maybe_throw(callback=callback, propagate=propagate) - def _wait_for_pending(self, result, timeout=None, interval=0.5, - no_ack=True, on_interval=None, on_message=None, - callback=None, propagate=True): + def _wait_for_pending(self, result, + timeout=None, on_interval=None, on_message=None, + **kwargs): return self.result_consumer._wait_for_pending( - result, timeout=timeout, interval=interval, - no_ack=no_ack, on_interval=on_interval, - callback=callback, on_message=on_message, propagate=propagate, + result, timeout=timeout, + on_interval=on_interval, on_message=on_message, ) @property @@ -205,21 +204,25 @@ def drain_events_until(self, p, timeout=None, on_interval=None): return self.drainer.drain_events_until( p, timeout=timeout, on_interval=on_interval) - def _wait_for_pending(self, result, timeout=None, interval=0.5, - no_ack=True, on_interval=None, callback=None, - on_message=None, propagate=True): + def _wait_for_pending(self, result, + timeout=None, on_interval=None, on_message=None, + **kwargs): + self.on_wait_for_pending(result, timeout=timeout, **kwargs) prev_on_m, self.on_message = self.on_message, on_message try: for _ in self.drain_events_until( result.on_ready, timeout=timeout, on_interval=on_interval): yield - time.sleep(0) + sleep(0) except socket.timeout: raise TimeoutError('The operation timed out.') finally: self.on_message = prev_on_m + def on_wait_for_pending(self, result, timeout=None, **kwargs): + pass + def on_out_of_band_result(self, message): self.on_state_change(message.payload, message) @@ -238,4 +241,4 @@ def on_state_change(self, meta, message): buckets.pop(result) except KeyError: pass - time.sleep(0) + sleep(0) diff --git a/celery/backends/base.py b/celery/backends/base.py index 4077a5ac8..6fe734cec 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -537,7 +537,7 @@ def _mget_to_results(self, values, keys): } def get_many(self, task_ids, timeout=None, interval=0.5, no_ack=True, - on_message=None, on_interval=None, + on_message=None, on_interval=None, max_iterations=None, READY_STATES=states.READY_STATES): interval = 0.5 if interval is None else interval ids = task_ids if isinstance(task_ids, set) else set(task_ids) @@ -571,6 +571,8 @@ def get_many(self, task_ids, timeout=None, interval=0.5, no_ack=True, on_interval() time.sleep(interval) # don't busy loop. iterations += 1 + if max_iterations and iterations >= max_iterations: + break def _forget(self, task_id): self.delete(self.get_key_for_task(task_id)) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 8cbb8fe27..5daecd381 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -65,6 +65,11 @@ def start(self, initial_task_id): ) self._consume_from(initial_task_id) + def on_wait_for_pending(self, result, **kwargs): + for meta in result._iter_meta(): + if meta is not None: + self.on_state_change(meta, None) + def stop(self): if self._pubsub is not None: self._pubsub.close() diff --git a/celery/result.py b/celery/result.py index c6fe26ee2..ff5f89ce0 100644 --- a/celery/result.py +++ b/celery/result.py @@ -15,7 +15,7 @@ from copy import copy from kombu.utils import cached_property -from vine import Thenable, promise +from vine import Thenable, barrier, promise from . import current_app from . import states @@ -356,6 +356,9 @@ def _get_task_meta(self): return self._maybe_set_cache(self.backend.get_task_meta(self.id)) return self._cache + def _iter_meta(self): + return iter([self._get_task_meta()]) + def _set_cache(self, d): children = d.get('children') if children: @@ -438,7 +441,7 @@ def __init__(self, results, app=None, ready_barrier=None, **kwargs): self._cache = None self.results = results self.on_ready = promise(args=(self,)) - self._on_full = ready_barrier + self._on_full = ready_barrier or barrier(results) if self._on_full: self._on_full.then(promise(self.on_ready)) @@ -737,6 +740,11 @@ def join_native(self, timeout=None, propagate=True, acc[order_index[task_id]] = value return acc + def _iter_meta(self): + return (meta for _, meta in self.backend.get_many( + {r.id for r in self.results}, max_iterations=1, + )) + def _failed_join_report(self): return (res for res in self.results if res.backend.is_cached(res.id) and diff --git a/celery/tests/tasks/test_result.py b/celery/tests/tasks/test_result.py index b1b6c100e..64829a443 100644 --- a/celery/tests/tasks/test_result.py +++ b/celery/tests/tasks/test_result.py @@ -320,8 +320,11 @@ def test_resultset_repr(self): [self.app.AsyncResult(t) for t in ['1', '2', '3']]))) def test_eq_other(self): - self.assertFalse(self.app.ResultSet([1, 3, 3]) == 1) - self.assertTrue(self.app.ResultSet([1]) == self.app.ResultSet([1])) + self.assertFalse(self.app.ResultSet( + [self.app.AsyncResult(t) for t in [1, 3, 3]]) == 1) + rs1 = self.app.ResultSet([self.app.AsyncResult(1)]) + rs2 = self.app.ResultSet([self.app.AsyncResult(1)]) + self.assertTrue(rs1 == rs2) def test_get(self): x = self.app.ResultSet([self.app.AsyncResult(t) for t in [1, 2, 3]]) @@ -336,18 +339,18 @@ def test_get(self): self.assertTrue(x.join_native.called) def test_eq_ne(self): - g1 = self.app.ResultSet( + g1 = self.app.ResultSet([ self.app.AsyncResult('id1'), self.app.AsyncResult('id2'), - ) - g2 = self.app.ResultSet( + ]) + g2 = self.app.ResultSet([ self.app.AsyncResult('id1'), self.app.AsyncResult('id2'), - ) - g3 = self.app.ResultSet( + ]) + g3 = self.app.ResultSet([ self.app.AsyncResult('id3'), self.app.AsyncResult('id1'), - ) + ]) self.assertEqual(g1, g2) self.assertNotEqual(g1, g3) self.assertNotEqual(g1, object()) @@ -366,10 +369,10 @@ def test_get_empty(self): self.assertTrue(x.join.called) def test_add(self): - x = self.app.ResultSet([1]) - x.add(2) + x = self.app.ResultSet([self.app.AsyncResult(1)]) + x.add(self.app.AsyncResult(2)) self.assertEqual(len(x), 2) - x.add(2) + x.add(self.app.AsyncResult(2)) self.assertEqual(len(x), 2) @contextmanager