"""Worker Consumer Blueprint.

This module contains the components responsible for consuming messages
from the broker, processing the messages and keeping the broker connections
up and running.
"""
import errno
import logging
import os
import warnings
from collections import defaultdict, deque
from time import sleep

from billiard.common import restart_state
from billiard.exceptions import RestartFreqExceeded
from kombu.asynchronous.semaphore import DummyLock
from kombu.common import ignore_errors
from kombu.exceptions import ContentDisallowed, DecodeError
from kombu.utils.compat import _detect_environment
from kombu.utils.encoding import safe_repr
from kombu.utils.limits import TokenBucket
from vine import ppartial, promise

from celery import bootsteps, signals
from celery.app.trace import build_tracer
from celery.exceptions import (CPendingDeprecationWarning, InvalidTaskError, NotRegistered, WorkerShutdown,
                               WorkerTerminate)
from celery.utils.functional import noop
from celery.utils.log import get_logger
from celery.utils.nodenames import gethostname
from celery.utils.objects import Bunch
from celery.utils.text import truncate
from celery.utils.threads import bound_open_broker_sockets
from celery.utils.time import humanize_seconds, rate
from celery.worker import loops
from celery.worker.state import (active_requests, maybe_shutdown, requests, reserved_requests, scheduled_requests,
                                 successful_requests, task_reserved)

__all__ = ('Consumer', 'Evloop', 'dump_body')

CLOSE = bootsteps.CLOSE
TERMINATE = bootsteps.TERMINATE
STOP_CONDITIONS = {CLOSE, TERMINATE}
logger = get_logger(__name__)
debug, info, warn, error, crit = (logger.debug, logger.info, logger.warning,
                                  logger.error, logger.critical)

CONNECTION_RETRY = """\
consumer: Connection to broker lost. \
Trying to re-establish the connection...\
"""

CONNECTION_RETRY_STEP = """\
Trying again {when}... ({retries}/{max_retries})\
"""

CONNECTION_ERROR = """\
consumer: Cannot connect to %s: %s.
%s
"""

CONNECTION_FAILOVER = """\
Will retry using next failover.\
"""

#: Timeout (seconds) passed to ``connection.collect()`` when cleaning up
#: a broken broker connection.  Prevents the cleanup path from blocking
#: indefinitely on a dead socket (see :issue:`9705`).
COLLECT_SOCKET_TIMEOUT = 5.0

UNKNOWN_FORMAT = """\
Received and deleted unknown message.  Wrong destination?!?

The full contents of the message body was: %s
"""

#: Error message for when an unregistered task is received.
UNKNOWN_TASK_ERROR = """\
Received unregistered task of type %s.
The message has been ignored and discarded.

Did you remember to import the module containing this task?
Or maybe you're using relative imports?

Please see
https://docs.celeryq.dev/en/latest/internals/protocol.html
for more information.

The full contents of the message body was:
%s

The full contents of the message headers:
%s

The delivery info for this task is:
%s
"""

#: Error message for when an invalid task message is received.
INVALID_TASK_ERROR = """\
Received invalid task message: %s
The message has been ignored and discarded.

Please ensure your message conforms to the task
message protocol as described here:
https://docs.celeryq.dev/en/latest/internals/protocol.html

The full contents of the message body was:
%s
"""

MESSAGE_DECODE_ERROR = """\
Can't decode message body: %r [type:%r encoding:%r headers:%s]

body: %s
"""

MESSAGE_REPORT = """\
body: {0}
{{content_type:{1} content_encoding:{2}
  delivery_info:{3} headers={4}}}
"""

TERMINATING_TASK_ON_RESTART_AFTER_A_CONNECTION_LOSS = """\
Task %s cannot be acknowledged after a connection loss since late acknowledgement is enabled for it.
Terminating it instead.
"""

CANCEL_TASKS_BY_DEFAULT = """
In Celery 5.1 we introduced an optional breaking change which
on connection loss cancels all currently executed tasks with late acknowledgement enabled.
These tasks cannot be acknowledged as the connection is gone, and the tasks are automatically redelivered
back to the queue. You can enable this behavior using the worker_cancel_long_running_tasks_on_connection_loss
setting. In Celery 5.1 it is set to False by default. The setting will be set to True by default in Celery 6.0.
"""


def dump_body(m, body):
    """Format message body for debugging purposes."""
    # v2 protocol does not deserialize body
    body = m.body if body is None else body
    return '{} ({}b)'.format(truncate(safe_repr(body), 1024),
                             len(m.body))


class Consumer:
    """Consumer blueprint."""

    Strategies = dict

    #: Optional callback called the first time the worker
    #: is ready to receive tasks.
    init_callback = None

    #: The current worker pool instance.
    pool = None

    #: A timer used for high-priority internal tasks, such
    #: as sending heartbeats.
    timer = None

    #: Entries on the event loop timer before the consumer last started
    #: connecting to the broker.
    timer_entries_before_start = None

    #: Entries the event loop registered on its timer for the current
    #: connection, or :const:`None` if it has not registered them yet.
    timer_entries_registered = None

    restart_count = -1  # first start is the same as a restart

    #: This flag will be turned off after the first failed
    #: connection attempt.
    first_connection_attempt = True

    #: Counter to track number of conn retry attempts
    #: to broker. Will be reset to 0 once successful
    broker_connection_retry_attempt = 0

    class Blueprint(bootsteps.Blueprint):
        """Consumer blueprint."""

        name = 'Consumer'
        default_steps = [
            'celery.worker.consumer.connection:Connection',
            'celery.worker.consumer.mingle:Mingle',
            'celery.worker.consumer.events:Events',
            'celery.worker.consumer.gossip:Gossip',
            'celery.worker.consumer.heart:Heart',
            'celery.worker.consumer.control:Control',
            'celery.worker.consumer.tasks:Tasks',
            'celery.worker.consumer.delayed_delivery:DelayedDelivery',
            'celery.worker.consumer.consumer:Evloop',
            'celery.worker.consumer.agent:Agent',
        ]

        def shutdown(self, parent):
            self.send_all(parent, 'shutdown')

    def __init__(self, on_task_request,
                 init_callback=noop, hostname=None,
                 pool=None, app=None,
                 timer=None, controller=None, hub=None, amqheartbeat=None,
                 worker_options=None, disable_rate_limits=False,
                 initial_prefetch_count=2, prefetch_multiplier=1, **kwargs):
        self.app = app
        self.controller = controller
        self.init_callback = init_callback
        self.hostname = hostname or gethostname()
        self.pid = os.getpid()
        self.pool = pool
        self.timer = timer
        self.strategies = self.Strategies()
        self.conninfo = self.app.connection_for_read()
        self.connection_errors = self.conninfo.connection_errors
        self.channel_errors = self.conninfo.channel_errors
        self._restart_state = restart_state(maxR=5, maxT=1)

        self._does_info = logger.isEnabledFor(logging.INFO)
        self._limit_order = 0
        self.on_task_request = on_task_request
        self.on_task_message = set()
        self.amqheartbeat_rate = self.app.conf.broker_heartbeat_checkrate
        self.disable_rate_limits = disable_rate_limits
        self.initial_prefetch_count = initial_prefetch_count
        self.prefetch_multiplier = prefetch_multiplier
        self._maximum_prefetch_restored = True
        # Effective QoS mode, recorded by the Tasks bootstep once the
        # connection is established. ``None`` means "unknown" and preserves
        # legacy behavior; ``False`` indicates per-consumer QoS (e.g. quorum
        # queues) where ``basic.qos`` updates do not propagate to already
        # running consumers. See ``on_connection_error_after_connected``.
        self.qos_global = None

        # this contains a tokenbucket for each task type by name, used for
        # rate limits, or None if rate limits are disabled for that task.
        self.task_buckets = defaultdict(lambda: None)
        self.reset_rate_limits()

        self.hub = hub
        if self.hub or getattr(self.pool, 'is_green', False):
            self.amqheartbeat = amqheartbeat
            if self.amqheartbeat is None:
                self.amqheartbeat = self.app.conf.broker_heartbeat
        else:
            self.amqheartbeat = 0

        # Reused for every buffered event, so that the hub's ready set
        # holds at most one pending flush.
        self._flush_events_promise = promise(self._flush_events)

        if not hasattr(self, 'loop'):
            self.loop = loops.asynloop if hub else loops.synloop

        if _detect_environment() == 'gevent':
            # there's a gevent bug that causes timeouts to not be reset,
            # so if the connection timeout is exceeded once, it can NEVER
            # connect again.
            self.app.conf.broker_connection_timeout = None

        self._pending_operations = deque()

        self.steps = []
        self.blueprint = self.Blueprint(
            steps=self.app.steps['consumer'],
            on_close=self.on_close,
        )
        self.blueprint.apply(self, **dict(worker_options or {}, **kwargs))

    def call_soon(self, p, *args, **kwargs):
        """Schedule a callback.

        Callback ordering is not guaranteed across pool implementations.
        """
        p = ppartial(p, *args, **kwargs)
        if self.hub:
            return self.hub.call_soon(p)
        self._pending_operations.append(p)
        return p

    def call_soon_ack(self, p, *args, **kwargs):
        """Execute ack/reject callback immediately, bypassing _pending_operations.

        In synloop (gevent/eventlet) workers, the standard call_soon defers
        callbacks to _pending_operations, which are only drained at the top
        of the next synloop iteration — after drain_events() returns.  With
        acks_late and prefetch_multiplier=1 this means the broker cannot
        deliver the next message until an unrelated AMQP frame arrives,
        adding 50-400 ms of latency between every pair of tasks.

        This method is intentionally scoped to ack/reject callbacks, which
        only write an AMQP basic.ack/basic.reject frame to the broker socket.
        Other call_soon users (e.g. remote-control commands from gPidbox)
        continue to use deferred execution to preserve greenlet-safety.
        """
        p = ppartial(p, *args, **kwargs)
        if self.hub:
            return self.hub.call_soon(p)
        try:
            p()
        except Exception as exc:  # pylint: disable=broad-except
            logger.exception('call_soon_ack immediate exec failed: %r', exc)
        return p

    def perform_pending_operations(self):
        if not self.hub:
            while self._pending_operations:
                try:
                    self._pending_operations.popleft()()
                except Exception as exc:  # pylint: disable=broad-except
                    logger.exception('Pending callback raised: %r', exc)

    def bucket_for_task(self, type):
        limit = rate(getattr(type, 'rate_limit', None))
        return TokenBucket(limit, capacity=1) if limit else None

    def reset_rate_limits(self):
        self.task_buckets.update(
            (n, self.bucket_for_task(t)) for n, t in self.app.tasks.items()
        )

    def _update_prefetch_count(self, index=0):
        """Update prefetch count after pool/shrink grow operations.

        Index must be the change in number of processes as a positive
        (increasing) or negative (decreasing) number.

        Note:
            Currently pool grow operations will end up with an offset
            of +1 if the initial size of the pool was 0 (e.g.
            :option:`--autoscale=1,0 <celery worker --autoscale>`).
        """
        num_processes = self.pool.num_processes
        if not self.initial_prefetch_count or not num_processes:
            return  # prefetch disabled
        self.initial_prefetch_count = (
            self.pool.num_processes * self.prefetch_multiplier
        )
        return self._update_qos_eventually(index)

    def _update_qos_eventually(self, index):
        return (self.qos.decrement_eventually if index < 0
                else self.qos.increment_eventually)(
            abs(index) * self.prefetch_multiplier)

    def _limit_move_to_pool(self, request):
        task_reserved(request)
        self.on_task_request(request)

    def _schedule_bucket_request(self, bucket):
        while True:
            try:
                request, tokens = bucket.pop()
            except IndexError:
                # no request, break
                break

            if bucket.can_consume(tokens):
                self._limit_move_to_pool(request)
                continue
            else:
                # requeue to head, keep the order.
                bucket.contents.appendleft((request, tokens))

                pri = self._limit_order = (self._limit_order + 1) % 10
                hold = bucket.expected_time(tokens)
                self.timer.call_after(
                    hold, self._schedule_bucket_request, (bucket,),
                    priority=pri,
                )
                # no tokens, break
                break

    def _limit_task(self, request, bucket, tokens):
        bucket.add((request, tokens))
        return self._schedule_bucket_request(bucket)

    def _limit_post_eta(self, request, bucket, tokens):
        self.qos.decrement_eventually()
        bucket.add((request, tokens))
        return self._schedule_bucket_request(bucket)

    def start(self):
        blueprint = self.blueprint
        while blueprint.state not in STOP_CONDITIONS:
            maybe_shutdown()
            if self.restart_count:
                try:
                    self._restart_state.step()
                except RestartFreqExceeded as exc:
                    crit('Frequent restarts detected: %r', exc, exc_info=1)
                    sleep(1)
            self.restart_count += 1
            if self.app.conf.broker_channel_error_retry:
                recoverable_errors = (self.connection_errors + self.channel_errors)
            else:
                recoverable_errors = self.connection_errors
            if self.hub is not None:
                # Entries added to the timer from here on, by connecting
                # and starting the event loop, belong to this connection
                # (see :func:`celery.worker.loops.asynloop`).
                self.timer_entries_before_start = loops._timer_entries(
                    self.hub.timer)
                self.timer_entries_registered = None
            try:
                blueprint.start(self)
            except recoverable_errors as exc:
                # If we're not retrying connections, we need to properly shutdown or terminate
                # the Celery main process instead of abruptly aborting the process without any cleanup.
                is_connection_loss_on_startup = self.first_connection_attempt
                self.first_connection_attempt = False
                connection_retry_type = self._get_connection_retry_type(is_connection_loss_on_startup)
                connection_retry = self.app.conf[connection_retry_type]
                if not connection_retry:
                    crit(
                        f"Retrying to {'establish' if is_connection_loss_on_startup else 're-establish'} "
                        f"a connection to the message broker after a connection loss has "
                        f"been disabled (app.conf.{connection_retry_type}=False). Shutting down..."
                    )
                    raise WorkerShutdown(1) from exc
                if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
                    crit("Too many open files. Aborting...")
                    raise WorkerTerminate(1) from exc
                maybe_shutdown()
                if blueprint.state not in STOP_CONDITIONS:
                    if self.connection:
                        self.on_connection_error_after_connected(exc)
                    else:
                        self.on_connection_error_before_connected(exc)
                    self.on_close()
                    self._cancel_connection_timer_entries()
                    blueprint.restart(self)

    def _cancel_connection_timer_entries(self):
        """Cancel the timer entries registered for the failed connection.

        :func:`~celery.worker.loops.asynloop` records the entries it
        registered for the connection.  If the connection failed before
        that, e.g. in a bootstep that runs after the transport connected,
        every entry added since the consumer started connecting belongs to
        it: the event loop was not running, so nothing else added entries
        in the meantime.  Without this, those entries would be part of the
        next attempt's snapshot and never be cancelled.

        Cancelling an entry twice is harmless, so this also runs after
        :func:`~celery.worker.loops.asynloop` cancelled its own entries.
        """
        if self.hub is None or self.timer_entries_before_start is None:
            return
        entries = self.timer_entries_registered
        if entries is None:
            entries = (loops._timer_entries(self.hub.timer) -
                       self.timer_entries_before_start)
        loops._cancel_timer_entries(self.hub.timer, entries)

    def _get_connection_retry_type(self, is_connection_loss_on_startup):
        return ('broker_connection_retry_on_startup'
                if (is_connection_loss_on_startup
                    and self.app.conf.broker_connection_retry_on_startup is not None)
                else 'broker_connection_retry')

    def on_connection_error_before_connected(self, exc):
        error(CONNECTION_ERROR, self.conninfo.as_uri(), exc,
              'Trying to reconnect...')

    def on_connection_error_after_connected(self, exc):
        # Keep the exception in the warning so default-level logs still say
        # what went wrong, but leave the traceback for debug logging where
        # application log handlers will not capture it.
        warn('%s (%r)', CONNECTION_RETRY, exc)
        debug('Connection error: %r', exc, exc_info=exc)
        # Bound the sockets that are already open before cleanup reads from
        # them; collect()'s socket_timeout only applies to new sockets.
        bound_open_broker_sockets(self.connection, COLLECT_SOCKET_TIMEOUT)
        try:
            self.connection.collect(socket_timeout=COLLECT_SOCKET_TIMEOUT)
        except Exception:  # pylint: disable=broad-except
            pass

        # Close the broken connection so the old socket is released
        # before blueprint.restart() begins the reconnect cycle.
        # We close here rather than in Connection.stop() because stop()
        # also runs during graceful shutdown where the connection must
        # stay open for in-flight task acks.
        connection, self.connection = self.connection, None
        if connection:
            ignore_errors(connection, connection.close)

        if self.app.conf.worker_cancel_long_running_tasks_on_connection_loss:
            for request in tuple(active_requests):
                if request.task.acks_late and not request.acknowledged:
                    warn(TERMINATING_TASK_ON_RESTART_AFTER_A_CONNECTION_LOSS,
                         request)
                    try:
                        request.cancel(self.pool)
                    except Exception:  # pylint: disable=broad-except
                        warn("Failed to cancel active request %r after connection loss", request, exc_info=True)
        else:
            warnings.warn(CANCEL_TASKS_BY_DEFAULT, CPendingDeprecationWarning, stacklevel=2)

        if self.app.conf.worker_enable_prefetch_count_reduction:
            # Per-consumer QoS mode (quorum queues, apply_global=False) does
            # not propagate ``basic.qos`` updates to already-running consumers
            # so the gradual restoration step in
            # ``_restore_prefetch_count_after_connection_restart`` is a no-op
            # and the worker would stay stuck at the reduced count after one
            # reconnect. Skip the reduction entirely in that mode. See #9512.
            if self.qos_global is False:
                # Also clear any reduced state left over from an earlier
                # reconnect that took the legacy path (e.g. before
                # ``Tasks.start()`` had a chance to record ``qos_global``).
                # Without this reset the new consumer would be created with
                # the stale reduced prefetch count.
                self.initial_prefetch_count = self.max_prefetch_count
                self._maximum_prefetch_restored = True
                logger.info(
                    "Skipping prefetch count reduction after connection "
                    "restart because per-consumer QoS (apply_global=False) "
                    "is in effect and broker-side prefetch updates would "
                    "not reach the running consumer."
                )
            else:
                # Snapshot the active request count once so the reduction
                # math and the log message agree, and to avoid the O(n)
                # ``tuple(active_requests)`` allocation that was being
                # used purely to call ``len()`` on a WeakSet.
                active_count = len(active_requests)
                self.initial_prefetch_count = max(
                    self.prefetch_multiplier,
                    self.max_prefetch_count - active_count * self.prefetch_multiplier
                )

                self._maximum_prefetch_restored = self.initial_prefetch_count == self.max_prefetch_count
                if not self._maximum_prefetch_restored:
                    logger.info(
                        f"Temporarily reducing the prefetch count to {self.initial_prefetch_count} to avoid "
                        f"over-fetching since {active_count} tasks are currently being processed.\n"
                        f"The prefetch count will be gradually restored to {self.max_prefetch_count} as the tasks "
                        "complete processing."
                    )

    def register_with_event_loop(self, hub):
        self.blueprint.send_all(
            self, 'register_with_event_loop', args=(hub,),
            description='Hub.register',
        )

    def shutdown(self):
        self.perform_pending_operations()
        self.blueprint.shutdown(self)

    def stop(self):
        self.blueprint.stop(self)

    def on_ready(self):
        callback, self.init_callback = self.init_callback, None
        if callback:
            callback(self)

    def loop_args(self):
        return (self, self.connection, self.task_consumer,
                self.blueprint, self.hub, self.qos, self.amqheartbeat,
                self.app.clock, self.amqheartbeat_rate)

    def on_decode_error(self, message, exc):
        """Callback called if an error occurs while decoding a message.

        Simply logs the error and acknowledges the message so it
        doesn't enter a loop.

        Arguments:
            message (kombu.Message): The message received.
            exc (Exception): The exception being handled.
        """
        crit(MESSAGE_DECODE_ERROR,
             exc, message.content_type, message.content_encoding,
             safe_repr(message.headers), dump_body(message, message.body),
             exc_info=1)
        message.ack()

    def on_close(self):
        # Clear internal queues to get rid of old messages.
        # They can't be acked anyway, as a delivery tag is specific
        # to the current channel.
        if self.controller and self.controller.semaphore:
            self.controller.semaphore.clear()
        for bucket in self.task_buckets.values():
            if bucket:
                bucket.clear_pending()
        for r in tuple(reserved_requests):
            if r not in active_requests:
                requests.pop(r.id, None)
        reserved_requests.clear()
        reserved_requests.update(tuple(active_requests))
        # Scheduled (ETA/countdown) requests never became reserved, so they
        # aren't covered by the cleanup above. Cancel their pending timer
        # entries (through self.timer.cancel(), not entry.cancel()
        # directly, since e.g. the Eventlet timer relies on that to catch
        # GreenletExit) so the callback can't fire after we've torn down
        # this connection (neither event loop cancels them: synloop's error
        # path cancels nothing, and asynloop's only cancels the entries it
        # registered for the connection, which doesn't include these), then
        # drop our copies since the broker may redeliver these tasks to
        # another worker on reconnect.
        for r in tuple(scheduled_requests):
            entry = r._eta_timer_entry
            if entry is not None and self.timer is not None:
                try:
                    self.timer.cancel(entry)
                except Exception as exc:  # pylint: disable=broad-except
                    logger.exception(
                        'Error cancelling ETA timer entry: %r', exc)
            # A request can be in both sets: with a threaded timer an ETA
            # already in the past fires immediately, so task_reserved() may
            # have run before the strategy registered the request as
            # scheduled.  Never drop one that's reserved/running.
            if r not in active_requests and r not in reserved_requests:
                requests.pop(r.id, None)
        scheduled_requests.clear()
        if self.pool and self.pool.flush:
            self.pool.flush()

    def connect(self):
        """Establish the broker connection used for consuming tasks.

        Retries establishing the connection if the
        :setting:`broker_connection_retry` setting is enabled
        """
        conn = self.connection_for_read(heartbeat=self.amqheartbeat)
        if self.hub:
            conn.transport.register_with_event_loop(conn.connection, self.hub)
        return conn

    def connection_for_read(self, heartbeat=None):
        return self.ensure_connected(
            self.app.connection_for_read(heartbeat=heartbeat))

    def connection_for_write(self, url=None, heartbeat=None):
        return self.ensure_connected(
            self.app.connection_for_write(url=url, heartbeat=heartbeat))

    def ensure_connected(self, conn):
        # Callback called for each retry while the connection
        # can't be established.
        def _error_handler(exc, interval, next_step=CONNECTION_RETRY_STEP):
            if getattr(conn, 'alt', None) and interval == 0:
                next_step = CONNECTION_FAILOVER
            elif interval > 0:
                self.broker_connection_retry_attempt += 1
            next_step = next_step.format(
                when=humanize_seconds(interval, 'in', ' '),
                retries=self.broker_connection_retry_attempt,
                max_retries=self.app.conf.broker_connection_max_retries)
            error(CONNECTION_ERROR, conn.as_uri(), exc, next_step)

        # Remember that the connection is lazy, it won't establish
        # until needed.

        # TODO: Rely only on broker_connection_retry_on_startup to determine whether connection retries are disabled.
        #       We will make the switch in Celery 6.0.

        retry_disabled = False

        if self.app.conf.broker_connection_retry_on_startup is None:
            # If broker_connection_retry_on_startup is not set, revert to broker_connection_retry
            # to determine whether connection retries are disabled.
            retry_disabled = not self.app.conf.broker_connection_retry

            if retry_disabled:
                warnings.warn(
                    CPendingDeprecationWarning(
                        "The broker_connection_retry configuration setting will no longer determine\n"
                        "whether broker connection retries are made during startup in Celery 6.0 and above.\n"
                        "If you wish to refrain from retrying connections on startup,\n"
                        "you should set broker_connection_retry_on_startup to False instead."),
                    stacklevel=2,
                )
        else:
            if self.first_connection_attempt:
                retry_disabled = not self.app.conf.broker_connection_retry_on_startup
            else:
                retry_disabled = not self.app.conf.broker_connection_retry

        if retry_disabled:
            # Retry disabled, just call connect directly.
            conn.connect()
            self.first_connection_attempt = False
            return conn

        conn = conn.ensure_connection(
            _error_handler, self.app.conf.broker_connection_max_retries,
            callback=maybe_shutdown,
        )
        self.first_connection_attempt = False
        self.broker_connection_retry_attempt = 0
        return conn

    def _flush_events(self):
        if self.event_dispatcher:
            self.event_dispatcher.flush()

    def on_send_event_buffered(self):
        if self.hub:
            self.hub.call_soon(self._flush_events_promise)

    def add_task_queue(self, queue, exchange=None, exchange_type=None,
                       routing_key=None, **options):
        cset = self.task_consumer
        queues = self.app.amqp.queues
        # Must use in' here, as __missing__ will automatically
        # create queues when :setting:`task_create_missing_queues` is enabled.
        # (Issue #1079)
        if queue in queues:
            q = queues.select_add(queues[queue])
        else:
            exchange = queue if exchange is None else exchange
            exchange_type = ('direct' if exchange_type is None
                             else exchange_type)
            q = queues.select_add(queue,
                                  exchange=exchange,
                                  exchange_type=exchange_type,
                                  routing_key=routing_key, **options)
        if not cset.consuming_from(queue):
            cset.add_queue(q)
            cset.consume()
            info('Started consuming from %s', queue)

    def cancel_task_queue(self, queue):
        info('Canceling queue %s', queue)
        queues = self.app.amqp.queues
        queues.deselect(queue)
        self.task_consumer.cancel_by_queue(queues.aliases.get(queue, queue))

    def apply_eta_task(self, task):
        """Method called by the timer to apply a task with an ETA/countdown."""
        task_reserved(task)
        self.on_task_request(task)
        self.qos.decrement_eventually()

    def _message_report(self, body, message):
        return MESSAGE_REPORT.format(dump_body(message, body),
                                     safe_repr(message.content_type),
                                     safe_repr(message.content_encoding),
                                     safe_repr(message.delivery_info),
                                     safe_repr(message.headers))

    def on_unknown_message(self, body, message):
        warn(UNKNOWN_FORMAT, self._message_report(body, message))
        message.reject_log_error(logger, self.connection_errors)
        signals.task_rejected.send(sender=self, message=message, exc=None)

    def on_unknown_task(self, body, message, exc):
        error(UNKNOWN_TASK_ERROR,
              exc,
              dump_body(message, body),
              message.headers,
              message.delivery_info,
              exc_info=True)
        try:
            id_, name = message.headers['id'], message.headers['task']
            root_id = message.headers.get('root_id')
        except KeyError:  # proto1
            payload = message.payload
            id_, name = payload['id'], payload['task']
            root_id = None
        request = Bunch(
            name=name, chord=None, root_id=root_id,
            correlation_id=message.properties.get('correlation_id'),
            reply_to=message.properties.get('reply_to'),
            errbacks=None,
        )
        message.reject_log_error(logger, self.connection_errors)
        self.app.backend.mark_as_failure(
            id_, NotRegistered(name), request=request,
        )
        if self.event_dispatcher:
            self.event_dispatcher.send(
                'task-failed', uuid=id_,
                exception=f'NotRegistered({name!r})',
            )
        signals.task_unknown.send(
            sender=self, message=message, exc=exc, name=name, id=id_,
        )

    def on_invalid_task(self, body, message, exc):
        error(INVALID_TASK_ERROR, exc, dump_body(message, body),
              exc_info=True)
        message.reject_log_error(logger, self.connection_errors)
        signals.task_rejected.send(sender=self, message=message, exc=exc)

    def update_strategies(self):
        loader = self.app.loader
        for name, task in self.app.tasks.items():
            self.strategies[name] = task.start_strategy(self.app, self)
            task.__trace__ = build_tracer(name, task, loader, self.hostname,
                                          app=self.app)

    def create_task_handler(self, promise=promise):
        strategies = self.strategies
        on_unknown_message = self.on_unknown_message
        on_unknown_task = self.on_unknown_task
        on_invalid_task = self.on_invalid_task
        callbacks = self.on_task_message
        call_soon_ack = self.call_soon_ack

        def on_task_received(message):
            # payload will only be set for v1 protocol, since v2
            # will defer deserializing the message body to the pool.
            payload = None
            try:
                type_ = message.headers['task']  # protocol v2
            except TypeError:
                return on_unknown_message(None, message)
            except KeyError:
                try:
                    payload = message.decode()
                except Exception as exc:  # pylint: disable=broad-except
                    return self.on_decode_error(message, exc)
                try:
                    type_, payload = payload['task'], payload  # protocol v1
                except (TypeError, KeyError):
                    return on_unknown_message(payload, message)
            try:
                strategy = strategies[type_]
            except KeyError as exc:
                return on_unknown_task(None, message, exc)
            else:
                try:
                    ack_log_error_promise = promise(
                        call_soon_ack,
                        (message.ack_log_error,),
                        on_error=self._restore_prefetch_count_after_connection_restart,
                    )
                    reject_log_error_promise = promise(
                        call_soon_ack,
                        (message.reject_log_error,),
                        on_error=self._restore_prefetch_count_after_connection_restart,
                    )

                    if (
                        not self._maximum_prefetch_restored
                        and self.restart_count > 0
                        and self._new_prefetch_count <= self.max_prefetch_count
                    ):
                        ack_log_error_promise.then(self._restore_prefetch_count_after_connection_restart,
                                                   on_error=self._restore_prefetch_count_after_connection_restart)
                        reject_log_error_promise.then(self._restore_prefetch_count_after_connection_restart,
                                                      on_error=self._restore_prefetch_count_after_connection_restart)

                    strategy(
                        message, payload,
                        ack_log_error_promise,
                        reject_log_error_promise,
                        callbacks,
                    )
                except (InvalidTaskError, ContentDisallowed) as exc:
                    return on_invalid_task(payload, message, exc)
                except DecodeError as exc:
                    return self.on_decode_error(message, exc)

        return on_task_received

    def _restore_prefetch_count_after_connection_restart(self, p, *args):
        with self.qos._mutex:
            if any((
                not self.app.conf.worker_enable_prefetch_count_reduction,
                self._maximum_prefetch_restored,
            )):
                return

            new_prefetch_count = min(self.max_prefetch_count, self._new_prefetch_count)
            self.qos.value = self.initial_prefetch_count = new_prefetch_count
            self.qos.set(self.qos.value)

            already_restored = self._maximum_prefetch_restored
            self._maximum_prefetch_restored = new_prefetch_count == self.max_prefetch_count

            if already_restored is False and self._maximum_prefetch_restored is True:
                logger.info(
                    "Resuming normal operations following a restart.\n"
                    f"Prefetch count has been restored to the maximum of {self.max_prefetch_count}"
                )

    @property
    def max_prefetch_count(self):
        return self.pool.num_processes * self.prefetch_multiplier

    @property
    def _new_prefetch_count(self):
        return self.qos.value + self.prefetch_multiplier

    def __repr__(self):
        """``repr(self)``."""
        return '<Consumer: {self.hostname} ({state})>'.format(
            self=self, state=self.blueprint.human_state(),
        )

    def cancel_active_requests(self):
        """Cancel active requests during shutdown.

        Cancels all active requests that either do not require late acknowledgments or,
        if they do, have not been acknowledged yet.

        Does not cancel successful tasks, even if they have not been acknowledged yet.
        """

        def should_cancel(request):
            if not request.task.acks_late:
                # Task does not require late acknowledgment, cancel it.
                return True

            if not request.acknowledged:
                # Task is late acknowledged, but it has not been acknowledged yet, cancel it.
                if request.id in successful_requests:
                    # Unless it was successful, in which case we don't want to cancel it.
                    return False
                return True

            # Task is late acknowledged, but it has already been acknowledged.
            return False  # Do not cancel and allow it to gracefully finish as it has already been acknowledged.

        requests_to_cancel = tuple(filter(should_cancel, active_requests))

        if requests_to_cancel:
            for request in requests_to_cancel:
                # For acks_late tasks, don't emit RETRY signal since broker will handle redelivery
                # For non-acks_late tasks, emit RETRY signal as usual
                emit_retry = not request.task.acks_late
                request.cancel(self.pool, emit_retry=emit_retry)


class Evloop(bootsteps.StartStopStep):
    """Event loop service.

    Note:
        This is always started last.
    """

    label = 'event loop'
    last = True

    def start(self, c):
        self.patch_all(c)
        c.loop(*c.loop_args())

    def patch_all(self, c):
        c.qos._mutex = DummyLock()
