grab.spider.queue_backend.redis

Spider task queue backend powered by redis.

Module Contents

Classes

CustomPriorityQueue
RedisTaskQueue

Attributes

system_random

grab.spider.queue_backend.redis.system_random

class grab.spider.queue_backend.redis.CustomPriorityQueue(key: str, **kwargs: Any)

Bases: fastrq.priorityqueue.PriorityQueue

connect() → redis.Redis[Any]

clear() → None

class grab.spider.queue_backend.redis.RedisTaskQueue(queue_name: None | str = None, connection_args: None | dict[str, Any] = None)

Bases: grab.spider.queue_backend.base.BaseTaskQueue

put(task: grab.spider.task.Task, priority: int, schedule_time: None | datetime.datetime = None) → None

get() → grab.spider.task.Task

Return Task object or raise Queue.Empty exception.

@returns: grab.spider.task.Task object @raises: Queue.Empty exception

size() → int

clear() → None

Remove all tasks from the queue.

close() → None