Skip to content

Commit

Permalink
Move priority setting to schedule.json webservice
Browse files Browse the repository at this point in the history
Priorities are of interest to schedulers and queues, not processes.
  • Loading branch information
Digenis committed Jun 21, 2019
1 parent 0ceed69 commit f80e0b0
Show file tree
Hide file tree
Showing 4 changed files with 6 additions and 5 deletions.
1 change: 1 addition & 0 deletions docs/api.rst
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ Schedule a spider run (also known as a job), returning the job id.
* ``spider`` (string, required) - the spider name
* ``setting`` (string, optional) - a Scrapy setting to use when running the spider
* ``jobid`` (string, optional) - a job id used to identify the job, overrides the default generated UUID
* ``priority`` (float, optional) - priority for this project's spider queue — 0 by default
* ``_version`` (string, optional) - the version of the project to use
* any other parameter is passed as spider argument

Expand Down
4 changes: 2 additions & 2 deletions scrapyd/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ def __init__(self, config):
self.config = config
self.update_projects()

def schedule(self, project, spider_name, **spider_args):
def schedule(self, project, spider_name, priority, **spider_args):
q = self.queues[project]
q.add(spider_name, **spider_args)
q.add(spider_name, priority, **spider_args)

def list_projects(self):
return self.queues.keys()
Expand Down
3 changes: 1 addition & 2 deletions scrapyd/spiderqueue.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,9 @@ class SqliteSpiderQueue(object):
def __init__(self, database=None, table='spider_queue'):
self.q = JsonSqlitePriorityQueue(database, table)

def add(self, name, **spider_args):
def add(self, name, priority, **spider_args):
d = spider_args.copy()
d['name'] = name
priority = float(d.pop('priority', 0))
self.q.put(d, priority)

def pop(self):
Expand Down
3 changes: 2 additions & 1 deletion scrapyd/webservice.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,14 @@ def render_POST(self, txrequest):
project = args.pop('project')
spider = args.pop('spider')
version = args.get('_version', '')
priority = float(args.pop('priority', 0))
spiders = get_spider_list(project, version=version)
if not spider in spiders:
return {"status": "error", "message": "spider '%s' not found" % spider}
args['settings'] = settings
jobid = args.pop('jobid', uuid.uuid1().hex)
args['_job'] = jobid
self.root.scheduler.schedule(project, spider, **args)
self.root.scheduler.schedule(project, spider, priority, **args)
return {"node_name": self.root.nodename, "status": "ok", "jobid": jobid}

class Cancel(WsResource):
Expand Down

0 comments on commit f80e0b0

Please sign in to comment.