Skip to content

Commit

Permalink
Merge branch 'harvest-object-create'
Browse files Browse the repository at this point in the history
  • Loading branch information
amercader committed Oct 4, 2013
2 parents 71aedf3 + da2fd45 commit c5f4d68
Show file tree
Hide file tree
Showing 7 changed files with 158 additions and 4 deletions.
44 changes: 41 additions & 3 deletions ckanext/harvest/logic/action/create.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,27 @@
import logging

import ckan
from ckan import logic

from ckan.logic import NotFound, check_access
from ckanext.harvest.logic import HarvestJobExists

from ckanext.harvest.plugin import DATASET_TYPE_NAME
from ckanext.harvest.model import (HarvestSource, HarvestJob)
from ckanext.harvest.logic.dictization import harvest_job_dictize
from ckanext.harvest.logic.schema import harvest_source_show_package_schema
from ckanext.harvest.model import (HarvestSource, HarvestJob, HarvestObject,
HarvestObjectExtra)
from ckanext.harvest.logic.dictization import (harvest_job_dictize,
harvest_object_dictize)
from ckanext.harvest.logic.schema import (harvest_source_show_package_schema,
harvest_object_create_schema)
from ckanext.harvest.logic.action.get import harvest_source_list,harvest_job_list

log = logging.getLogger(__name__)

_validate = ckan.lib.navl.dictization_functions.validate

class InactiveSource(Exception):
pass

def harvest_source_create(context,data_dict):
'''
Creates a new harvest source
Expand Down Expand Up @@ -136,3 +145,32 @@ def _check_for_existing_jobs(context, source_id):
exist = len(exist_new + exist_running) > 0

return exist

def harvest_object_create(context, data_dict):
""" Create a new harvest object
:type guid: string (optional)
:type content: string (optional)
:type job_id: string
:type source_id: string (optional)
:type package_id: string (optional)
:type extras: dict (optional)
"""
check_access('harvest_object_create', context, data_dict)
data, errors = _validate(data_dict, harvest_object_create_schema(), context)

if errors:
raise logic.ValidationError(errors)

obj = HarvestObject(
guid=data.get('guid'),
content=data.get('content'),
job=data['job_id'],
harvest_source_id=data.get('source_id'),
package_id=data.get('package_id'),
extras=[ HarvestObjectExtra(key=k, value=v)
for k, v in data.get('extras', {}).items() ]
)

obj.save()
return harvest_object_dictize(obj, context)
10 changes: 10 additions & 0 deletions ckanext/harvest/logic/auth/create.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,13 @@ def harvest_job_create_all(context, data_dict):
return {'success': False, 'msg': pt._('Only sysadmins can create harvest jobs for all sources')}
else:
return {'success': True}

def harvest_object_create(context, data_dict):
"""
Auth check for creating a harvest object
only the sysadmins can create harvest objects
"""
# sysadmins can run all actions if we've got to this point we're not a sysadmin
return {'success': False, 'msg': pt._('Only the sysadmins can create harvest objects')}

16 changes: 16 additions & 0 deletions ckanext/harvest/logic/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@
harvest_source_frequency_exists,
dataset_type_exists,
harvest_source_convert_from_config,
harvest_source_id_exists,
harvest_job_exists,
harvest_object_extras_validator,
)

def harvest_source_schema():
Expand Down Expand Up @@ -74,3 +77,16 @@ def harvest_source_show_package_schema():
})

return schema

def harvest_object_create_schema():
schema = {
'guid': [ignore_missing, unicode],
'content': [ignore_missing, unicode],
'state': [ignore_missing, unicode],
'job_id': [harvest_job_exists],
'source_id': [ignore_missing, harvest_source_id_exists],
'package_id': [ignore_missing, package_id_exists],
'extras': [ignore_missing, harvest_object_extras_validator],
}
return schema

18 changes: 17 additions & 1 deletion ckanext/harvest/logic/validators.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from ckan.plugins import PluginImplementations

from ckanext.harvest.plugin import DATASET_TYPE_NAME
from ckanext.harvest.model import HarvestSource, UPDATE_FREQUENCIES
from ckanext.harvest.model import HarvestSource, UPDATE_FREQUENCIES, HarvestJob
from ckanext.harvest.interfaces import IHarvester

from ckan.lib.navl.validators import keep_extras
Expand All @@ -22,6 +22,14 @@ def harvest_source_id_exists(value, context):
raise Invalid('Harvest Source with id %r does not exist.' % str(value))
return value

def harvest_job_exists(value, context):
"""Check if a harvest job exists and returns the model if it does"""
result = HarvestJob.get(value, None)

if not result:
raise Invalid('Harvest Job with id %r does not exist.' % str(value))
return result

def _normalize_url(url):
o = urlparse.urlparse(url)

Expand Down Expand Up @@ -196,3 +204,11 @@ def dataset_type_exists(value):
if value != DATASET_TYPE_NAME:
value = DATASET_TYPE_NAME
return value

def harvest_object_extras_validator(value, context):
if not isinstance(value, dict):
raise Invalid('extras must be a dict')
for v in value.values():
if not isinstance(v, basestring):
raise Invalid('extras must be a dict of strings')
return value
14 changes: 14 additions & 0 deletions ckanext/harvest/tests/factories.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import factory
from ckanext.harvest.model import HarvestSource, HarvestJob

class HarvestSourceFactory(factory.Factory):
FACTORY_FOR = HarvestSource

url = "http://harvest.test.com"
type = "test-harvest-source"

class HarvestJobFactory(factory.Factory):
FACTORY_FOR = HarvestJob

status = "New"
source = factory.SubFactory(HarvestSourceFactory)
59 changes: 59 additions & 0 deletions ckanext/harvest/tests/test_action.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,16 @@
import ckan
import paste
import pylons.test
import factories
import unittest

from ckan import tests
from ckan import plugins as p
from ckan.plugins import toolkit
from ckanext.harvest.interfaces import IHarvester
import ckanext.harvest.model as harvest_model


class MockHarvesterForActionTests(p.SingletonPlugin):
p.implements(IHarvester)
def info(self):
Expand Down Expand Up @@ -203,3 +207,58 @@ def test_update(self):
assert source.url == source_dict['url']
assert source.type == source_dict['source_type']

class TestHarvestObject(unittest.TestCase):
@classmethod
def setup_class(cls):
harvest_model.setup()

@classmethod
def teardown_class(cls):
ckan.model.repo.rebuild_db()

def test_create(self):
job = factories.HarvestJobFactory()
job.save()

context = {
'model' : ckan.model,
'session': ckan.model.Session,
'ignore_auth': True,
}
data_dict = {
'guid' : 'guid',
'content' : 'content',
'job_id' : job.id,
'extras' : { 'a key' : 'a value' },
}
harvest_object = toolkit.get_action('harvest_object_create')(
context, data_dict)

# fetch the object from database to check it was created
created_object = harvest_model.HarvestObject.get(harvest_object['id'])
assert created_object.guid == harvest_object['guid'] == data_dict['guid']

def test_create_bad_parameters(self):
source_a = factories.HarvestSourceFactory()
source_a.save()
job = factories.HarvestJobFactory()
job.save()

context = {
'model' : ckan.model,
'session': ckan.model.Session,
'ignore_auth': True,
}
data_dict = {
'job_id' : job.id,
'source_id' : source_a.id,
'extras' : 1
}
harvest_object_create = toolkit.get_action('harvest_object_create')
self.assertRaises(ckan.logic.ValidationError, harvest_object_create,
context, data_dict)

data_dict['extras'] = {'test': 1 }

self.assertRaises(ckan.logic.ValidationError, harvest_object_create,
context, data_dict)
1 change: 1 addition & 0 deletions pip-requirements.txt
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
pika==0.9.8
factory-boy>=2

0 comments on commit c5f4d68

Please sign in to comment.