Fix portia_server

portia_server/portia_api/jsonapi/serializers.py

@@ -109,7 +109,8 @@ def __new__(mcs, name, bases, attrs):
 
 
 class JsonApiSerializerOpts(SchemaOpts):
-    def __init__(self, meta):
+    def __init__(self, meta, ordered):
+        # ordered is not used, but needed for marshmallow
         super(JsonApiSerializerOpts, self).__init__(meta)
         if meta is BaseSchema.Meta:
             return
@@ -210,7 +211,7 @@ def __init__(self, instance=None, data=None, storage=None, only=(),
 
     @property
     def data(self):
-        return self.dump(self.instance).data
+        return self.dump(self.instance)
 
     @cached_property
     def errors(self):
@@ -535,9 +536,7 @@ def format_item(self, item):
         return order_dict(item, RESOURCE_OBJECT_ORDER)
 
     def get_top_level_links(self, data, many):
-        if self.current_url:
-            return OrderedDict([('self', self.current_url)])
-        return None
+        return OrderedDict([('self', self.current_url)])
 
     def get_resource_links(self, item):
         url = item.get('_url')

portia_server/portia_api/jsonapi/utils.py

@@ -55,7 +55,9 @@ def dasherize(value):
 
 
 def type_from_model_name(value):
-    return '{}s'.format(camel_case_to_dashes(value))
+    type_ = camel_case_to_dashes(value) # singular
+    #type_ = type_ + "s" # plural
+    return type_
 
 
 def deep_getattr(obj, key):

portia_server/portia_api/resources/models.py

@@ -81,11 +81,11 @@ class ProjectSchema(SlydSchema):
     )
     project = fields.Relationship(
         self_url='/api/projects/{project_id}',
-        self_url_kwargs={'project_id': '<id>'}, type_='projects'
+        self_url_kwargs={'project_id': '<id>'}, type_='project'
     )
 
     class Meta:
-        type_ = 'projects'
+        type_ = 'project'
 
 
 class SchemaSchema(SlydSchema):
@@ -95,7 +95,7 @@ class SchemaSchema(SlydSchema):
     project = fields.Relationship(
         related_url='/api/projects/{project_id}',
         related_url_kwargs={'project_id': '<project_id>'},
-        type_='projects',
+        type_='project',
         include_resource_linkage=True
     )
     fields = fields.Relationship(
@@ -119,7 +119,7 @@ class FieldSchema(SlydSchema):
     project = fields.Relationship(
         related_url='/api/projects/{project_id}',
         related_url_kwargs={'project_id': '<project_id>'},
-        type_='projects',
+        type_='project',
         include_resource_linkage=True
     )
     schema = fields.Relationship(
@@ -159,7 +159,7 @@ class SpiderSchema(SlydSchema):
     project = fields.Relationship(
         related_url='/api/projects/{project_id}',
         related_url_kwargs={'project_id': '<project_id>'},
-        type_='projects',
+        type_='project',
         include_resource_linkage=True
     )
 
@@ -202,7 +202,7 @@ class SampleSchema(SlydSchema):
     project = fields.Relationship(
         related_url='/api/projects/{project_id}',
         related_url_kwargs={'project_id': '<project_id>'},
-        type_='projects', include_resource_linkage=True
+        type_='project', include_resource_linkage=True
     )
     spider = fields.Relationship(
         related_url='/api/projects/{project_id}/spiders/{spider_id}',
@@ -351,7 +351,7 @@ class ExtractorSchema(SlydSchema):
     project = fields.Relationship(
         related_url='/api/projects/{project_id}',
         related_url_kwargs={'project_id': '<project_id>'},
-        type_='projects',
+        type_='project',
         include_resource_linkage=True
     )
 

portia_server/portia_api/resources/projects.py

@@ -3,7 +3,7 @@
 from django.conf import settings
 from django.utils.functional import cached_property
 from dulwich.objects import Commit
-from rest_framework.decorators import detail_route
+from rest_framework.decorators import action
 from rest_framework.response import Response
 from rest_framework.status import HTTP_200_OK, HTTP_201_CREATED
 from six import iteritems
@@ -25,7 +25,7 @@
 
 
 class ProjectDownloadMixin(object):
-    @detail_route(methods=['get'])
+    @action(detail=True, methods=['get'])
     def download(self, *args, **kwargs):
         fmt = self.query.get('format', 'spec')
         version = self.query.get('version', None)
@@ -134,7 +134,7 @@ def create(self, request):
     # def destroy(self):
     #     """Delete the requested project"""
 
-    @detail_route(methods=['get'])
+    @action(detail=True, methods=['get'])
     def status(self, *args, **kwargs):
         response = self.retrieve()
         data = OrderedDict()
@@ -146,7 +146,7 @@ def status(self, *args, **kwargs):
         data.update(response.data)
         return Response(data, status=HTTP_200_OK)
 
-    @detail_route(methods=['put', 'patch', 'post'])
+    @action(detail=True, methods=['put', 'patch', 'post'])
     def publish(self, *args, **kwargs):
         if not self.storage.version_control and hasattr(self.storage, 'repo'):
             raise JsonApiFeatureNotAvailableError()
@@ -167,12 +167,12 @@ def publish(self, *args, **kwargs):
         response = self.retrieve()
         return Response(response.data, status=HTTP_200_OK)
 
-    @detail_route(methods=['POST'])
+    @action(detail=True, methods=['POST'])
     def deploy(self, *args, **kwargs):
         data = self._deploy()
         return Response(data, HTTP_200_OK)
 
-    @detail_route(methods=['put', 'patch', 'post'])
+    @action(detail=True, methods=['put', 'patch', 'post'])
     def reset(self, *args, **kwargs):
         if not self.storage.version_control and hasattr(self.storage, 'repo'):
             raise JsonApiFeatureNotAvailableError()
@@ -181,7 +181,7 @@ def reset(self, *args, **kwargs):
         self.storage.repo.refs['refs/heads/%s' % branch] = master
         return self.retrieve()
 
-    @detail_route(methods=['post'])
+    @action(detail=True, methods=['post'])
     def copy(self, *args, **kwargs):
         from_project_id = self.query.get('from') or self.data.get('from')
         if not from_project_id:
@@ -205,7 +205,7 @@ def copy(self, *args, **kwargs):
         response = self.retrieve()
         return Response(response.data, status=HTTP_201_CREATED)
 
-    @detail_route(methods=['post'])
+    @action(detail=True, methods=['post'])
     def rollback(self, *args, **kwargs):
         if not self.storage.version_control and hasattr(self.storage, 'repo'):
             raise JsonApiFeatureNotAvailableError()

portia_server/portia_api/resources/spiders.py

@@ -1,6 +1,6 @@
 from django.http.response import Http404
 
-from rest_framework.decorators import detail_route
+from rest_framework.decorators import action
 from rest_framework.response import Response
 from rest_framework.status import HTTP_200_OK, HTTP_400_BAD_REQUEST
 
@@ -26,7 +26,7 @@ def get_instance(self):
     def get_collection(self):
         return self.project.spiders
 
-    @detail_route(methods=['post'])
+    @action(detail=True, methods=['post'])
     def extract(self, *args, **kwargs):
         try:
             instance = self.get_instance()
@@ -49,7 +49,7 @@ def extract(self, *args, **kwargs):
     def _build_pages(self, spider):
         return Pages(self.data, spider)
 
-    @detail_route(methods=['post'])
+    @action(detail=True, methods=['post'])
     def rename(self, *args, **kwargs):
         try:
             spider = self.get_instance()
@@ -74,7 +74,7 @@ def rename(self, *args, **kwargs):
         data = self.get_serializer(spider).data
         return Response(data, status=HTTP_200_OK)
 
-    @detail_route(methods=['post'])
+    @action(detail=True, methods=['post'])
     def schedule(self, *args, **kwargs):
         spider_id = self.data['data']['id']
         data = Deployer(self.project).schedule(spider_id)

portia_server/portia_api/urls.py

@@ -11,22 +11,22 @@
 from .resources.spiders import SpiderRoute
 
 router = Router()
-router.register(r'projects', ProjectRoute, base_name='projects')
+router.register(r'projects', ProjectRoute, basename='projects')
 
 project_router = NestedRouter(router, r'projects')
-project_router.register(r'schemas', SchemaRoute, base_name='schemas')
-project_router.register(r'spiders', SpiderRoute, base_name='spiders')
-project_router.register(r'extractors', ExtractorRoute, base_name='extractors')
+project_router.register(r'schemas', SchemaRoute, basename='schemas')
+project_router.register(r'spiders', SpiderRoute, basename='spiders')
+project_router.register(r'extractors', ExtractorRoute, basename='extractors')
 
 schema_router = NestedRouter(project_router, r'schemas')
-schema_router.register(r'fields', FieldRoute, base_name='fields')
+schema_router.register(r'fields', FieldRoute, basename='fields')
 
 spider_router = NestedRouter(project_router, r'spiders')
-spider_router.register(r'samples', SampleRoute, base_name='samples')
+spider_router.register(r'samples', SampleRoute, basename='samples')
 
 sample_router = NestedRouter(spider_router, r'samples')
-sample_router.register(r'items', ItemRoute, base_name='items')
-sample_router.register(r'annotations', AnnotationRoute, base_name='annotations')
+sample_router.register(r'items', ItemRoute, basename='items')
+sample_router.register(r'annotations', AnnotationRoute, basename='annotations')
 
 urlpatterns = [
     url(r'^', include(router.urls)),

portia_server/portia_orm/base.py

@@ -558,6 +558,8 @@ def load(cls, storage, instance=None, **kwargs):
                 return cls.collection()
             return instance  # may be None
 
+        # TODO logging.debug
+        print(f"loading file: projects/{storage.name}/{path}")
         file_data = storage.open(path).read()
         if not cls.opts.raw:
             try:

portia_server/portia_orm/fields.py

@@ -85,13 +85,15 @@ def get_dependencies(self, cls):
 
     def serialize(self, attr, obj, accessor=None):
         if self._CHECK_ATTRIBUTE:
-            value = self.get_value(attr, obj, accessor=accessor)
+            value = accessor(obj, attr, None)
             self._validate_missing(value)
             self._validate(value)
         return super(Field, self).serialize(attr, obj, accessor)
 
 
-class ValidatedField(fields.ValidatedField, Field):
+# marshmallow.fields.ValidatedField was removed in https://github.com/marshmallow-code/marshmallow/pull/712
+#class ValidatedField(fields.ValidatedField, Field):
+class ValidatedField(Field):
     default_error_messages = {
         'invalid': u"Invalid value.",
     }

portia_server/portia_orm/models.py

@@ -237,7 +237,7 @@ class Spider(Model):
     project = BelongsTo(Project, related_name='spiders', on_delete=CASCADE,
                         ignore_in_file=True)
     samples = HasMany('Sample', related_name='spider', on_delete=CLEAR,
-                      only='id')
+                      only=('id',))
 
     class Meta:
         path = u'spiders/{self.id}.json'
@@ -383,7 +383,7 @@ class Sample(Model, OrderedAnnotationsMixin):
     original_body = HasOne('OriginalBody', related_name='sample',
                            on_delete=CLEAR, ignore_in_file=True)
     spider = BelongsTo(Spider, related_name='samples', on_delete=CASCADE,
-                       only='id')
+                       only=('id',))
 
     class Meta:
         path = u'spiders/{self.spider.id}/{self.id}.json'
@@ -602,7 +602,7 @@ def clean(self, data):
 class BaseAnnotation(Model):
     id = String(primary_key=True)
     parent = BelongsTo('Item', related_name='annotations', on_delete=CASCADE,
-                       only='id')
+                       only=('id',))
 
     class Meta:
         polymorphic = True

portia_server/portia_orm/serializers.py

@@ -11,15 +11,15 @@
 
 
 class FileSerializerOpts(schema.SchemaOpts):
-    def __init__(self, meta):
+    def __init__(self, meta, ordered=True):
         super(FileSerializerOpts, self).__init__(meta)
         if meta is schema.BaseSchema.Meta:
             return
 
         self.strict = True
         # make marshmallow use OrderedDicts, so that collections of enveloped
         # objects maintain their order when loaded
-        self.ordered = True
+        self.ordered = ordered
         # the model from which the Schema was created, required
         self.model = getattr(meta, 'model')
         self.polymorphic = getattr(meta, 'polymorphic', False)
@@ -71,7 +71,7 @@ def order_keys(self, data):
         """
         return OrderedDict((item for item in sorted(iteritems(data))))
 
-    def _do_load(self, data, many=None, *args, **kwargs):
+    def _do_load(self, data, many=None, **kwargs):
         # support the case where we have only a single field to load and we get
         # it directly rather than wrapped in a dict. this happens when loading
         # a relationship with a single field in 'only'
@@ -81,15 +81,15 @@ def _do_load(self, data, many=None, *args, **kwargs):
         elif isinstance(data, Sequence):
             data = [self._wrap_only(value) for value in data]
 
-        result, errors = super(FileSerializer, self)._do_load(
-            data, many, *args, **kwargs)
+        result = super(FileSerializer, self)._do_load(
+            data, many=many, **kwargs)
 
         # we need to wrap the result of a many load in a ModelCollection, but
         # post_load(pass_many=True) processors are called before the Model
         # instances are created in the post_load(pass_many=False) processor
         if many:
             result = self.opts.model.collection(result)
-        return result, errors
+        return result
 
     def _wrap_only(self, data):
         if self.only and len(self.only) == 1 and not isinstance(data, dict):

portia_server/portia_orm/tests/models.py

@@ -44,7 +44,7 @@ class OneToOneModel1(Model):
     id = fields.String(primary_key=True)
     field = fields.Field()
     m2 = fields.BelongsTo('OneToOneModel2', related_name='m1',
-                          on_delete=fields.CLEAR, only='id')
+                          on_delete=fields.CLEAR, only=('id',))
 
     class Meta:
         path = u'o2o-model-1.json'
@@ -64,7 +64,7 @@ class ChildModel(Model):
     id = fields.String(primary_key=True)
     field = fields.Field()
     parent = fields.BelongsTo('ParentModel', related_name='children',
-                              on_delete=fields.CLEAR, only='id')
+                              on_delete=fields.CLEAR, only=('id',))
 
     class Meta:
         path = u'{self.parent.id}/children.json'
@@ -85,7 +85,7 @@ class ManyToManyModel1(Model):
     id = fields.String(primary_key=True)
     field = fields.Field()
     m2 = fields.HasMany('ManyToManyModel2', related_name='m1',
-                        on_delete=fields.CLEAR, only='id')
+                        on_delete=fields.CLEAR, only=('id',))
 
     class Meta:
         path = u'm2m-model-1.json'
@@ -107,7 +107,7 @@ class PolymorphicParentModel(Model):
     field = fields.Field()
     children = fields.HasMany('PolymorphicChildBase', related_name='parent',
                               polymorphic=True, on_delete=fields.CASCADE,
-                              only='id')
+                              only=('id',))
 
     class Meta:
         path = u'parents.json'
@@ -116,7 +116,7 @@ class Meta:
 class PolymorphicChildBase(Model):
     id = fields.String(primary_key=True)
     parent = fields.BelongsTo(PolymorphicParentModel, related_name='children',
-                              on_delete=fields.CLEAR, only='id')
+                              on_delete=fields.CLEAR, only=('id',))
 
     class Meta:
         path = u'children.json'