Allow subscription to changes of object matching a Query

adminapi/dataset/__init__.py

@@ -1,6 +1,7 @@
 from distutils.util import strtobool
 from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network
 from itertools import chain
+from time import sleep
 from types import GeneratorType
 
 from adminapi.datatype import validate_value, json_to_datatype
@@ -11,14 +12,15 @@
 COMMIT_ENDPOINT = '/dataset/commit'
 QUERY_ENDPOINT = '/dataset/query'
 CREATE_ENDPOINT = '/dataset/create'
+GET_COMMITS_ENDPOINT = '/dataset/get_commits'
 
 
 class DatasetError(Exception):
  pass
 
 
 class BaseQuery(object):
- def __init__(self, filters=None, restrict=None, order_by=None):
+ def __init__(self, filters=None, restrict=[], order_by=None):
  if filters is None:
  self._filters = None
  self._results = []
@@ -52,7 +54,7 @@ def __repr__(self):
 
  def _get_results(self):
  if self._results is None:
- self._results = list(self._fetch_results())
+ self._results = self._fetch_results()
  return self._results
 
  def _fetch_results(self):
@@ -194,6 +196,17 @@ def get_free_ip_addrs(self):
  if host not in used_hosts:
  yield host
 
+ def subscribe_changes(self, callback, newer_than_mins=None, slot=None):
+ self._results = self._fetch_results()
+ while True:
+ for commit in self._fetch_commits(newer_than_mins, slot):
+ callback(commit)
+ self._save_changes_slot(commit)
+ sleep(1)
+
+ def _apply_commit(self, commit):
+ pass
+
 
 class Query(BaseQuery):
 
@@ -217,14 +230,31 @@ def commit(self):
  def _fetch_results(self):
  request_data = {'filters': self._filters}
  if self._restrict is not None:
- request_data['restrict'] = self._restrict
+ request_data['restrict'] = list(self._restrict) + ['object_id']
  if self._order_by is not None:
  request_data['order_by'] = self._order_by
 
  response = send_request(QUERY_ENDPOINT, post_params=request_data)
  if response['status'] == 'error':
  _handle_exception(response)
- return (_format_obj(s) for s in response['result'])
+ return [_format_obj(s) for s in response['result']]
+
+ def _fetch_commits(self, newer_than_mins=None, slot=None):
+ request_data = {'filters': self._filters}
+ if self._restrict is not None:
+ request_data['restrict'] = self._restrict
+ if self._order_by is not None:
+ request_data['order_by'] = self._order_by
+ if newer_than_mins is not None:
+ request_data['newer_than_mins'] = newer_than_mins
+ if slot is not None:
+ request_data['slot'] = slot
+
+ response = send_request(GET_COMMITS_ENDPOINT, post_params=request_data)
+ for result in response['result']:
+ print(result)
+ if False:
+ yield result
 
 
 class DatasetObject(dict):
@@ -502,7 +532,7 @@ def _handle_exception(result):
 
 # XXX: Deprecated, use Query() instead
 def query(**kwargs):
- return Query(kwargs)
+ return Query(kwargs, None)
 
 
 # XXX: Deprecated, use Query().new_object() instead

serveradmin/access_control/migrations/0001_initial.py

@@ -9,21 +9,18 @@ class Migration(migrations.Migration):
 
  dependencies = [
  migrations.swappable_dependency(settings.AUTH_USER_MODEL),
- ('apps', '__first__'),
+ ('apps', '0001_initial'),
  ]
 
  operations = [
  migrations.CreateModel(
  name='AccessControlGroup',
  fields=[
- ('id', models.AutoField(primary_key=True, auto_created=True, serialize=False, verbose_name='ID')),
+ ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)),
  ('name', models.CharField(max_length=80, unique=True)),
- ('create_server_query', models.CharField(max_length=1000)),
- ('edit_server_query', models.CharField(max_length=1000)),
- ('commit_server_query', models.CharField(max_length=1000)),
- ('delete_server_query', models.CharField(max_length=1000)),
- ('applications', models.ManyToManyField(blank=True, to='apps.Application', related_name='access_control_groups')),
- ('members', models.ManyToManyField(blank=True, to=settings.AUTH_USER_MODEL, related_name='access_control_groups')),
+ ('query', models.CharField(max_length=1000)),
+ ('applications', models.ManyToManyField(related_name='access_control_groups', blank=True, to='apps.Application')),
+ ('members', models.ManyToManyField(related_name='access_control_groups', blank=True, to=settings.AUTH_USER_MODEL)),
  ],
  options={
  'db_table': 'access_control_group',

serveradmin/access_control/migrations/0002_accesscontrolgroup_attributes.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.7 on 2018-04-25 15:58
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('serverdb', '0001_initial'),
+ ('access_control', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='accesscontrolgroup',
+ name='attributes',
+ field=models.ManyToManyField(blank=True, related_name='access_control_groups', to='serverdb.Attribute'),
+ ),
+ ]

serveradmin/access_control/models.py

@@ -3,14 +3,12 @@
 
 from adminapi.parse import parse_query
 from serveradmin.apps.models import Application
+from serveradmin.serverdb.models import Attribute
 
 
 class AccessControlGroup(Model):
  name = CharField(max_length=80, unique=True)
- create_server_query = CharField(max_length=1000)
- edit_server_query = CharField(max_length=1000)
- commit_server_query = CharField(max_length=1000)
- delete_server_query = CharField(max_length=1000)
+ query = CharField(max_length=1000)
  members = ManyToManyField(
  User,
  blank=True,
@@ -23,24 +21,24 @@ class AccessControlGroup(Model):
  limit_choices_to={'disabled': False, 'superuser': False},
  related_name='access_control_groups',
  )
+ attributes = ManyToManyField(
+ Attribute,
+ blank=True,
+ limit_choices_to={'readonly': False},
+ related_name='access_control_groups',
+ )
 
  class Meta:
  db_table = 'access_control_group'
 
  def __init__(self, *args, **kwargs):
  super().__init__(*args, **kwargs)
- self._filters = {}
+ self._filters = None
 
  def __str__(self):
  return self.name
 
- def get_filters(self, action):
- if action not in self._filters:
- query = getattr(self, action + '_server_query')
- self._filters[action] = parse_query(query)
- return self._filters[action]
-
- def match_server(self, action, server):
- return all(
- f.matches(server[a]) for a, f in self.get_filters(action).items()
- )
+ def get_filters(self):
+ if self._filters is None:
+ self._filters = parse_query(self.query)
+ return self._filters

serveradmin/api/urls.py

@@ -6,6 +6,7 @@
  dataset_commit,
  dataset_new_object,
  dataset_create,
+ dataset_get_commits,
  api_call,
 )
 
@@ -15,5 +16,6 @@
  url('^dataset/commit$', dataset_commit),
  url('^dataset/new_object$', dataset_new_object),
  url('^dataset/create$', dataset_create),
+ url('^dataset/get_commits$', dataset_get_commits),
  url('^call$', api_call),
 ]

serveradmin/api/views.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timedelta
 from operator import itemgetter
 
 from django.core.exceptions import (
@@ -13,11 +14,12 @@
 from serveradmin.api import ApiError, AVAILABLE_API_FUNCTIONS
 from serveradmin.api.decorators import api_view
 from serveradmin.api.utils import build_function_description
+from serveradmin.apps.models import ApplicationSlot
+from serveradmin.changes.models import Commit
 from serveradmin.serverdb.query_committer import QueryCommitter
-from serveradmin.serverdb.query_filterer import QueryFilterer
+from serveradmin.serverdb.query_executer import execute_query
 from serveradmin.serverdb.query_materializer import (
- QueryMaterializer,
- get_default_attribute_values,
+ get_default_attribute_values
 )
 
 
@@ -71,21 +73,12 @@ def dataset_query(request, app, data):
  for attr, filter_obj in data['filters'].items():
  filters[attr] = filter_from_obj(filter_obj)
 
- # Empty list means query all attributes to the older versions of
- # the adminapi.
- if not data.get('restrict'):
- restrict = None
- else:
- restrict = data['restrict']
-
+ restrict = data.get('restrict')
  order_by = data.get('order_by')
 
- filterer = QueryFilterer(filters)
- materializer = QueryMaterializer(filterer, restrict, order_by)
-
  return {
  'status': 'success',
- 'result': list(materializer),
+ 'result': execute_query(filters, restrict, order_by),
  }
  except (FilterValueError, ValidationError) as error:
  return {
@@ -221,6 +214,43 @@ def dataset_create(request, app, data):
  }
 
 
+@api_view
+def dataset_get_commits(request, app, data):
+ if request.method != 'POST':
+ raise SuspiciousOperation('Method not allowed')
+ if not isinstance(data, dict):
+ raise SuspiciousOperation('Invalid payload')
+
+ if 'filters' not in data or not isinstance(data['filters'], dict):
+ raise SuspiciousOperation('"filters" must be a dictionary')
+
+ queryset = Commit.objects
+
+ if 'newer_than_mins' in data:
+ if not isinstance(data['newer_than_mins'], int):
+ raise SuspiciousOperation('"newer_than_mins" must be an int')
+ queryset = queryset.filter(change_on__gt=(
+ datetime.now() - timedelta(minutes=data['newer_than_mins'])
+ ))
+
+ if 'slot' in data:
+ if not isinstance(data['slot'], str):
+ raise SuspiciousOperation('"newer_than_mins" must be a string')
+ try:
+ slot = ApplicationSlot.objects.get(
+ application=app, name=data['slot']
+ )
+ except ApplicationSlot.DoesNotExist:
+ pass
+ else:
+ queryset = queryset.filter(id__gt=slot.commit_id)
+
+ return [{
+ 'user': c.user.username,
+ 'application': c.app.application_id,
+ } for c in queryset]
+
+
 @api_view
 def api_call(request, app, data):
  try:

serveradmin/changes/admin.py

@@ -0,0 +1,12 @@
+from django.contrib.admin import site, ModelAdmin, TabularInline
+
+from serveradmin.changes.models import Commit, Addition, Modification, Deletion
+
+
+site.register(Commit, type('CommitAdmin', (ModelAdmin, ), {
+ 'inlines': [
+ type('AdditionInline', (TabularInline, ), {'model': Addition}),
+ type('ModificationInline', (TabularInline, ), {'model': Modification}),
+ type('DeletionInline', (TabularInline, ), {'model': Deletion}),
+ ]
+}))

serveradmin/changes/migrations/0001_initial.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+from django.conf import settings
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('apps', '0001_initial'),
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='Addition',
+ fields=[
+ ('id', models.AutoField(serialize=False, auto_created=True, verbose_name='ID', primary_key=True)),
+ ('server_id', models.IntegerField(db_index=True)),
+ ('attributes_json', models.TextField()),
+ ],
+ ),
+ migrations.CreateModel(
+ name='Commit',
+ fields=[
+ ('id', models.AutoField(serialize=False, auto_created=True, verbose_name='ID', primary_key=True)),
+ ('change_on', models.DateTimeField(default=django.utils.timezone.now, db_index=True)),
+ ('app', models.ForeignKey(to='apps.Application', on_delete=django.db.models.deletion.PROTECT, null=True)),
+ ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=django.db.models.deletion.PROTECT, null=True)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='Deletion',
+ fields=[
+ ('id', models.AutoField(serialize=False, auto_created=True, verbose_name='ID', primary_key=True)),
+ ('server_id', models.IntegerField(db_index=True)),
+ ('attributes_json', models.TextField()),
+ ('commit', models.ForeignKey(to='changes.Commit')),
+ ],
+ ),
+ migrations.CreateModel(
+ name='Modification',
+ fields=[
+ ('id', models.AutoField(serialize=False, auto_created=True, verbose_name='ID', primary_key=True)),
+ ('server_id', models.IntegerField(db_index=True)),
+ ('updates_json', models.TextField()),
+ ('commit', models.ForeignKey(to='changes.Commit')),
+ ],
+ ),
+ migrations.AddField(
+ model_name='addition',
+ name='commit',
+ field=models.ForeignKey(to='changes.Commit'),
+ ),
+ migrations.AlterUniqueTogether(
+ name='modification',
+ unique_together=set([('commit', 'server_id')]),
+ ),
+ migrations.AlterUniqueTogether(
+ name='deletion',
+ unique_together=set([('commit', 'server_id')]),
+ ),
+ migrations.AlterUniqueTogether(
+ name='addition',
+ unique_together=set([('commit', 'server_id')]),
+ ),
+ ]