Issue #278 - Refactor Database (#281)

* Refactor Officer -> Suspect
First step of the refactor is to separate the Officer table from the version that is a child of the incident table. We'll set up a standalone officer and agency table where we'll store records related to known officers and law enforcement agencies. The old officer table, now called Suspects will store information about those officers suspected of committing a crime against the public. The suspect table will be connected to officers via the accusation table.

* Refactor: Source is now parent of incident
 - Remove source from other models
 - Remove SourceMixin

* Many-to-Many Relationships added.
Tags and Agencies linked to Incidents
Suspects and Agencies linked to officers
Style changes.

* Refactor: suspect --> perpetrator

* Flake8 fix

* Update perpetrator backref
Update tests
Flake8 fix

* Additional Refactoring changes

* UI Errors

* UI Test Fixes
- Updated Snapshots
- Update models

* Fix mock data issue

* Continued refactoring

* Mismatched Source ID type

* Update victim table

* Jest corrections

---------

Co-authored-by: Darrell Malone <[email protected]>

alembic/dev_seeds.py

@@ -2,12 +2,11 @@
 from backend.database import User, UserRole
 from backend.auth import user_manager
 from backend.database.models.incident import Incident
-from backend.database.models.officer import Officer
+from backend.database.models.perpetrator import Perpetrator
 from backend.database.models.source import Source
 from backend.database.models.use_of_force import UseOfForce
 
 
-
 def create_user(user):
  user_exists = (
  db.session.query(User).filter_by(email=user.email).first() is not None
@@ -61,6 +60,7 @@ def create_user(user):
  )
 )
 
+
 def create_source(source):
  source_exists = (
  db.session.query(Source).filter_by(id=source.id).first() is not None
@@ -69,36 +69,37 @@ def create_source(source):
  if not source_exists:
  source.create()
 
+
 create_source(
  Source(
- id="mpv",
- publication_name="Mapping Police Violence",
- publication_date="01/01/2015",
- author="Samuel Sinyangwe",
- URL="https://mappingpoliceviolence.us",
+ id=10000000,
+ name="Mapping Police Violence",
+ url="https://mappingpoliceviolence.us",
+ contact_email="[email protected]"
  )
 )
 
 
 def create_incident(key=1, date="10-01-2019", lon=84, lat=34):
  base_id = 10000000
  id = base_id + key
+ mpv = db.session.query(Source).filter_by(
+ name="Mapping Police Violence").first()
  incident = Incident(
  id=id,
+ source=mpv,
  location=f"Test location {key}",
  longitude=lon,
  latitude=lat,
  description=f"Test description {key}",
- department=f"Small Police Department {key}",
  time_of_incident=f"{date} 00:00:00",
- officers=[
- Officer(
+ perpetrators=[
+ Perpetrator(
  first_name=f"TestFirstName {key}",
  last_name=f"TestLastName {key}",
  )
  ],
- use_of_force=[UseOfForce(item=f"gunshot {key}")],
- source="mpv",
+ use_of_force=[UseOfForce(item=f"gunshot {key}")]
  )
  exists = db.session.query(Incident).filter_by(id=id).first() is not None
 

backend/database/__init__.py

@@ -16,7 +16,8 @@
 from .models.incident import *
 from .models.investigation import *
 from .models.legal_case import *
-from .models.multimedia import *
+from .models.attachment import *
+from .models.perpetrator import *
 from .models.officer import *
 from .models.participant import *
 from .models.tag import *
@@ -25,5 +26,4 @@
 from .models.use_of_force import *
 from .models.user import *
 from .models.victim import *
-from .models.accusation import *
 from .models.source import *

backend/database/core.py

@@ -15,9 +15,7 @@
 from flask_sqlalchemy import SQLAlchemy
 from psycopg2 import connect
 from psycopg2.extensions import connection
-from sqlalchemy import ForeignKey
 from sqlalchemy.exc import ResourceClosedError
-from sqlalchemy.ext.declarative import declared_attr
 from werkzeug.utils import secure_filename
 
 from ..config import TestingConfig
@@ -50,34 +48,6 @@ def get(cls, id: Any, abort_if_null: bool = True):
  return obj
 
 
-class SourceMixin:
- """Adds support for unique, external source ID's"""
-
- # Identifies the source dataset or organization
- @declared_attr
- def source(self):
- return db.Column(db.Text, ForeignKey("source.id"))
-
- # Identifies the unique primary key in the source
- source_id = db.Column(db.Text)
-
- def __init_subclass__(cls, **kwargs):
- # Require that source ID's be unique within each source. Postgres does
- # not enforce uniqueness if either value is null.
- # https://www.postgresql.org/docs/9.0/ddl-constraints.html#AEN2445
- uc = db.UniqueConstraint(
- "source", "source_id", name=f"{cls.__name__.lower()}_source_uc"
- )
-
- cls.__table_args__ = tuple(
- a
- for a in (uc, getattr(cls, "__table_args__", None))
- if a is not None
- )
-
- super().__init_subclass__(**kwargs)
-
-
 QUERIES_DIR = os.path.abspath(
  os.path.join(os.path.dirname(__file__), "queries")
 )

backend/database/models/_assoc_tables.py

@@ -0,0 +1,40 @@
+from .. import db
+from backend.database.models.officer import Rank
+
+incident_agency = db.Table(
+ 'incident_agency',
+ db.Column('incident_id', db.Integer, db.ForeignKey('incident.id'),
+ primary_key=True),
+ db.Column('agency_id', db.Integer, db.ForeignKey('agency.id'),
+ primary_key=True),
+ db.Column('officers_present', db.Integer)
+)
+
+incident_tag = db.Table(
+ 'incident_tag',
+ db.Column('incident_id', db.Integer, db.ForeignKey('incident.id'),
+ primary_key=True),
+ db.Column('tag_id', db.Integer, db.ForeignKey('tag.id'), primary_key=True)
+)
+
+agency_officer = db.Table(
+ 'agency_officer',
+ db.Column('agency_id', db.Integer, db.ForeignKey('agency.id'),
+ primary_key=True),
+ db.Column('officer_id', db.Integer, db.ForeignKey('officer.id'),
+ primary_key=True),
+ db.Column('earliest_employment', db.Text),
+ db.Column('latest_employment', db.Text),
+ db.Column('badge_number', db.Text),
+ db.Column('unit', db.Text),
+ db.Column('highest_rank', db.Enum(Rank)),
+ db.Column('currently_employed', db.Boolean)
+)
+
+perpetrator_officer = db.Table(
+ 'perpetrator_officer',
+ db.Column('perpetrator_id', db.Integer, db.ForeignKey('perpetrator.id'),
+ primary_key=True),
+ db.Column('officer_id', db.Integer, db.ForeignKey('officer.id'),
+ primary_key=True)
+)

backend/database/models/agency.py

@@ -1,7 +1,27 @@
+import enum
+from backend.database.models._assoc_tables import agency_officer
 from .. import db
 
 
+class JURISDICTION(enum.Enum):
+ FEDERAL = 1
+ STATE = 2
+ COUNTY = 3
+ MUNICIPAL = 4
+ PRIVATE = 5
+ OTHER = 6
+
+
 class Agency(db.Model):
  id = db.Column(db.Integer, primary_key=True)
- incident_id = db.Column(db.Integer, db.ForeignKey("incident.id"))
  name = db.Column(db.Text)
+ website_url = db.Column(db.Text)
+ hq_address = db.Column(db.Text)
+ hq_city = db.Column(db.Text)
+ hq_zip = db.Column(db.Text)
+ jurisdiction = db.Column(db.Enum(JURISDICTION))
+ known_officers = db.relationship(
+ "Officer", secondary=agency_officer, backref="known_employers")
+
+ def __repr__(self):
+ return f"<Agency {self.name}>"

backend/database/models/attachment.py

@@ -0,0 +1,10 @@
+from .. import db
+
+
+class Attachment(db.Model):
+ id = db.Column(db.Integer, primary_key=True)
+ incident_id = db.Column(db.Integer, db.ForeignKey("incident.id"))
+ title = db.Column(db.Text)
+ hash = db.Column(db.Text)
+ location = db.Column(db.Text)
+ filetype = db.Column(db.Text)

backend/database/models/incident.py

@@ -1,10 +1,15 @@
 """Define the SQL classes for Users."""
 import enum
 
+from ..core import CrudMixin, db
+from backend.database.models._assoc_tables import incident_agency, incident_tag
 
-from ..core import CrudMixin, SourceMixin, db
 
-# Question: Should we be doing string enums?
+class RecordType(enum.Enum):
+ NEWS_REPORT = 1
+ GOVERNMENT_RECORD = 2
+ LEGAL_ACTION = 3
+ PERSONAL_ACCOUNT = 4
 
 
 class InitialEncounter(enum.Enum):
@@ -50,31 +55,26 @@ class VictimStatus(enum.Enum):
  DECEASED = 5
 
 
-# TODO: This file's a bit of a mess (my fault!)
-# There are a lot of association tables in here, and the incidents table is
-# not clearly either a facts table or component table.
-# We need to get a better idea of the relationships we want and then we should
-# implement them accordingly.
-
-
-class Incident(db.Model, CrudMixin, SourceMixin):
+class Incident(db.Model, CrudMixin):
  """The incident table is the fact table."""
 
  id = db.Column(db.Integer, primary_key=True, autoincrement=True)
+ source_id = db.Column(
+ db.String, db.ForeignKey("source.id"))
+ source_details = db.relationship(
+ "SourceDetails", backref="incident", uselist=False)
  time_of_incident = db.Column(db.DateTime)
+ time_confidence = db.Column(db.Integer)
  complaint_date = db.Column(db.Date)
  closed_date = db.Column(db.Date)
  location = db.Column(db.Text) # TODO: location object
  # Float is double precision (8 bytes) by default in Postgres
  longitude = db.Column(db.Float)
  latitude = db.Column(db.Float)
- # TODO: neighborhood seems like a weird identifier that may not always
- # apply in consistent ways across municipalities.
- neighborhood = db.Column(db.Text)
  description = db.Column(db.Text)
  stop_type = db.Column(db.Text) # TODO: enum
  call_type = db.Column(db.Text) # TODO: enum
- has_multimedia = db.Column(db.Boolean)
+ has_attachments = db.Column(db.Boolean)
  from_report = db.Column(db.Boolean)
  # These may require an additional table. Also can dox a victim
  was_victim_arrested = db.Column(db.Boolean)
@@ -83,31 +83,38 @@ class Incident(db.Model, CrudMixin, SourceMixin):
  criminal_case_brought = db.Column(db.Boolean)
  case_id = db.Column(db.Integer) # TODO: foreign key of some sort?
  victims = db.relationship("Victim", backref="incident")
- # TODO: Remove this. incident-officer relationship is many-many using
- # accusation as the join table.
- officers = db.relationship("Officer", backref="incident")
- department = db.Column(db.Text)
+ perpetrators = db.relationship("Perpetrator", backref="incident")
  # descriptions = db.relationship("Description", backref="incident")
- tags = db.relationship("Tag", backref="incident")
+ tags = db.relationship("Tag", secondary=incident_tag, backref="incidents")
+ agencies_present = db.relationship(
+ "Agency", secondary=incident_agency, backref="recorded_incidents")
  participants = db.relationship("Participant", backref="incident")
- multimedias = db.relationship("Multimedia", backref="incident")
+ attachments = db.relationship("Attachment", backref="incident")
  investigations = db.relationship("Investigation", backref="incident")
  results_of_stop = db.relationship("ResultOfStop", backref="incident")
  actions = db.relationship("Action", backref="incident")
  use_of_force = db.relationship("UseOfForce", backref="incident")
  legal_case = db.relationship("LegalCase", backref="incident")
- accusations = db.relationship("Accusation", backref="incident")
-
 
-class Description(db.Model):
- id = db.Column(db.Integer, primary_key=True) # description id
- incident_id = db.Column(
- db.Integer, db.ForeignKey("incident.id"), nullable=False
- )
- text = db.Column(db.Text)
- type = db.Column(db.Text) # TODO: enum
+ def __repr__(self):
+ """Represent instance as a unique string."""
+ return f"<Incident {self.id}>"
+
+
+# On the Description object:
+# Seems like this is based on the WITNESS standard. It also appears that the
+# original intention of that standard is to allow multiple descriptions to be
+# applied to a single incident. I recomend we handle this as part of a
+# larger epic when we add the annotation system, which is related.
+# class Description(db.Model):
+# id = db.Column(db.Integer, primary_key=True) # description id
+# incident_id = db.Column(
+# db.Integer, db.ForeignKey("incident.id"), nullable=False
+# )
+# text = db.Column(db.Text)
+# type = db.Column(db.Text) # TODO: enum
  # TODO: are there rules for this column other than text?
- source = db.Column(db.Text)
+ # source = db.Column(db.Text)
  # location = db.Column(db.Text) # TODO: location object
  # # TODO: neighborhood seems like a weird identifier that may not always
  # # apply in consistent ways across municipalities.
@@ -122,3 +129,27 @@ class Description(db.Model):
  # # Does an existing warrant count here?
  # criminal_case_brought = db.Column(db.Boolean)
  # case_id = db.Column(db.Integer) # TODO: foreign key of some sort?
+
+
+class SourceDetails(db.Model):
+ id = db.Column(db.Integer, primary_key=True) # source details id
+ incident_id = db.Column(
+ db.Integer, db.ForeignKey("incident.id"), nullable=False
+ )
+ record_type = db.Column(db.Enum(RecordType))
+ # For Journalistic Publications
+ publication_name = db.Column(db.Text)
+ publication_date = db.Column(db.Date)
+ publication_url = db.Column(db.Text)
+ author = db.Column(db.Text)
+ author_url = db.Column(db.Text)
+ author_email = db.Column(db.Text)
+ # For Government Records
+ reporting_organization = db.Column(db.Text)
+ reporting_organization_url = db.Column(db.Text)
+ reporting_organization_email = db.Column(db.Text)
+ # For Legal Records
+ court = db.Column(db.Text)
+ judge = db.Column(db.Text)
+ docket_number = db.Column(db.Text)
+ date_of_action = db.Column(db.Date)