popit exporter scripts

Author: tfgg

_scripts/popit_create_posts.py

@@ -0,0 +1,70 @@
+import sys
+import requests
+import json
+import yaml
+
+from local_settings import POPIT_USERNAME, POPIT_PASSWORD, POPIT_APIKEY
+
+instance = "http://yournextmep.popit.mysociety.org/api/v0.1/"
+
+instance_meta = requests.get(instance).json()
+
+organizations_api_url = instance_meta['meta']['organizations_api_url']
+posts_api_url = instance_meta['meta']['posts_api_url']
+
+headers = {'content-type': 'application/json',
+           'Apikey': POPIT_APIKEY}
+
+def get_or_create_eup():
+  url = instance + "search/organizations?q=name:\"European Parliament\""
+
+  resp = requests.get(url).json()
+
+  if resp['result']:
+    return resp['result'][0]
+
+  eup = {u'name': u'European Parliament',
+         u'classification': u'Parliament'} 
+
+  resp = requests.post(organizations_api_url,
+                       data=json.dumps(eup),
+                       auth=(POPIT_USERNAME, POPIT_PASSWORD),
+                       headers=headers)
+  
+  return resp.json()['result']
+
+def get_or_create_post(post):
+  url = instance + "search/posts?q=label:\"{}\"".format(post['label'])
+  resp = requests.get(url).json()
+
+  if resp['result']:
+    return resp['result'][0]
+  else:
+    resp = requests.post(posts_api_url,
+                        data=json.dumps(post),
+                        auth=(POPIT_USERNAME, POPIT_PASSWORD),
+                        headers=headers)
+  
+    return resp.json()['result']
+
+eup = get_or_create_eup()
+
+regions = yaml.load(open('_data/regions.yaml'))
+
+for _, region in regions.items():
+  print region['name']
+
+  post = {'organization_id': eup['id'],
+          'role': 'Member of the European Parliament',
+          'label': 'Member of the European Parliament for {}'.format(region['name']),
+          "area": {
+            "identifier": "http://mapit.mysociety.org/area/{}".format(region['identifiers'][0]['identifier']),
+            "id": "mapit:{}".format(region['identifiers'][0]['identifier']),
+            "name": region['name']
+            },
+          }
+
+  post = get_or_create_post(post)
+
+  print post
+

_scripts/popit_export_people.py

@@ -3,55 +3,79 @@
 import json
 import yaml
 
-from local_settings import POPIT_USERNAME, POPIT_PASSWORD
+from local_settings import POPIT_USERNAME, POPIT_PASSWORD, POPIT_APIKEY
 
 instance = "http://yournextmep.popit.mysociety.org/api/v0.1/"
 
 instance_meta = requests.get(instance).json()
 
 people_api_url = instance_meta['meta']['persons_api_url']
+posts_api_url = instance_meta['meta']['posts_api_url']
 memberships_api_url = instance_meta['meta']['memberships_api_url']
 
+headers = {'content-type': 'application/json',
+           'Apikey': POPIT_APIKEY}
+
 def add_person(person):
-  print person
   return requests.post(people_api_url,
                        data=json.dumps(person),
                        auth=(POPIT_USERNAME, POPIT_PASSWORD),
-                       headers={'content-type': 'application/json'})
+                       headers=headers)
 
 def update_person(person):
   return requests.put(person['url'],
                      data=json.dumps(person),
                      auth=(POPIT_USERNAME, POPIT_PASSWORD),
-                     headers={'content-type': 'application/json'})
+                     headers=headers)
 
+def delete_person(person):
+  return requests.delete(person['url'],
+                         auth=(POPIT_USERNAME, POPIT_PASSWORD),
+                         headers=headers)
 
 def add_membership(membership):
   return requests.post(memberships_api_url,
                        data=json.dumps(membership),
                        auth=(POPIT_USERNAME, POPIT_PASSWORD),
-                       headers={'content-type': 'application/json'})
+                       headers=headers)
 
 def update_membership(membership):
   return requests.put(membership['url'],
                      data=json.dumps(membership),
                      auth=(POPIT_USERNAME, POPIT_PASSWORD),
-                     headers={'content-type': 'application/json'})
+                     headers=headers)
 
 def delete_membership(membership):
   return requests.delete(membership['url'],
                          auth=(POPIT_USERNAME, POPIT_PASSWORD),
-                         headers={'content-type': 'application/json'})
+                         headers=headers)
 
 def popitize(person, party_id, party):
   person = dict(person)
 
   person['identifiers'] = []
   person['identifiers'].append({'identifier': party_id + "/" + person['id'],
-                                'scheme': 'yournextmep'})
+                                'scheme': 'yournextmep-candidate'})
+
+  person['contact_details'] = []
+
+  if 'email' in person:
+    person['contact_details'].append({'type': 'email',
+                                      'value': person['email'],})
+    del person['email']
+
+  if 'links' in person:
+    for link in person['links']:
+      if 'twitter' in link['url']:
+        twitter_id = link['url'].split('/')[-1]
+
+        person['contact_details'].append({'type': 'twitter',
+                                          'value': twitter_id})
 
   if 'image' in person:
-    person['image'] = "http://yournextmep.com" + person['image']
+    image = {'url': "http://yournextmep.com" + person['image']}
+
+    person['images'] = [image] 
 
   del person['id']
 
@@ -70,26 +94,23 @@ def find_org_id(org_id):
 
   return None
 
-def find_person_id(person_id):
+def find_persons_id(person_id):
   url = instance + "search/persons?q=identifiers.identifier:\"{}\"".format(person_id)
 
   resp = requests.get(url).json()
 
-  print url
-
   if len(resp['result']) > 1:
-    print "MULTIPLE HITS", url
+    print "MULTIPLE PEOPLE HITS", url
 
   if resp['result']:
-    for org in resp['result']:
-      for ident in org['identifiers']:
-        if ident['scheme'] == "yournextmep" and ident['identifier'] == person_id:
-          return org
+    for person in resp['result']:
+      for ident in person['identifiers']:
+        if ident['scheme'] == "yournextmep-candidate" and ident['identifier'] == person_id:
+          yield person
 
-  return None
-
+  return
 
-def find_membership_id(person_id, org_id):
+def find_memberships_id(person_id, org_id):
   url = instance + "search/memberships?q=(person_id:\"{}\") AND (organization_id:\"{}\")".format(person_id, org_id)
 
   resp = requests.get(url).json()
@@ -98,10 +119,17 @@ def find_membership_id(person_id, org_id):
     print "MULTIPLE HITS", url
 
   if resp['result']:
-    return resp['result'][0]
+    return resp['result']
 
   return None
 
+def find_post(region_name):
+  url = instance + "search/posts?q=label:\"Member of the European Parliament for {}\"".format(region_name)
+
+  resp = requests.get(url).json()
+
+  return resp['result'][0]
+
 if __name__ == "__main__":
   party_id = sys.argv[1]
   candidates = yaml.load(open("_data/{}_candidates.yaml".format(party_id)))
@@ -111,50 +139,58 @@ def find_membership_id(person_id, org_id):
   party = find_org_id(party_id)
 
   for region_id, region_candidates in candidates.items():
+    popit_post = find_post(regions[region_id]['name'])
+
     for candidate in region_candidates:
       person_id = candidate['id']
 
       person = people[person_id]
       person = popitize(person, party_id, party)
 
-      popit_person = find_person_id(party_id + "/" + person_id) 
+      print person_id
+
+      popit_persons = list(find_persons_id(party_id + "/" + person_id))
 
-      if popit_person is None:
+      if not popit_persons:
         print "  Adding"
         resp = add_person(person)
-        print resp.status_code, resp.text
       else:
-        print "  Already up", popit_person['url']
-        #resp = update_person(popit_person)
-
-
-      popit_person = find_person_id(party_id + "/" + person_id) 
-      
-      popit_membership = find_membership_id(popit_person['id'], party_id) 
+        for popit_person in popit_persons[1:]:
+          print "Deleting duplicate person"
+          delete_person(popit_person)
 
-      if popit_membership is not None:
-        delete_membership(popit_membership)
+        popit_person = popit_persons[0]
+        print "  Updating", popit_person['url']
+        popit_person.update(person)
+        resp = update_person(popit_person)
 
-      popit_membership = find_membership_id(popit_person['id'], party['id']) 
+      #popit_person = find_person_id(party_id + "/" + person_id)
+      popit_person = resp.json()['result']
+     
+      popit_memberships = find_memberships_id(popit_person['id'], party['id'])
 
-      print popit_membership
+      if popit_memberships:
+        for popit_membership in popit_memberships[1:]:
+          print "Deleting membership"
+          delete_membership(popit_membership)
 
       membership = {
         "label": "European electoral candidate for {}".format(regions[region_id]['name']),
-        "role": "European electoral candidate",
+        "role": "candidate",
+        "post_id": popit_post['id'],
         "person_id": popit_person['id'],
         "organization_id": party['id'],
-        "area": {"name": regions[region_id]['name'],
-                 "classification": "EUR",
-                 "id": "mapit.mysociety.org/area/{}".format(regions[region_id]['identifiers'][0]['identifier'])}
       }
 
-      if popit_membership is None:
+      if popit_memberships is None:
+        print "Adding membership"
         resp = add_membership(membership) 
       else:
+        print "Updating membership"
+        popit_membership = popit_memberships[0]
         membership['id'] = popit_membership['id']
         membership['url'] = popit_membership['url']
         resp = update_membership(membership)
 
-      print resp.status_code, resp.text
+      print resp.status_code