diff --git a/src/core/admin.py b/src/core/admin.py index d93dfc381c..b47b813121 100755 --- a/src/core/admin.py +++ b/src/core/admin.py @@ -40,20 +40,20 @@ class SettingAdmin(admin.ModelAdmin): class AccountAdmin(UserAdmin): """Displays Account objects in the Django admin interface.""" list_display = ('id', 'email', 'orcid', 'first_name', 'middle_name', - 'last_name', 'institution', '_roles_in', 'last_login') + 'last_name', '_roles_in', 'last_login') list_display_links = ('id', 'email') list_filter = ('accountrole__journal', 'repositoryrole__repository__short_name', 'is_active', 'is_staff', 'is_admin', 'is_superuser', 'last_login') search_fields = ('id', 'username', 'email', 'first_name', 'middle_name', - 'last_name', 'orcid', 'institution', + 'last_name', 'orcid', 'biography', 'signature') fieldsets = UserAdmin.fieldsets + ( (None, {'fields': ( 'name_prefix', 'middle_name', 'orcid', - 'institution', 'department', 'country', 'twitter', + 'twitter', 'linkedin', 'facebook', 'github', 'website', 'biography', 'enable_public_profile', 'signature', 'profile_image', 'interest', "preferred_timezone", )}), @@ -71,6 +71,7 @@ class AccountAdmin(UserAdmin): raw_id_fields = ('interest',) inlines = [ + admin_utils.AffiliationInline, admin_utils.AccountRoleInline, admin_utils.RepositoryRoleInline, admin_utils.EditorialGroupMemberInline, @@ -398,6 +399,84 @@ class AccessRequestAdmin(admin.ModelAdmin): date_hierarchy = ('requested') +class OrganizationAdmin(admin.ModelAdmin): + list_display = ('pk', 'ror', '_ror_display', '_custom_label', + 'website', '_locations', 'ror_status') + list_display_links = ('pk', 'ror') + list_filter = ('ror_status', 'locations__country') + search_fields = ('pk', 'ror_display__value', 'custom_label__value', 'labels__value', + 'aliases__value', 'acronyms__value', 'website', 'ror') + raw_id_fields = ('locations', ) + + def _ror_display(self, obj): + return obj.ror_display if obj and obj.ror_display else '' + + def _locations(self, obj): + return '; '.join([str(l) for l in obj.locations.all()]) if obj else '' + + def _custom_label(self, obj): + return obj.custom_label if obj and obj.custom_label else '' + + +class OrganizationNameAdmin(admin.ModelAdmin): + list_display = ('pk', 'value', 'language') + list_display_links = ('pk', 'value') + search_fields = ('pk', 'value') + raw_id_fields = ('ror_display_for', 'custom_label_for', + 'label_for', 'alias_for', 'acronym_for') + + def _ror_display(self, obj): + return obj.ror_display if obj and obj.ror_display else '' + + def _locations(self, obj): + return '; '.join([str(l) for l in obj.locations.all()]) if obj else '' + + def _custom_label(self, obj): + return obj.custom_label if obj and obj.custom_label else '' + + +class LocationAdmin(admin.ModelAdmin): + list_display = ('pk', 'name', 'country', 'geonames_id') + list_display_links = ('pk', 'name') + list_filter = ('country',) + search_fields = ('pk', 'name', 'country__code', 'country__name', + 'geonames_id') + + +class AffiliationAdmin(admin.ModelAdmin): + list_display = ('pk', '_person', 'organization', + 'title', 'department', 'start', 'end') + list_display_links = ('pk', '_person') + list_filter = ('start', 'end', 'organization__locations__country') + search_fields = ( + 'pk', + 'title', + 'department', + 'organization__ror_display__value', + 'organization__custom_label__value', + 'organization__labels__value', + 'organization__aliases__value', + 'organization__acronyms__value', + 'account__first_name', + 'account__last_name', + 'account__email', + 'frozen_author__first_name', + 'frozen_author__last_name', + 'frozen_author__frozen_email', + 'preprint_author__account__first_name', + 'preprint_author__account__last_name', + 'preprint_author__account__email', + ) + raw_id_fields = ('account', 'frozen_author', + 'preprint_author', 'organization') + + def _person(self, obj): + if obj: + return obj.account or obj.frozen_author or obj.preprint_author + else: + return '' + + admin_list = [ (models.AccountRole, AccountRoleAdmin), (models.Account, AccountAdmin), @@ -427,6 +506,10 @@ class AccessRequestAdmin(admin.ModelAdmin): (models.Contacts, ContactsAdmin), (models.Contact, ContactAdmin), (models.AccessRequest, AccessRequestAdmin), + (models.Organization, OrganizationAdmin), + (models.OrganizationName, OrganizationNameAdmin), + (models.Location, LocationAdmin), + (models.Affiliation, AffiliationAdmin), ] [admin.site.register(*t) for t in admin_list] diff --git a/src/core/forms/__init__.py b/src/core/forms/__init__.py index 168dc67668..39874a730a 100644 --- a/src/core/forms/__init__.py +++ b/src/core/forms/__init__.py @@ -2,10 +2,12 @@ AccessRequestForm, AccountRoleForm, AdminUserForm, + AffiliationForm, ArticleMetaImageForm, CBVFacetForm, ConfirmableForm, ConfirmableIfErrorsForm, + ConfirmDeleteForm, EditAccountForm, EditKey, EditorialGroupForm, @@ -24,6 +26,7 @@ JournalSubmissionForm, LoginForm, NotificationForm, + OrganizationNameForm, PasswordResetForm, PressJournalAttrForm, QuickUserForm, diff --git a/src/core/forms/forms.py b/src/core/forms/forms.py index bb1359b6a7..a681231f83 100755 --- a/src/core/forms/forms.py +++ b/src/core/forms/forms.py @@ -183,7 +183,7 @@ class RegistrationForm(forms.ModelForm, CaptchaForm): class Meta: model = models.Account fields = ('email', 'salutation', 'first_name', 'middle_name', - 'last_name', 'department', 'institution', 'country', 'orcid',) + 'last_name', 'orcid',) widgets = {'orcid': forms.HiddenInput() } def __init__(self, *args, **kwargs): @@ -532,7 +532,7 @@ def __init__(self, *args, **kwargs): class QuickUserForm(forms.ModelForm): class Meta: model = models.Account - fields = ('email', 'salutation', 'first_name', 'last_name', 'institution',) + fields = ('email', 'salutation', 'first_name', 'last_name',) class LoginForm(CaptchaForm): @@ -940,3 +940,33 @@ class AccountRoleForm(forms.ModelForm): class Meta: model = models.AccountRole fields = '__all__' + + +class OrganizationNameForm(forms.ModelForm): + + class Meta: + model = models.OrganizationName + fields = ('value',) + + +class AffiliationForm(forms.ModelForm): + + class Meta: + model = models.Affiliation + fields = '__all__' + widgets = { + 'account': forms.HiddenInput, + 'frozen_author': forms.HiddenInput, + 'preprint_author': forms.HiddenInput, + 'organization': forms.HiddenInput, + 'start': HTMLDateInput, + 'end': HTMLDateInput, + } + + +class ConfirmDeleteForm(forms.Form): + """ + A generic form for use on confirm-delete pages + where a valid form with POST data means yes, delete. + """ + pass diff --git a/src/core/include_urls.py b/src/core/include_urls.py index dea31e2e0b..fda6804112 100644 --- a/src/core/include_urls.py +++ b/src/core/include_urls.py @@ -118,6 +118,38 @@ re_path(r'^manager/user/(?P\d+)/edit/$', core_views.user_edit, name='core_user_edit'), re_path(r'^manager/user/(?P\d+)/history/$', core_views.user_history, name='core_user_history'), + ## Affiliations + re_path( + r'^profile/organization/search/$', + core_views.OrganizationListView.as_view(), + name='core_organization_search' + ), + re_path( + r'^profile/organization_name/create/$', + core_views.organization_name_create, + name='core_organization_name_create' + ), + re_path( + r'^profile/organization_name/(?P\d+)/update/$', + core_views.organization_name_update, + name='core_organization_name_update' + ), + re_path( + r'^profile/organization/(?P\d+)/affiliation/create/$', + core_views.affiliation_create, + name='core_affiliation_create' + ), + re_path( + r'^profile/affiliation/(?P\d+)/update/$', + core_views.affiliation_update, + name='core_affiliation_update' + ), + re_path( + r'^profile/affiliation/(?P\d+)/delete/$', + core_views.affiliation_delete, + name='core_affiliation_delete' + ), + # Templates re_path(r'^manager/templates/$', core_views.email_templates, name='core_email_templates'), diff --git a/src/core/logic.py b/src/core/logic.py index 98268f3ae4..24ae68cbd7 100755 --- a/src/core/logic.py +++ b/src/core/logic.py @@ -868,7 +868,7 @@ def check_for_bad_login_attempts(request): time = timezone.now() - timedelta(minutes=10) attempts = models.LoginAttempt.objects.filter(user_agent=user_agent, ip_address=ip_address, timestamp__gte=time) - print(time, attempts.count()) + logger.info(time, attempts.count()) return attempts.count() diff --git a/src/core/migrations/0100_location_organization_affiliation.py b/src/core/migrations/0100_location_organization_affiliation.py new file mode 100644 index 0000000000..4dd191d886 --- /dev/null +++ b/src/core/migrations/0100_location_organization_affiliation.py @@ -0,0 +1,75 @@ +# Generated by Django 4.2.14 on 2024-07-26 13:26 + +import core.models +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0099_alter_accountrole_options'), + ('utils', '0035_rorimport_rorimporterror'), + ] + + operations = [ + migrations.CreateModel( + name='Location', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(blank=True, help_text='City or place name', max_length=200)), + ('latitude', models.DecimalField(blank=True, decimal_places=6, max_digits=9, null=True)), + ('longitude', models.DecimalField(blank=True, decimal_places=6, max_digits=9, null=True)), + ('geonames_id', models.IntegerField(blank=True, null=True)), + ('country', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.country')), + ], + ), + migrations.CreateModel( + name='Organization', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ror', models.URLField(blank=True, help_text='Research Organization Registry identifier (URL)', validators=[core.models.validate_ror], verbose_name='ROR')), + ('ror_status', models.CharField(blank=True, choices=[('active', 'Active'), ('inactive', 'Inactive'), ('withdrawn', 'Withdrawn'), ('unknown', 'Unknown')], default='unknown', max_length=10)), + ('ror_record_timestamp', models.CharField(blank=True, help_text='The admin.last_modified.date string from ROR data', max_length=10)), + ('website', models.CharField(blank=True, max_length=2000)), + ('locations', models.ManyToManyField(blank=True, null=True, to='core.location')), + ], + options={'ordering': ['ror_display__value']}, + ), + migrations.CreateModel( + name='OrganizationName', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('value', models.CharField(blank=True, max_length=200, verbose_name='Organization name')), + ('language', models.CharField(blank=True, choices=[('eng', 'English'), ('abk', 'Abkhazian'), ('ace', 'Achinese'), ('ach', 'Acoli'), ('ada', 'Adangme'), ('ady', 'Adyghe; Adygei'), ('aar', 'Afar'), ('afh', 'Afrihili'), ('afr', 'Afrikaans'), ('afa', 'Afro-Asiatic languages'), ('ain', 'Ainu'), ('aka', 'Akan'), ('akk', 'Akkadian'), ('sqi', 'Albanian'), ('ale', 'Aleut'), ('alg', 'Algonquian languages'), ('tut', 'Altaic languages'), ('amh', 'Amharic'), ('anp', 'Angika'), ('apa', 'Apache languages'), ('ara', 'Arabic'), ('arg', 'Aragonese'), ('arp', 'Arapaho'), ('arw', 'Arawak'), ('hye', 'Armenian'), ('rup', 'Aromanian; Arumanian; Macedo-Romanian'), ('art', 'Artificial languages'), ('asm', 'Assamese'), ('ast', 'Asturian; Bable; Leonese; Asturleonese'), ('ath', 'Athapascan languages'), ('aus', 'Australian languages'), ('map', 'Austronesian languages'), ('ava', 'Avaric'), ('ave', 'Avestan'), ('awa', 'Awadhi'), ('aym', 'Aymara'), ('aze', 'Azerbaijani'), ('ban', 'Balinese'), ('bat', 'Baltic languages'), ('bal', 'Baluchi'), ('bam', 'Bambara'), ('bai', 'Bamileke languages'), ('bad', 'Banda languages'), ('bnt', 'Bantu languages'), ('bas', 'Basa'), ('bak', 'Bashkir'), ('eus', 'Basque'), ('btk', 'Batak languages'), ('bej', 'Beja; Bedawiyet'), ('bel', 'Belarusian'), ('bem', 'Bemba'), ('ben', 'Bengali'), ('ber', 'Berber languages'), ('bho', 'Bhojpuri'), ('bih', 'Bihari languages'), ('bik', 'Bikol'), ('bin', 'Bini; Edo'), ('bis', 'Bislama'), ('byn', 'Blin; Bilin'), ('zbl', 'Blissymbols; Blissymbolics; Bliss'), ('nob', 'Bokmål, Norwegian; Norwegian Bokmål'), ('bos', 'Bosnian'), ('bra', 'Braj'), ('bre', 'Breton'), ('bug', 'Buginese'), ('bul', 'Bulgarian'), ('bua', 'Buriat'), ('mya', 'Burmese'), ('cad', 'Caddo'), ('cat', 'Catalan; Valencian'), ('cau', 'Caucasian languages'), ('ceb', 'Cebuano'), ('cel', 'Celtic languages'), ('cai', 'Central American Indian languages'), ('khm', 'Central Khmer'), ('chg', 'Chagatai'), ('cmc', 'Chamic languages'), ('cha', 'Chamorro'), ('che', 'Chechen'), ('chr', 'Cherokee'), ('chy', 'Cheyenne'), ('chb', 'Chibcha'), ('nya', 'Chichewa; Chewa; Nyanja'), ('zho', 'Chinese'), ('chn', 'Chinook jargon'), ('chp', 'Chipewyan; Dene Suline'), ('cho', 'Choctaw'), ('chu', 'Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic'), ('chk', 'Chuukese'), ('chv', 'Chuvash'), ('nwc', 'Classical Newari; Old Newari; Classical Nepal Bhasa'), ('syc', 'Classical Syriac'), ('cop', 'Coptic'), ('cor', 'Cornish'), ('cos', 'Corsican'), ('cre', 'Cree'), ('mus', 'Creek'), ('crp', 'Creoles and pidgins'), ('cpe', 'Creoles and pidgins, English based'), ('cpf', 'Creoles and pidgins, French-based'), ('cpp', 'Creoles and pidgins, Portuguese-based'), ('crh', 'Crimean Tatar; Crimean Turkish'), ('hrv', 'Croatian'), ('cus', 'Cushitic languages'), ('ces', 'Czech'), ('dak', 'Dakota'), ('dan', 'Danish'), ('dar', 'Dargwa'), ('del', 'Delaware'), ('din', 'Dinka'), ('div', 'Divehi; Dhivehi; Maldivian'), ('doi', 'Dogri'), ('dgr', 'Dogrib'), ('dra', 'Dravidian languages'), ('dua', 'Duala'), ('dum', 'Dutch, Middle (ca. 1050-1350)'), ('nld', 'Dutch; Flemish'), ('dyu', 'Dyula'), ('dzo', 'Dzongkha'), ('frs', 'Eastern Frisian'), ('efi', 'Efik'), ('egy', 'Egyptian (Ancient)'), ('eka', 'Ekajuk'), ('elx', 'Elamite'), ('enm', 'English, Middle (1100-1500)'), ('ang', 'English, Old (ca. 450-1100)'), ('myv', 'Erzya'), ('epo', 'Esperanto'), ('est', 'Estonian'), ('ewe', 'Ewe'), ('ewo', 'Ewondo'), ('fan', 'Fang'), ('fat', 'Fanti'), ('fao', 'Faroese'), ('fij', 'Fijian'), ('fil', 'Filipino; Pilipino'), ('fin', 'Finnish'), ('fiu', 'Finno-Ugrian languages'), ('fon', 'Fon'), ('fra', 'French'), ('frm', 'French, Middle (ca. 1400-1600)'), ('fro', 'French, Old (842-ca. 1400)'), ('fur', 'Friulian'), ('ful', 'Fulah'), ('gaa', 'Ga'), ('gla', 'Gaelic; Scottish Gaelic'), ('car', 'Galibi Carib'), ('glg', 'Galician'), ('lug', 'Ganda'), ('gay', 'Gayo'), ('gba', 'Gbaya'), ('gez', 'Geez'), ('kat', 'Georgian'), ('deu', 'German'), ('gmh', 'German, Middle High (ca. 1050-1500)'), ('goh', 'German, Old High (ca. 750-1050)'), ('gem', 'Germanic languages'), ('gil', 'Gilbertese'), ('gon', 'Gondi'), ('gor', 'Gorontalo'), ('got', 'Gothic'), ('grb', 'Grebo'), ('grc', 'Greek, Ancient (to 1453)'), ('ell', 'Greek, Modern (1453-)'), ('grn', 'Guarani'), ('guj', 'Gujarati'), ('gwi', "Gwich'in"), ('hai', 'Haida'), ('hat', 'Haitian; Haitian Creole'), ('hau', 'Hausa'), ('haw', 'Hawaiian'), ('heb', 'Hebrew'), ('her', 'Herero'), ('hil', 'Hiligaynon'), ('him', 'Himachali languages; Western Pahari languages'), ('hin', 'Hindi'), ('hmo', 'Hiri Motu'), ('hit', 'Hittite'), ('hmn', 'Hmong; Mong'), ('hun', 'Hungarian'), ('hup', 'Hupa'), ('iba', 'Iban'), ('isl', 'Icelandic'), ('ido', 'Ido'), ('ibo', 'Igbo'), ('ijo', 'Ijo languages'), ('ilo', 'Iloko'), ('smn', 'Inari Sami'), ('inc', 'Indic languages'), ('ine', 'Indo-European languages'), ('ind', 'Indonesian'), ('inh', 'Ingush'), ('ina', 'Interlingua (International Auxiliary Language Association)'), ('ile', 'Interlingue; Occidental'), ('iku', 'Inuktitut'), ('ipk', 'Inupiaq'), ('ira', 'Iranian languages'), ('gle', 'Irish'), ('mga', 'Irish, Middle (900-1200)'), ('sga', 'Irish, Old (to 900)'), ('iro', 'Iroquoian languages'), ('ita', 'Italian'), ('jpn', 'Japanese'), ('jav', 'Javanese'), ('jrb', 'Judeo-Arabic'), ('jpr', 'Judeo-Persian'), ('kbd', 'Kabardian'), ('kab', 'Kabyle'), ('kac', 'Kachin; Jingpho'), ('kal', 'Kalaallisut; Greenlandic'), ('xal', 'Kalmyk; Oirat'), ('kam', 'Kamba'), ('kan', 'Kannada'), ('kau', 'Kanuri'), ('kaa', 'Kara-Kalpak'), ('krc', 'Karachay-Balkar'), ('krl', 'Karelian'), ('kar', 'Karen languages'), ('kas', 'Kashmiri'), ('csb', 'Kashubian'), ('kaw', 'Kawi'), ('kaz', 'Kazakh'), ('kha', 'Khasi'), ('khi', 'Khoisan languages'), ('kho', 'Khotanese;Sakan'), ('kik', 'Kikuyu; Gikuyu'), ('kmb', 'Kimbundu'), ('kin', 'Kinyarwanda'), ('kir', 'Kirghiz; Kyrgyz'), ('tlh', 'Klingon; tlhIngan-Hol'), ('kom', 'Komi'), ('kon', 'Kongo'), ('kok', 'Konkani'), ('kor', 'Korean'), ('kos', 'Kosraean'), ('kpe', 'Kpelle'), ('kro', 'Kru languages'), ('kua', 'Kuanyama; Kwanyama'), ('kum', 'Kumyk'), ('kur', 'Kurdish'), ('kru', 'Kurukh'), ('kut', 'Kutenai'), ('lad', 'Ladino'), ('lah', 'Lahnda'), ('lam', 'Lamba'), ('day', 'Land Dayak languages'), ('lao', 'Lao'), ('lat', 'Latin'), ('lav', 'Latvian'), ('lez', 'Lezghian'), ('lim', 'Limburgan; Limburger; Limburgish'), ('lin', 'Lingala'), ('lit', 'Lithuanian'), ('jbo', 'Lojban'), ('nds', 'Low German; Low Saxon; German, Low; Saxon, Low'), ('dsb', 'Lower Sorbian'), ('loz', 'Lozi'), ('lub', 'Luba-Katanga'), ('lua', 'Luba-Lulua'), ('lui', 'Luiseno'), ('smj', 'Lule Sami'), ('lun', 'Lunda'), ('luo', 'Luo (Kenya and Tanzania)'), ('lus', 'Lushai'), ('ltz', 'Luxembourgish; Letzeburgesch'), ('mkd', 'Macedonian'), ('mad', 'Madurese'), ('mag', 'Magahi'), ('mai', 'Maithili'), ('mak', 'Makasar'), ('mlg', 'Malagasy'), ('msa', 'Malay'), ('mal', 'Malayalam'), ('mlt', 'Maltese'), ('mnc', 'Manchu'), ('mdr', 'Mandar'), ('man', 'Mandingo'), ('mni', 'Manipuri'), ('mno', 'Manobo languages'), ('glv', 'Manx'), ('mri', 'Maori'), ('arn', 'Mapudungun; Mapuche'), ('mar', 'Marathi'), ('chm', 'Mari'), ('mah', 'Marshallese'), ('mwr', 'Marwari'), ('mas', 'Masai'), ('myn', 'Mayan languages'), ('men', 'Mende'), ('mic', "Mi'kmaq; Micmac"), ('min', 'Minangkabau'), ('mwl', 'Mirandese'), ('moh', 'Mohawk'), ('mdf', 'Moksha'), ('mol', 'Moldavian; Moldovan'), ('mkh', 'Mon-Khmer languages'), ('lol', 'Mongo'), ('mon', 'Mongolian'), ('mos', 'Mossi'), ('mul', 'Multiple languages'), ('mun', 'Munda languages'), ('nqo', "N'Ko"), ('nah', 'Nahuatl languages'), ('nau', 'Nauru'), ('nav', 'Navajo; Navaho'), ('nde', 'Ndebele, North; North Ndebele'), ('nbl', 'Ndebele, South; South Ndebele'), ('ndo', 'Ndonga'), ('nap', 'Neapolitan'), ('new', 'Nepal Bhasa; Newari'), ('nep', 'Nepali'), ('nia', 'Nias'), ('nic', 'Niger-Kordofanian languages'), ('ssa', 'Nilo-Saharan languages'), ('niu', 'Niuean'), ('zxx', 'No linguistic content; Not applicable'), ('nog', 'Nogai'), ('non', 'Norse, Old'), ('nai', 'North American Indian languages'), ('frr', 'Northern Frisian'), ('sme', 'Northern Sami'), ('nor', 'Norwegian'), ('nno', 'Norwegian Nynorsk; Nynorsk, Norwegian'), ('nub', 'Nubian languages'), ('nym', 'Nyamwezi'), ('nyn', 'Nyankole'), ('nyo', 'Nyoro'), ('nzi', 'Nzima'), ('oci', 'Occitan (post 1500)'), ('arc', 'Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)'), ('oji', 'Ojibwa'), ('ori', 'Oriya'), ('orm', 'Oromo'), ('osa', 'Osage'), ('oss', 'Ossetian; Ossetic'), ('oto', 'Otomian languages'), ('pal', 'Pahlavi'), ('pau', 'Palauan'), ('pli', 'Pali'), ('pam', 'Pampanga; Kapampangan'), ('pag', 'Pangasinan'), ('pan', 'Panjabi; Punjabi'), ('pap', 'Papiamento'), ('paa', 'Papuan languages'), ('nso', 'Pedi; Sepedi; Northern Sotho'), ('fas', 'Persian'), ('peo', 'Persian, Old (ca. 600-400 B.C.)'), ('phi', 'Philippine languages'), ('phn', 'Phoenician'), ('pon', 'Pohnpeian'), ('pol', 'Polish'), ('por', 'Portuguese'), ('pra', 'Prakrit languages'), ('pro', 'Provençal, Old (to 1500); Occitan, Old (to 1500)'), ('pus', 'Pushto; Pashto'), ('que', 'Quechua'), ('raj', 'Rajasthani'), ('rap', 'Rapanui'), ('rar', 'Rarotongan; Cook Islands Maori'), ('qaa-qtz', 'Reserved for local use'), ('roa', 'Romance languages'), ('ron', 'Romanian'), ('roh', 'Romansh'), ('rom', 'Romany'), ('run', 'Rundi'), ('rus', 'Russian'), ('sal', 'Salishan languages'), ('sam', 'Samaritan Aramaic'), ('smi', 'Sami languages'), ('smo', 'Samoan'), ('sad', 'Sandawe'), ('sag', 'Sango'), ('san', 'Sanskrit'), ('sat', 'Santali'), ('srd', 'Sardinian'), ('sas', 'Sasak'), ('sco', 'Scots'), ('sel', 'Selkup'), ('sem', 'Semitic languages'), ('srp', 'Serbian'), ('srr', 'Serer'), ('shn', 'Shan'), ('sna', 'Shona'), ('iii', 'Sichuan Yi; Nuosu'), ('scn', 'Sicilian'), ('sid', 'Sidamo'), ('sgn', 'Sign Languages'), ('bla', 'Siksika'), ('snd', 'Sindhi'), ('sin', 'Sinhala; Sinhalese'), ('sit', 'Sino-Tibetan languages'), ('sio', 'Siouan languages'), ('sms', 'Skolt Sami'), ('den', 'Slave (Athapascan)'), ('sla', 'Slavic languages'), ('slk', 'Slovak'), ('slv', 'Slovenian'), ('sog', 'Sogdian'), ('som', 'Somali'), ('son', 'Songhai languages'), ('snk', 'Soninke'), ('wen', 'Sorbian languages'), ('sot', 'Sotho, Southern'), ('sai', 'South American Indian languages'), ('alt', 'Southern Altai'), ('sma', 'Southern Sami'), ('spa', 'Spanish; Castilian'), ('srn', 'Sranan Tongo'), ('zgh', 'Standard Moroccan Tamazight'), ('suk', 'Sukuma'), ('sux', 'Sumerian'), ('sun', 'Sundanese'), ('sus', 'Susu'), ('swa', 'Swahili'), ('ssw', 'Swati'), ('swe', 'Swedish'), ('gsw', 'Swiss German; Alemannic; Alsatian'), ('syr', 'Syriac'), ('tgl', 'Tagalog'), ('tah', 'Tahitian'), ('tai', 'Tai languages'), ('tgk', 'Tajik'), ('tmh', 'Tamashek'), ('tam', 'Tamil'), ('tat', 'Tatar'), ('tel', 'Telugu'), ('ter', 'Tereno'), ('tet', 'Tetum'), ('tha', 'Thai'), ('bod', 'Tibetan'), ('tig', 'Tigre'), ('tir', 'Tigrinya'), ('tem', 'Timne'), ('tiv', 'Tiv'), ('tli', 'Tlingit'), ('tpi', 'Tok Pisin'), ('tkl', 'Tokelau'), ('tog', 'Tonga (Nyasa)'), ('ton', 'Tonga (Tonga Islands)'), ('tsi', 'Tsimshian'), ('tso', 'Tsonga'), ('tsn', 'Tswana'), ('tum', 'Tumbuka'), ('tup', 'Tupi languages'), ('tur', 'Turkish'), ('ota', 'Turkish, Ottoman (1500-1928)'), ('tuk', 'Turkmen'), ('tvl', 'Tuvalu'), ('tyv', 'Tuvinian'), ('twi', 'Twi'), ('udm', 'Udmurt'), ('uga', 'Ugaritic'), ('uig', 'Uighur; Uyghur'), ('ukr', 'Ukrainian'), ('umb', 'Umbundu'), ('mis', 'Uncoded languages'), ('und', 'Undetermined'), ('hsb', 'Upper Sorbian'), ('urd', 'Urdu'), ('uzb', 'Uzbek'), ('vai', 'Vai'), ('ven', 'Venda'), ('vie', 'Vietnamese'), ('vol', 'Volapük'), ('vot', 'Votic'), ('wak', 'Wakashan languages'), ('wln', 'Walloon'), ('war', 'Waray'), ('was', 'Washo'), ('cym', 'Welsh'), ('fry', 'Western Frisian'), ('wal', 'Wolaitta; Wolaytta'), ('wol', 'Wolof'), ('xho', 'Xhosa'), ('sah', 'Yakut'), ('yao', 'Yao'), ('yap', 'Yapese'), ('yid', 'Yiddish'), ('yor', 'Yoruba'), ('ypk', 'Yupik languages'), ('znd', 'Zande languages'), ('zap', 'Zapotec'), ('zza', 'Zaza; Dimili; Dimli; Kirdki; Kirmanjki; Zazaki'), ('zen', 'Zenaga'), ('zha', 'Zhuang; Chuang'), ('zul', 'Zulu'), ('zun', 'Zuni')], max_length=10)), + ('acronym_for', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='acronyms', to='core.organization', help_text='This name is a ROR-provided acronym.')), + ('alias_for', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='aliases', to='core.organization', help_text='This name is a less preferred ROR-provided alternative.')), + ('custom_label_for', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='custom_label', to='core.organization', help_text='This name is a custom label entered by the end user. Only exists in Janeway, independent of ROR.')), + ('label_for', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='labels', to='core.organization', help_text='This name is a preferred ROR-provided alternative, often in a different language from the ROR display name.')), + ('ror_display_for', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='ror_display', to='core.organization', help_text='This name is a preferred ROR-provided display name.')), + ], + ), + migrations.CreateModel( + name='Affiliation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(blank=True, max_length=300, verbose_name='Title, position, or role')), + ('department', models.CharField(blank=True, max_length=300, verbose_name='Department, unit, or team')), + ('is_primary', models.BooleanField(default=False, help_text='Each account can have one primary affiliation')), + ('start', models.DateField(blank=True, null=True, verbose_name='Start date')), + ('end', models.DateField(blank=True, null=True, help_text='Leave empty for a current affiliation', verbose_name='End date')), + ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('frozen_author', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='submission.frozenauthor')), + ('organization', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='core.organization')), + ('preprint_author', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='repository.preprintauthor')), + ], + options={ + 'ordering': ['is_primary', '-pk'], + }, + ), + migrations.AddConstraint( + model_name='organization', + constraint=models.UniqueConstraint(condition=models.Q(('ror__exact', ''), _negated=True), fields=('ror',), name='filled_unique'), + ), + ] diff --git a/src/core/migrations/0101_migrate_affiliation_institution.py b/src/core/migrations/0101_migrate_affiliation_institution.py new file mode 100644 index 0000000000..b9a256482b --- /dev/null +++ b/src/core/migrations/0101_migrate_affiliation_institution.py @@ -0,0 +1,114 @@ +# Generated by Django 4.2.16 on 2025-01-17 16:21 + +from django.db import migrations +from django.db.models import Q + + +def get_or_create_location_with_country_only(apps, old_country=None): + if not old_country: + return None + + Location = apps.get_model("core", "Location") + location, created = Location.objects.get_or_create( + name='', + country=old_country, + ) + return location + + +def create_organization(apps, old_institution=None, old_country=None): + if not old_institution: + return None + + Organization = apps.get_model("core", "Organization") + OrganizationName = apps.get_model("core", "OrganizationName") + + organization = Organization.objects.create() + location = get_or_create_location_with_country_only(apps, old_country) + if location: + organization.locations.add(location) + OrganizationName.objects.create( + value=old_institution, + custom_label_for=organization, + ) + return organization + + +def create_affiliation( + apps, + old_institution, + old_department, + old_country, + account=None, + frozen_author=None, + preprint_author=None, +): + Affiliation = apps.get_model("core", "Affiliation") + organization = create_organization(apps, old_institution, old_country) + + # Create or update the actual affiliation if the associated + # account / frozen author / preprint author has been saved already + affiliation = Affiliation.objects.create( + account=account, + frozen_author=frozen_author, + preprint_author=preprint_author, + organization=organization, + department=old_department, + is_primary=True, + ) + return affiliation + + +def migrate_affiliation_institution(apps, schema_editor): + Account = apps.get_model("core", "Account") + FrozenAuthor = apps.get_model("submission", "FrozenAuthor") + PreprintAuthor = apps.get_model("repository", "PreprintAuthor") + + for account in Account.objects.filter( + ~Q(institution__exact='') + | ~Q(department__exact='') + ): + create_affiliation( + apps, + account.institution, + account.department, + account.country, + account=account, + ) + + for frozen_author in FrozenAuthor.objects.filter( + ~Q(institution__exact='') + | ~Q(department__exact='') + ): + create_affiliation( + apps, + frozen_author.institution, + frozen_author.department, + frozen_author.country, + frozen_author=frozen_author, + ) + + for preprint_author in PreprintAuthor.objects.filter( + affiliation__isnull=True, + ): + create_affiliation( + apps, + preprint_author.affiliation, + '', + None, + preprint_author=preprint_author, + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0100_location_organization_affiliation'), + ] + + operations = [ + migrations.RunPython( + migrate_affiliation_institution, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/src/core/migrations/0102_remove_account_country_affiliation_organization.py b/src/core/migrations/0102_remove_account_country_affiliation_organization.py new file mode 100644 index 0000000000..08c95a07f9 --- /dev/null +++ b/src/core/migrations/0102_remove_account_country_affiliation_organization.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.14 on 2024-07-26 13:26 + +import core.models +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('submission', '0080_remove_frozenauthor_country_and_more'), + ('core', '0101_migrate_affiliation_institution'), + ] + + operations = [ + migrations.RemoveField( + model_name='account', + name='country', + ), + migrations.RemoveField( + model_name='account', + name='department', + ), + migrations.RemoveField( + model_name='account', + name='institution', + ), + ] diff --git a/src/core/model_utils.py b/src/core/model_utils.py index 48733dbe7d..a1d255bac2 100644 --- a/src/core/model_utils.py +++ b/src/core/model_utils.py @@ -14,6 +14,7 @@ from django import forms from django.apps import apps from django.contrib import admin +from django.core.paginator import EmptyPage, Paginator from django.contrib.postgres.lookups import SearchLookup as PGSearchLookup from django.contrib.postgres.search import ( SearchVector as DjangoSearchVector, @@ -739,3 +740,18 @@ def formfield(self, **kwargs): @property def NotImplementedField(self): raise NotImplementedError + + +class SafePaginator(Paginator): + """ + A paginator for avoiding an uncaught exception + caused by passing a page parameter that is out of range. + """ + def validate_number(self, number): + try: + return super().validate_number(number) + except EmptyPage: + if number > 1: + return self.num_pages + else: + raise diff --git a/src/core/models.py b/src/core/models.py index 3d19f64893..e928e32741 100644 --- a/src/core/models.py +++ b/src/core/models.py @@ -3,7 +3,9 @@ __license__ = "AGPL v3" __maintainer__ = "Birkbeck Centre for Technology and Publishing" +from decimal import Decimal import os +import re import uuid import statistics import json @@ -12,6 +14,8 @@ import pytz from hijack.signals import hijack_started, hijack_ended import warnings +import tqdm +import zipfile from bs4 import BeautifulSoup from django.conf import settings @@ -26,7 +30,6 @@ from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.contrib.postgres.search import SearchVector, SearchVectorField -from django.core.exceptions import ValidationError from django.core.validators import validate_email from django.utils.translation import gettext_lazy as _ from django.db.models.signals import post_save @@ -50,6 +53,8 @@ ) from review import models as review_models from copyediting import models as copyediting_models +from repository import models as repository_models +from utils.models import RORImportError from submission import models as submission_models from utils.logger import get_logger from utils import logic as utils_logic @@ -230,6 +235,28 @@ def create_superuser(self, email, password, **kwargs): def get_queryset(self): return AccountQuerySet(self.model) + def get_or_create(self, defaults=None, **kwargs): + """ + Backwards-compatible override for affiliation-related kwargs + """ + # check for deprecated fields related to affiliation + institution = kwargs.pop('institution', '') + department = kwargs.pop('department', '') + country = kwargs.pop('country', '') + + account, created = super().get_or_create(defaults, **kwargs) + + # create or update affiliation + if institution or department or country: + Affiliation.naive_get_or_create( + institution=institution, + department=department, + country=country, + account=account, + ) + + return account, created + class Account(AbstractBaseUser, PermissionsMixin): email = PGCaseInsensitiveEmailField(unique=True, verbose_name=_('Email')) @@ -274,18 +301,6 @@ class Account(AbstractBaseUser, PermissionsMixin): verbose_name=_('Biography'), ) orcid = models.CharField(max_length=40, null=True, blank=True, verbose_name=_('ORCiD')) - institution = models.CharField( - max_length=1000, - blank=True, - verbose_name=_('Institution'), - validators=[plain_text_validator], - ) - department = models.CharField( - max_length=300, - blank=True, - verbose_name=_('Department'), - validators=[plain_text_validator], - ) twitter = models.CharField(max_length=300, null=True, blank=True, verbose_name=_('Twitter Handle')) facebook = models.CharField(max_length=300, null=True, blank=True, verbose_name=_('Facebook Handle')) linkedin = models.CharField(max_length=300, null=True, blank=True, verbose_name=_('Linkedin Profile')) @@ -300,13 +315,6 @@ class Account(AbstractBaseUser, PermissionsMixin): verbose_name=_("Signature"), ) interest = models.ManyToManyField('Interest', null=True, blank=True) - country = models.ForeignKey( - Country, - null=True, - blank=True, - verbose_name=_('Country'), - on_delete=models.SET_NULL, - ) preferred_timezone = DynamicChoiceField( max_length=300, null=True, blank=True, choices=tuple(), @@ -413,13 +421,40 @@ def initials(self): else: return 'N/A' - def affiliation(self): - if self.institution and self.department: - return "{}, {}".format(self.department, self.institution) - elif self.institution: - return self.institution - else: - return '' + def affiliation(self, obj=False, date=None): + return Affiliation.get_primary(account=self, obj=obj, date=date) + + @property + def affiliations(self): + return Affiliation.objects.filter(account=self).order_by('-is_primary') + + @property + def institution(self): + affil = self.affiliation(obj=True) + return str(affil.organization) if affil else '' + + @institution.setter + def institution(self, value): + Affiliation.naive_get_or_create(institution=value, account=self) + + @property + def department(self): + affil = self.affiliation(obj=True) + return str(affil.department) if affil else '' + + @department.setter + def department(self, value): + Affiliation.naive_set_primary_department(value, account=self) + + @property + def country(self): + affil = self.affiliation(obj=True) + organization = affil.organization if affil else None + return str(organization.country) if organization else None + + @country.setter + def country(self, value): + Affiliation.naive_get_or_create(country=value, account=self) def active_reviews(self): return review_models.ReviewAssignment.objects.filter( @@ -546,6 +581,18 @@ def is_preprint_editor(self, request): def is_reader(self, request): return self.check_role(request.journal, 'reader', staff_override=False) + def snapshot_affiliations(self, frozen_author): + """ + Delete any outdated affiliations on the frozen author and then + assign copies of account affiliations to the frozen author. + """ + frozen_author.affiliations.delete() + for affiliation in self.affiliations: + affiliation.pk = None + affiliation.account = None + affiliation.frozen_author = frozen_author + affiliation.save() + def snapshot_self(self, article, force_update=True): frozen_dict = { 'name_prefix': self.name_prefix, @@ -553,8 +600,6 @@ def snapshot_self(self, article, force_update=True): 'middle_name': self.middle_name, 'last_name': self.last_name, 'name_suffix': self.suffix, - 'institution': self.institution, - 'department': self.department, 'display_email': True if self == article.correspondence_author else False, } @@ -564,6 +609,7 @@ def snapshot_self(self, article, force_update=True): for k, v in frozen_dict.items(): setattr(frozen_author, k, v) frozen_author.save() + self.snapshot_affiliations(frozen_author) else: try: @@ -576,11 +622,13 @@ def snapshot_self(self, article, force_update=True): defaults={'order': order_integer} ) - submission_models.FrozenAuthor.objects.get_or_create( + frozen_author, created = submission_models.FrozenAuthor.objects.get_or_create( author=self, article=article, defaults=dict(order=order_object.order, **frozen_dict) ) + if created: + self.snapshot_affiliations(frozen_author) def frozen_author(self, article): try: @@ -1906,3 +1954,635 @@ def log_hijack_ended(sender, hijacker, hijacked, request, **kwargs): hijack_started.connect(log_hijack_started) hijack_ended.connect(log_hijack_ended) + + +class OrganizationName(models.Model): + value = models.CharField( + max_length=200, + verbose_name="Organization name", + blank=True, + ) + ror_display_for = models.OneToOneField( + 'Organization', + on_delete=models.CASCADE, + related_name='ror_display', + blank=True, + null=True, + help_text="This name is a preferred ROR-provided display name.", + ) + label_for = models.ForeignKey( + 'Organization', + on_delete=models.CASCADE, + related_name='labels', + blank=True, + null=True, + help_text="This name is a preferred ROR-provided alternative, " + "often in a different language from the ROR display name.", + ) + custom_label_for = models.OneToOneField( + 'Organization', + on_delete=models.CASCADE, + related_name='custom_label', + blank=True, + null=True, + help_text="This name is a custom label entered by the end user. " + "Only exists in Janeway, independent of ROR.", + ) + alias_for = models.ForeignKey( + 'Organization', + on_delete=models.CASCADE, + related_name='aliases', + blank=True, + null=True, + help_text="This name is a less preferred ROR-provided alternative.", + ) + acronym_for = models.ForeignKey( + 'Organization', + on_delete=models.CASCADE, + related_name='acronyms', + blank=True, + null=True, + help_text="This name is a ROR-provided acronym.", + ) + language = models.CharField( + max_length=10, + blank=True, + choices=submission_models.LANGUAGE_CHOICES, + ) + + def __str__(self): + return self.value or f'[Unnamed organization {self.pk}]' + + +def validate_ror(url): + ror = os.path.split(url)[-1] + ror_regex = '^0[a-hj-km-np-tv-z|0-9]{6}[0-9]{2}$' + if not re.match(ror_regex, ror): + raise ValidationError(f'{ror} is not a valid ROR identifier') + + +class Organization(models.Model): + + class RORStatus(models.TextChoices): + ACTIVE = 'active', _('Active') + INACTIVE = 'inactive', _('Inactive') + WITHDRAWN = 'withdrawn', _('Withdrawn') + UNKNOWN = 'unknown', _('Unknown') + + ror = models.URLField( + blank=True, + validators=[validate_ror], + verbose_name='ROR', + help_text='Research Organization Registry identifier (URL)', + ) + ror_status = models.CharField( + blank=True, + max_length=10, + choices=RORStatus.choices, + default=RORStatus.UNKNOWN, + ) + ror_record_timestamp = models.CharField( + max_length=10, + blank=True, + help_text='The admin.last_modified.date string from ROR data', + ) + website = models.CharField( + blank=True, + max_length=2000, + ) + locations = models.ManyToManyField( + 'Location', + blank=True, + null=True, + ) + + class Meta: + ordering = ['ror_display__value'] + constraints = [ + models.UniqueConstraint( + fields=['ror'], + condition=~models.Q(ror__exact=''), + name='filled_unique', + ) + ] + + def __str__(self): + elements = [ + str(self.name) if self.name else '', + str(self.location) if self.location else '', + ] + return ', '.join([element for element in elements if element]) + + @property + def name(self): + """ + Return the OrganizationName that ROR uses for display, or if none, + the first one that was manually entered, or if none + the first one designated by ROR as a label. + Can be expanded in future to support choosing a label by language. + """ + try: + return self.ror_display + except Organization.ror_display.RelatedObjectDoesNotExist: + try: + return self.custom_label + except Organization.custom_label.RelatedObjectDoesNotExist: + return self.labels.first() + + @property + def location(self): + """ + Return the first location. + """ + return self.locations.first() if self.locations else None + + @property + def country(self): + """ + Return the country of the first location. + """ + return self.location.country if self.location else None + + @property + def names(self): + """ + All names. + """ + return OrganizationName.objects.filter( + models.Q(ror_display_for=self) | + models.Q(custom_label_for=self) | + models.Q(label_for=self) | + models.Q(alias_for=self) | + models.Q(acronym_for=self), + ) + + @property + def also_known_as(self): + """ + All names excluding the ROR display name. + """ + return self.names.exclude(ror_display_for=self) + + @classmethod + def naive_get_or_create( + cls, + institution='', + country='', + account=None, + frozen_author=None, + preprint_author=None, + ): + """ + Backwards-compatible API for finding a matching organization, + or creating one along with a location that just records country. + Intended for use in batch importers where ROR data is not available + in the data being imported. + Does not support ROR ids, ROR name types, or geonames locations. + + :type institution: str + :param country: ISO-3166-1 alpha-2 country code or Janeway Country object + :type country: str or core.models.Country + :type account: core.models.Account + :type frozen_author: submission.models.FrozenAuthor + :type preprint_author: repository.models.PreprintAuthor + """ + + created = False + # Is there a single exact match in the + # canonical name data from ROR (e.g. labels)? + try: + organization = cls.objects.get(labels__value=institution) + except (cls.DoesNotExist, cls.MultipleObjectsReturned): + # Or maybe one in the past or alternate + # name data from ROR (e.g. aliases)? + try: + organization = cls.objects.get(aliases__value=institution) + except (cls.DoesNotExist, cls.MultipleObjectsReturned): + # Or maybe a primary affiliation has already been + # entered without a ROR for this + # account / frozen author / preprint author? + try: + organization = cls.objects.get( + affiliation__is_primary=True, + affiliation__account=account, + affiliation__frozen_author=frozen_author, + affiliation__preprint_author=preprint_author, + ror__exact='', + ) + except (cls.DoesNotExist, cls.MultipleObjectsReturned): + # Otherwise, create a naive, disconnected record. + organization = cls.objects.create() + created = True + + # Get or create a custom label + if institution: + organization_name, _created = OrganizationName.objects.get_or_create( + custom_label_for=organization, + ) + organization_name.value = institution + organization_name.save() + + # Set the country as a location + if country and not isinstance(country, Country): + try: + country = Country.objects.get(code=country) + except Country.DoesNotExist: + country = '' + + if country: + location, _created = Location.objects.get_or_create( + name='', + country=country, + ) + organization.locations.add(location) + + return organization, created + + def deduplicate_to_ror_record(self): + """ + Tries to merge with another organization that has ROR data, + if the custom label is an exact match with a preferred ROR label, + and if the country matches. + """ + if self.ror: + logger.warning( + "Cannot deduplicate Organization {{self.id}}: ROR present" + ) + return + if not self.custom_label: + logger.warning( + "Cannot deduplicate Organization {{self.id}}: No custom label" + ) + return + + if self.location: + matches = Organization.objects.filter( + labels__value=self.custom_label.value, + locations__country=self.location.country, + ) + else: + matches = Organization.objects.filter( + labels=self.custom_label, + ) + if matches.exists() and matches.count() == 1: + Affiliation.objects.filter( + organization=self, + ).update( + organization=matches.first(), + ) + self.delete() + + @classmethod + def create_from_ror_record(cls, record): + """ + Creates one organization object in Janeway from a ROR JSON record, + using version 2 of the ROR Schema. + See https://ror.readme.io/v2/docs/data-structure + """ + organization, created = cls.objects.get_or_create( + ror=record.get('id', ''), + ) + organization.ror_status = record.get('status', cls.RORStatus.UNKNOWN) + last_modified = record.get("admin", {}).get("last_modified", {}) + organization.ror_record_timestamp = last_modified.get("date", "") + for link in record.get('links', []): + if link.get('type') == 'website': + organization.website = link.get('value', '') + break + organization.save() + + for name in record.get('names'): + kwargs = {} + kwargs['value'] = name.get('value', '') + if name.get('lang'): + kwargs['language'] = name.get('language', '') + if 'ror_display' in name.get('types'): + kwargs['ror_display_for'] = organization + if 'label' in name.get('types'): + kwargs['label_for'] = organization + if 'alias' in name.get('types'): + kwargs['alias_for'] = organization + if 'acronym' in name.get('types'): + kwargs['acronym_for'] = organization + OrganizationName.objects.get_or_create(**kwargs) + + for record_location in record.get('locations'): + geonames_id = record_location.get('geonames_id') + if geonames_id: + location, created = Location.objects.get_or_create( + geonames_id=geonames_id, + ) + else: + location = Location.objects.create() + created = True + if created: + details = record_location.get('geonames_details', {}) + location.name = details.get('name', '') + country, created = Country.objects.get_or_create( + code=details.get('country_code', ''), + ) + location.country = country + location.latitude = Decimal(details.get('lat')) + location.longitude = Decimal(details.get('lng')) + location.save() + organization.locations.add(location) + + @classmethod + def import_ror_batch(cls, ror_import, limit=0): + """ + Opens a previously downloaded data dump from + ROR's Zenodo endpoint, processes the records, + and records errors for exceptions raised during creation. + https://ror.readme.io/v2/docs/data-dump + """ + organizations = cls.objects.exclude(ror="") + num_errors_before = RORImportError.objects.count() + with zipfile.ZipFile(ror_import.zip_path, mode='r') as zip_ref: + for file_info in zip_ref.infolist(): + if file_info.filename.endswith('v2.json'): + json_string = zip_ref.read(file_info).decode(encoding="utf-8") + data = json.loads(json_string) + data = ror_import.filter_new_records(data, organizations) + if len(data) == 0: + ror_import.status = ror_import.RORImportStatus.UNNECESSARY + if limit: + data = data[:limit] + description = f"Importing {len(data)} ROR records" + for item in tqdm.tqdm(data, desc=description): + try: + cls.create_from_ror_record(item) + except Exception as error: + message = f'{type(error)}: {error}\n{json.dumps(item)}' + RORImportError.objects.create( + ror_import=ror_import, + message=message, + ) + num_errors_after = RORImportError.objects.count() + if num_errors_after > num_errors_before: + logger.warn( + f'ROR import errors logged: { num_errors_after - num_errors_before }' + ) + + +class Affiliation(models.Model): + account = models.ForeignKey( + Account, + on_delete=models.CASCADE, + blank=True, + null=True, + ) + frozen_author = models.ForeignKey( + submission_models.FrozenAuthor, + on_delete=models.CASCADE, + blank=True, + null=True, + ) + preprint_author = models.ForeignKey( + repository_models.PreprintAuthor, + on_delete=models.CASCADE, + blank=True, + null=True, + ) + title = models.CharField( + blank=True, + max_length=300, + verbose_name=_('Title, position, or role'), + ) + department = models.CharField( + blank=True, + max_length=300, + verbose_name=_('Department, unit, or team'), + ) + organization = models.ForeignKey( + 'Organization', + blank=True, + null=True, + on_delete=models.CASCADE, + ) + is_primary = models.BooleanField( + default=False, + help_text="Each account can have one primary affiliation", + ) + start = models.DateField( + blank=True, + null=True, + verbose_name="Start date", + ) + end = models.DateField( + blank=True, + null=True, + verbose_name="End date", + help_text="Leave empty for a current affiliation", + ) + + class Meta: + ordering = ['is_primary', '-pk'] + + def title_department(self): + elements = [ + self.title, + self.department, + ] + return ', '.join([element for element in elements if element]) + + def __str__(self): + elements = [ + self.title, + self.department, + str(self.organization) if self.organization else '', + ] + return ', '.join([element for element in elements if element]) + + + @property + def is_current(self): + if self.start and self.start > timezone.now(): + return False + if self.end and self.end < timezone.now(): + return False + return True + + @classmethod + def set_primary_if_first(cls, obj): + other_affiliations = cls.objects.filter( + account=obj.account, + frozen_author=obj.frozen_author, + preprint_author=obj.preprint_author, + ).exclude( + pk=obj.pk + ).exists() + if not other_affiliations: + obj.is_primary = True + + @classmethod + def keep_is_primary_unique(cls, obj): + if obj.is_primary: + cls.objects.filter( + is_primary=True, + account=obj.account, + frozen_author=obj.frozen_author, + preprint_author=obj.preprint_author, + ).exclude( + pk=obj.pk + ).update( + is_primary=False, + ) + + @classmethod + def get_primary( + cls, + account=None, + frozen_author=None, + preprint_author=None, + obj=False, + date=None, + ): + """ + Get the primary affiliation, or if none, + the current affiliation with the most recent start date, or if none, + the affiliation with the highest pk, or if none, + an empty string. + :param obj: whether to return a Python object + :param date: the date relative to which to query + """ + person = account or frozen_author or preprint_author + if not person: + return None if obj else '' + if not person.affiliation_set.exists(): + return None if obj else '' + if date: + affils_with_at_least_one_date = person.affiliation_set.exclude( + start__isnull=True, + end__isnull=True, + ) + if affils_with_at_least_one_date: + affil = affils_with_at_least_one_date.exclude( + models.Q(start__gte=date) | models.Q(end__lte=date) + ).first() + return affil if obj else str(affil) + try: + affil = person.affiliation_set.get(is_primary=True) + return affil if obj else str(affil) + except Affiliation.DoesNotExist: + affil = person.affiliation_set.first() + return affil if obj else str(affil) + + @classmethod + def naive_get_or_create( + cls, + institution='', + department='', + country='', + account=None, + frozen_author=None, + preprint_author=None, + ): + """ + Backwards-compatible API for setting affiliation from unstructured text. + Intended for use in batch importers where ROR data is not available. + Does not support ROR ids, multiple affiliations, or start or end dates. + When possible, include department and country to create unified records. + Only include one of author, frozen_author, or preprint_author. + + :param institution: the uncontrolled organization name as a string + :type institution: str + :type department: str + :param country: ISO-3166-1 alpha-2 country code or Janeway Country object + :type country: str or core.models.Country + :type account: core.models.Account + :type frozen_author: submission.models.FrozenAuthor + :type preprint_author: repository.models.PreprintAuthor + """ + organization, _created = Organization.naive_get_or_create( + institution=institution, + country=country, + account=account, + frozen_author=frozen_author, + preprint_author=preprint_author, + ) + + # Create or update the actual affiliation if the associated + # account / frozen author / preprint author has been saved already + try: + affiliation, created = Affiliation.objects.get_or_create( + account=account, + frozen_author=frozen_author, + preprint_author=preprint_author, + organization=organization, + department=department, + ) + affiliation.is_primary = True + affiliation.save() + except ValueError: + logger.warn('The affiliation could not be created.') + affiliation = None + created = False + return affiliation, created + + @classmethod + def naive_set_primary_department( + cls, + value, + account=None, + frozen_author=None, + ): + """ + Backwards-compatible API for setting department names in isolation. + It is better to use Affiliation.naive_get_or_create where department + is set togther with institution and country. + Does not support ORCIDs, RORs or multiple affiliations. + """ + # Create or update an affiliation if the associated + # account / frozen author / preprint author has been saved already + try: + affiliation, _created = Affiliation.objects.get_or_create( + account=account, + frozen_author=frozen_author, + is_primary=True, + ) + affiliation.department = value + affiliation.save() + except ValueError: + logger.warn('The department could not be set.') + + def save(self, *args, **kwargs): + self.set_primary_if_first(self) + self.keep_is_primary_unique(self) + super().save(*args, **kwargs) + + +class Location(models.Model): + name = models.CharField( + max_length=200, + help_text="City or place name", + blank=True, + ) + country = models.ForeignKey( + Country, + blank=True, + null=True, + on_delete=models.SET_NULL, + ) + latitude = models.DecimalField( + max_digits=9, + decimal_places=6, + blank=True, + null=True, + ) + longitude = models.DecimalField( + max_digits=9, + decimal_places=6, + blank=True, + null=True, + ) + geonames_id = models.IntegerField( + blank=True, + null=True, + ) + + def __str__(self): + elements = [ + self.name if self.name else '', + str(self.country) if self.country else '', + ] + return ', '.join([element for element in elements if element]) diff --git a/src/core/tests/test_app.py b/src/core/tests/test_app.py index 0e8fbc6ecf..31d2fca332 100755 --- a/src/core/tests/test_app.py +++ b/src/core/tests/test_app.py @@ -206,7 +206,6 @@ def test_orcid_registration(self, record_mock): self.assertEqual(response.status_code, 200) self.assertContains(response, "Campbell") self.assertContains(response, "Kasey") - self.assertContains(response, "Elk Valley University") self.assertContains(response, "campbell@evu.edu") self.assertNotContains(response, "Register with ORCiD") self.assertContains(response, "http://sandbox.orcid.org/0000-0000-0000-0000") diff --git a/src/core/tests/test_models.py b/src/core/tests/test_models.py index e191a24c66..eae808b6a4 100644 --- a/src/core/tests/test_models.py +++ b/src/core/tests/test_models.py @@ -1,9 +1,10 @@ -from datetime import timedelta +from datetime import date, timedelta +from unittest.mock import patch from django.core.files.uploadedfile import SimpleUploadedFile from django.db import IntegrityError from django.http import HttpRequest, QueryDict -from django.forms import Form +from django.forms import Form, ValidationError from django.test import TestCase from django.utils import timezone from freezegun import freeze_time @@ -17,6 +18,7 @@ from journal import models as journal_models from utils.testing import helpers from submission import models as submission_models +from repository import models as repository_models FROZEN_DATETIME_20210101 = timezone.make_aware(timezone.datetime(2021, 1, 1, 0, 0, 0)) FROZEN_DATETIME_20210102 = timezone.make_aware(timezone.datetime(2021, 1, 2, 0, 0, 0)) @@ -397,3 +399,494 @@ def test_search_model_admin(self): self.account, results, ) + + +class TestOrganizationModels(TestCase): + + @classmethod + def setUpTestData(cls): + cls.press = helpers.create_press() + cls.repo_manager = helpers.create_user( + 'xulz5vepggxdvo8ngirw@example.org' + ) + cls.repository, cls.subject = helpers.create_repository( + cls.press, + [cls.repo_manager], + [], + domain='odhstswzfesoyhhjywzk.example.org', + ) + cls.country_gb = models.Country.objects.create( + code='GB', + name='United Kingdom', + ) + cls.country_us = models.Country.objects.create( + code='US', + name='United States', + ) + cls.location_london = models.Location.objects.create( + name='London', + country=cls.country_gb, + latitude=51.50853, + longitude=-0.12574, + ) + cls.location_farnborough = models.Location.objects.create( + name='Farnborough', + country=cls.country_gb, + latitude=51.29, + longitude=-0.75, + ) + cls.location_uk_legacy = models.Location.objects.create( + # Before integrating ROR we used country-wide locations + # with no geonames ID or coordinates + country=cls.country_gb, + ) + cls.organization_bbk = models.Organization.objects.create( + ror='https://ror.org/02mb95055', + ) + cls.name_bbk_uol = models.OrganizationName.objects.create( + value='Birkbeck, University of London', + language='en', + ror_display_for=cls.organization_bbk, + label_for=cls.organization_bbk, + ) + cls.name_bbk_cy = models.OrganizationName.objects.create( + value='Birkbeck, Prifysgol Llundain', + language='cy', + label_for=cls.organization_bbk, + ) + cls.name_bbk_custom = models.OrganizationName.objects.create( + value='Birkbeck', + custom_label_for=cls.organization_bbk, + ) + cls.name_bbk_college = models.OrganizationName.objects.create( + value='Birkbeck College', + language='en', + alias_for=cls.organization_bbk, + ) + cls.organization_bbk.locations.add(cls.location_london) + cls.organization_rae = models.Organization.objects.create( + ror='https://ror.org/0n7v1dg93', + ) + cls.name_rae = models.OrganizationName.objects.create( + value='Royal Aircraft Establishment', + language='en', + label_for=cls.organization_rae, + ror_display_for=cls.organization_rae + ) + cls.organization_rae.locations.add(cls.location_farnborough) + cls.organization_brp = models.Organization.objects.create( + ror='https://ror.org/0w7120h04', + ) + cls.name_brp = models.OrganizationName.objects.create( + value='British Rubber Producers', + language='en', + label_for=cls.organization_brp, + ror_display_for=cls.organization_brp, + ) + cls.organization_bbk_legacy = models.Organization.objects.create( + # Before integrating ROR we used institution names with no ROR IDs + ) + cls.organization_bbk_legacy.locations.add(cls.location_uk_legacy) + cls.name_bbk_custom_legacy = models.OrganizationName.objects.create( + value='Birkbeck, University of London', + custom_label_for=cls.organization_bbk_legacy, + ) + cls.kathleen_booth = helpers.create_user( + 'ehqak6rxknzw35ih47oc@bbk.ac.uk', + first_name='Kathleen', + last_name='Booth', + ) + cls.kathleen_booth_frozen = submission_models.FrozenAuthor.objects.create( + first_name='Kathleen', + last_name='Booth', + author=cls.kathleen_booth, + frozen_email='ehqak6rxknzw35ih47oc@bbk.ac.uk', + ) + cls.preprint_one = helpers.create_preprint( + cls.repository, + cls.kathleen_booth, + cls.subject, + title='Preprint for testing affiliations', + ) + cls.kathleen_booth_preprint, _created = repository_models.PreprintAuthor.objects.get_or_create( + preprint=cls.preprint_one, + account=cls.kathleen_booth, + ) + cls.affiliation_lecturer = models.Affiliation.objects.create( + account=cls.kathleen_booth, + title='Lecturer', + department='Department of Numerical Automation', + organization=cls.organization_bbk, + is_primary=True, + start=date.fromisoformat('1952-01-01'), + end=date.fromisoformat('1962-12-31'), + ) + cls.affiliation_lecturer_frozen = models.Affiliation.objects.create( + frozen_author=cls.kathleen_booth_frozen, + title='Lecturer', + department='Department of Numerical Automation', + organization=cls.organization_bbk, + is_primary=True, + ) + cls.affiliation_lecturer_preprint = models.Affiliation.objects.create( + preprint_author=cls.kathleen_booth_preprint, + title='Lecturer', + department='Department of Numerical Automation', + organization=cls.organization_bbk, + is_primary=True, + ) + cls.affiliation_scientist = models.Affiliation.objects.create( + account=cls.kathleen_booth, + department='Research Association', + organization=cls.organization_brp, + ) + cls.affiliation_officer = models.Affiliation.objects.create( + account=cls.kathleen_booth, + title='Junior Scientific Officer', + organization=cls.organization_rae, + start=date.fromisoformat('1944-01-01'), + ) + cls.t_s_eliot = helpers.create_user( + 'gene8rahhnmmitlvqiz9@bbk.ac.uk', + first_name='Thomas', + middle_name='Stearns', + last_name='Eliot', + ) + cls.e_hobsbawm = helpers.create_user( + 'dp0dcbdgtzq4e7ml50fe@example.org', + first_name='Eric', + last_name='Hobsbawm', + ) + cls.affiliation_historian = models.Affiliation.objects.create( + account=cls.e_hobsbawm, + title='Historian', + organization=cls.organization_bbk_legacy, + ) + + return super().setUpTestData() + + def test_account_institution_getter(self): + self.assertEqual( + self.kathleen_booth.institution, + 'Birkbeck, University of London, London, United Kingdom', + ) + + def test_frozen_author_institution_getter(self): + self.assertEqual( + self.kathleen_booth_frozen.institution, + 'Birkbeck, University of London, London, United Kingdom', + ) + + def test_account_institution_setter_canonical_label(self): + self.t_s_eliot.institution = 'Birkbeck, University of London' + self.assertEqual( + self.organization_bbk, + self.t_s_eliot.affiliation(obj=True).organization, + ) + + def test_account_institution_setter_canonical_alias(self): + self.t_s_eliot.institution = 'Birkbeck College' + self.assertEqual( + self.organization_bbk, + self.t_s_eliot.affiliation(obj=True).organization, + ) + + def test_account_institution_setter_custom_overwrite(self): + self.t_s_eliot.institution = 'Birkbek' + misspelled_bbk = models.Organization.objects.get( + custom_label__value='Birkbek' + ) + self.t_s_eliot.institution = 'Birkbck' + self.assertEqual( + misspelled_bbk, + self.t_s_eliot.affiliation(obj=True).organization, + ) + + def test_account_institution_setter_custom_value(self): + self.kathleen_booth.institution = 'Birkbeck McMillan' + bbk_mcmillan = models.Organization.objects.get( + custom_label__value='Birkbeck McMillan' + ) + self.assertEqual( + bbk_mcmillan, + self.kathleen_booth.affiliation(obj=True).organization, + ) + + def test_frozen_author_institution_setter_custom_value(self): + self.kathleen_booth_frozen.institution = 'Birkbeck McMillan' + bbk_mcmillan = models.Organization.objects.get( + custom_label__value='Birkbeck McMillan' + ) + self.assertEqual( + bbk_mcmillan, + self.kathleen_booth_frozen.affiliation(obj=True).organization, + ) + + def test_account_department_getter(self): + self.assertEqual( + self.kathleen_booth.department, + 'Department of Numerical Automation', + ) + + def test_frozen_author_department_getter(self): + self.assertEqual( + self.kathleen_booth_frozen.department, + 'Department of Numerical Automation', + ) + + def test_account_department_setter(self): + self.kathleen_booth.department = 'Computer Science' + self.assertEqual( + models.Affiliation.objects.get(department='Computer Science'), + self.kathleen_booth.affiliation(obj=True), + ) + + def test_account_department_setter_updates_existing_primary(self): + self.affiliation_lecturer.is_primary = True + self.affiliation_lecturer.save() + self.kathleen_booth.department = 'Computer Science' + self.affiliation_lecturer.refresh_from_db() + self.assertEqual( + self.affiliation_lecturer.department, + self.kathleen_booth.affiliation(obj=True).department, + ) + + def test_frozen_author_department_setter(self): + self.kathleen_booth_frozen.department = 'Computer Science' + self.assertEqual( + models.Affiliation.objects.get(department='Computer Science'), + self.kathleen_booth_frozen.affiliation(obj=True), + ) + + def test_organization_name_ror_display(self): + self.assertEqual( + self.organization_bbk.name, + self.name_bbk_uol, + ) + + def test_organization_name_label(self): + self.name_bbk_custom.delete() + self.name_bbk_uol.ror_display_for = None + self.name_bbk_uol.save() + self.organization_bbk.refresh_from_db() + self.assertEqual( + self.organization_bbk.name, + self.name_bbk_uol, + ) + self.name_bbk_uol.delete() + self.organization_bbk.refresh_from_db() + self.assertEqual( + self.organization_bbk.name, + self.name_bbk_cy, + ) + + def test_organization_name_custom_label(self): + self.name_bbk_uol.delete() + self.organization_bbk.refresh_from_db() + self.assertEqual( + self.organization_bbk.name, + self.name_bbk_custom, + ) + + def test_ror_validation(self): + for invalid_ror in [ + # URLValidator + '0v2w8z018', + 'ror.org/0v2w8z018', + # validate_ror + 'https://ror.org/0123456789', + 'https://ror.org/0lu42o079', + 'https://ror.org/abcdefghj', + ]: + with self.assertRaises(ValidationError): + org = models.Organization.objects.create(ror=invalid_ror) + org.clean_fields() + + def test_account_affiliation_with_primary(self): + self.assertEqual( + self.kathleen_booth.affiliation(), + 'Lecturer, Department of Numerical Automation, Birkbeck, University of London, London, United Kingdom', + ) + + def test_account_affiliation_with_no_title(self): + self.affiliation_lecturer.title = '' + self.affiliation_lecturer.save() + self.assertEqual( + self.kathleen_booth.affiliation(), + 'Department of Numerical Automation, Birkbeck, University of London, London, United Kingdom', + ) + + def test_account_affiliation_with_no_country(self): + self.location_london.country = None + self.location_london.save() + self.assertEqual( + self.kathleen_booth.affiliation(), + 'Lecturer, Department of Numerical Automation, Birkbeck, University of London, London', + ) + + def test_account_affiliation_with_no_location(self): + self.organization_bbk.locations.remove(self.location_london) + self.assertEqual( + self.kathleen_booth.affiliation(), + 'Lecturer, Department of Numerical Automation, Birkbeck, University of London', + ) + + def test_account_affiliation_with_no_organization(self): + self.affiliation_lecturer.organization = None + self.affiliation_lecturer.save() + self.assertEqual( + self.kathleen_booth.affiliation(), + 'Lecturer, Department of Numerical Automation', + ) + + def test_account_affiliation_for_past_date(self): + year_1950 = date.fromisoformat('1950-01-01') + self.assertEqual( + self.kathleen_booth.affiliation(date=year_1950), + 'Junior Scientific Officer, Royal Aircraft Establishment, Farnborough, United Kingdom', + ) + + def test_account_affiliation_with_no_primary(self): + self.affiliation_lecturer.is_primary = False + self.affiliation_lecturer.save() + self.assertEqual( + self.kathleen_booth.affiliation(), + 'Junior Scientific Officer, Royal Aircraft Establishment, Farnborough, United Kingdom', + ) + + def test_account_affiliation_with_no_dates_and_no_primary(self): + self.affiliation_lecturer.is_primary = False + self.affiliation_lecturer.start = None + self.affiliation_lecturer.end = None + self.affiliation_lecturer.save() + self.affiliation_officer.start = None + self.affiliation_officer.save() + self.assertEqual( + self.kathleen_booth.affiliation(), + 'Junior Scientific Officer, Royal Aircraft Establishment, Farnborough, United Kingdom', + ) + + def test_account_affiliation_with_no_affiliations(self): + self.affiliation_lecturer.delete() + self.affiliation_officer.delete() + self.affiliation_scientist.delete() + self.assertEqual( + self.kathleen_booth.affiliation(), + '', + ) + + def test_account_affiliation_obj_true(self): + self.affiliation_lecturer.delete() + self.assertEqual( + self.kathleen_booth.affiliation(obj=True), + self.affiliation_officer, + ) + + def test_frozen_author_affiliation(self): + self.assertEqual( + self.kathleen_booth_frozen.affiliation(obj=True), + self.affiliation_lecturer_frozen, + ) + + def test_preprint_author_affiliation_getter(self): + self.assertEqual( + self.kathleen_booth_preprint.affiliation, + str(self.affiliation_lecturer_preprint), + ) + + def test_preprint_author_affiliation_setter(self): + self.kathleen_booth_preprint.affiliation = 'Birkbeck McMillan' + self.kathleen_booth_preprint.refresh_from_db() + self.assertEqual( + str(self.kathleen_booth_preprint.affiliation), + 'Birkbeck McMillan' + ) + + @patch('core.models.timezone.now') + def test_affiliation_is_current(self, now): + now.return_value = date.fromisoformat('1963-01-31') + self.assertFalse(self.affiliation_lecturer.is_current) + self.assertTrue(self.affiliation_scientist.is_current) + self.assertTrue(self.affiliation_officer.is_current) + + def test_organization_location(self): + self.assertEqual( + self.organization_bbk.location, + self.location_london, + ) + + def test_set_primary_if_first_true(self): + first_affiliation, _created = models.Affiliation.objects.get_or_create( + account=self.t_s_eliot, + organization=self.organization_bbk, + ) + self.assertTrue(first_affiliation.is_primary) + + def test_set_primary_if_first_false(self): + _first_affiliation, _created = models.Affiliation.objects.get_or_create( + account=self.t_s_eliot, + organization=self.organization_bbk, + ) + second_affiliation, _created = models.Affiliation.objects.get_or_create( + account=self.t_s_eliot, + organization=self.organization_rae, + ) + self.assertFalse(second_affiliation.is_primary) + + def test_affiliation_naive_get_or_create(self): + affiliation, _created = models.Affiliation.naive_get_or_create( + institution='Birkbeck Coll', + department='Computer Sci', + country='GB', + ) + self.assertEqual( + models.Organization.objects.get( + custom_label__value='Birkbeck Coll' + ), + affiliation.organization, + ) + self.assertEqual( + 'Computer Sci', + affiliation.department, + ) + self.assertIn( + models.Location.objects.get( + name='', + country__code='GB' + ), + affiliation.organization.locations.all(), + ) + + def test_account_manager_get_or_create(self): + kwargs = { + 'first_name':'Michael', + 'last_name':'Warner', + 'email': 'twlwpky6omkqdsc40zlm@example.org', + 'institution': 'Yale', + 'department': 'English', + 'country': 'US', + } + account, _created = models.Account.objects.get_or_create(**kwargs) + self.assertListEqual( + list(kwargs.values()), + [ + account.first_name, + account.last_name, + account.email, + account.affiliation(obj=True).organization.custom_label.value, + account.affiliation(obj=True).department, + account.affiliation(obj=True).organization.locations.first().country.code, + ] + ) + + def test_organization_deduplicate_to_ror_record(self): + self.organization_bbk_legacy.deduplicate_to_ror_record() + self.affiliation_historian.refresh_from_db() + self.assertNotEqual( + self.affiliation_historian.organization, + self.organization_bbk_legacy, + ) + self.assertEqual( + self.affiliation_historian.organization, + self.organization_bbk, + ) diff --git a/src/core/views.py b/src/core/views.py index b8c1a36810..01ee88c6f8 100755 --- a/src/core/views.py +++ b/src/core/views.py @@ -15,11 +15,12 @@ from django.contrib.auth import authenticate, logout, login from django.contrib.auth.decorators import login_required from django.core.cache import cache -from django.urls import NoReverseMatch, reverse +from django.urls import NoReverseMatch, reverse, reverse_lazy from django.shortcuts import render, get_object_or_404, redirect, Http404 from django.utils import timezone from django.utils.decorators import method_decorator from django.http import HttpResponse, QueryDict +from django.contrib.messages.views import SuccessMessageMixin from django.contrib.sessions.models import Session from django.core.validators import validate_email from django.core.exceptions import ValidationError @@ -27,6 +28,7 @@ from django.conf import settings as django_settings from django.views.decorators.http import require_POST from django.views.decorators.csrf import ensure_csrf_cookie +from django.views.generic import CreateView, UpdateView, DeleteView from django.contrib.contenttypes.models import ContentType from django.utils.translation import gettext_lazy as _ from django.utils import translation @@ -34,7 +36,11 @@ from django.views import generic from core import models, forms, logic, workflow, files, models as core_models -from core.model_utils import NotImplementedField, search_model_admin +from core.model_utils import ( + NotImplementedField, + SafePaginator, + search_model_admin +) from security.decorators import ( editor_user_required, article_author_required, has_journal, any_editor_user_required, role_can_access, @@ -327,11 +333,6 @@ def register(request): initial["last_name"] = orcid_details.get("last_name", "") if orcid_details.get("emails"): initial["email"] = orcid_details["emails"][0] - if orcid_details.get("affiliation"): - initial['institution'] = orcid_details['affiliation'] - if orcid_details.get("country"): - if models.Country.objects.filter(code=orcid_details['country']).exists(): - initial["country"] = models.Country.objects.get(code=orcid_details['country']) form = forms.RegistrationForm( journal=request.journal, @@ -353,6 +354,10 @@ def register(request): if form.is_valid(): if token_obj: new_user = form.save() + if new_user.orcid: + orcid_details = orcid.get_orcid_record_details(token_obj.orcid) + if orcid_details.get("affiliation"): + new_user.institution = orcid_details['affiliation'] token_obj.delete() # If the email matches the user email on ORCID, log them in if new_user.email == initial.get("email"): @@ -2454,7 +2459,7 @@ class GenericFacetedListView(generic.ListView): """ model = NotImplementedField template_name = NotImplementedField - + paginator_class = SafePaginator paginate_by = '25' facets = {} @@ -2775,3 +2780,208 @@ def post(self, request, *args, **kwargs): messages.success(request, message) return super().post(request, *args, **kwargs) + + +@method_decorator(login_required, name='dispatch') +class OrganizationListView(GenericFacetedListView): + """ + Allows a user to search for an organization to add + as one of their own affiliations. + """ + + model = core_models.Organization + template_name = 'admin/core/organization_search.html' + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context['account'] = self.request.user + return context + + def get_queryset(self, *args, **kwargs): + queryset = super().get_queryset(*args, **kwargs) + # Exclude user-created organizations from search results + return queryset.exclude(custom_label__isnull=False) + + def get_facets(self): + return { + 'q': { + 'type': 'search', + 'field_label': 'Search', + }, + } + + +@login_required +def organization_name_create(request): + """ + Allows a user to create a custom organization name + if they cannot find one in ROR data. + """ + + form = forms.OrganizationNameForm() + + if request.POST: + form = forms.OrganizationNameForm(request.POST) + if form.is_valid(): + organization_name = form.save() + organization = core_models.Organization.objects.create() + organization_name.custom_label_for = organization + organization_name.save() + messages.add_message( + request, + messages.SUCCESS, + _("Custom organization created: %s") % organization_name, + ) + return redirect( + reverse( + 'core_affiliation_create', + kwargs={ + 'organization_id': organization.pk, + } + ) + ) + context = { + 'account': request.user, + 'form': form, + } + template = 'admin/core/organizationname_form.html' + return render(request, template, context) + + +@login_required +def organization_name_update(request, organization_name_id): + """ + Allows a user to update a custom organization name. + """ + + organization_name = get_object_or_404( + core_models.OrganizationName, + pk=organization_name_id, + custom_label_for__affiliation__account=request.user, + ) + form = forms.OrganizationNameForm(instance=organization_name) + + if request.POST: + form = forms.OrganizationNameForm( + request.POST, + instance=organization_name, + ) + if form.is_valid(): + organization = form.save() + messages.add_message( + request, + messages.SUCCESS, + _("Custom organization updated: %s") % organization, + ) + return redirect(reverse('core_edit_profile')) + + template = 'admin/core/organizationname_form.html' + context = { + 'account': request.user, + 'form': form, + } + return render(request, template, context) + + +@login_required +def affiliation_create(request, organization_id): + """ + Allows a user to create a new affiliation for themselves. + """ + + organization = core_models.Organization.objects.get( + pk=organization_id, + ) + initial = { + 'account': request.user, + 'organization': organization_id, + } + form = forms.AffiliationForm(initial=initial) + + if request.POST: + form = forms.AffiliationForm(request.POST) + if form.is_valid(): + affiliation = form.save() + messages.add_message( + request, + messages.SUCCESS, + _("Affiliation created: %s") % affiliation, + ) + return redirect(reverse('core_edit_profile')) + + template = 'admin/core/affiliation_form.html' + context = { + 'account': request.user, + 'form': form, + 'organization': organization, + } + return render(request, template, context) + + +@login_required +def affiliation_update(request, affiliation_id): + """ + Allows a user to update one of their own affiliations. + """ + + affiliation = get_object_or_404( + core_models.Affiliation, + pk=affiliation_id, + account=request.user, + ) + + form = forms.AffiliationForm(instance=affiliation) + + if request.POST: + form = forms.OrganizationNameForm(request.POST) + if form.is_valid(): + affiliation = form.save() + messages.add_message( + request, + messages.SUCCESS, + _("Affiliation updated: %s") % affiliation, + ) + return redirect(reverse('core_edit_profile')) + + template = 'admin/core/affiliation_form.html' + context = { + 'account': request.user, + 'form': form, + 'affiliation': affiliation, + 'organization': affiliation.organization, + } + return render(request, template, context) + + +@login_required +def affiliation_delete(request, affiliation_id): + """ + Allows a user to delete one of their own affiliations. + """ + + affiliation = get_object_or_404( + core_models.Affiliation, + pk=affiliation_id, + account=request.user, + ) + form = forms.ConfirmDeleteForm() + + if request.POST: + form = forms.ConfirmDeleteForm(request.POST) + if form.is_valid(): + affiliation.delete() + messages.add_message( + request, + messages.SUCCESS, + _("Affiliation deleted: %s") % affiliation, + ) + return redirect(reverse('core_edit_profile')) + + template = 'admin/core/affiliation_confirm_delete.html' + context = { + 'account': request.user, + 'form': form, + 'affiliation': affiliation, + 'organization': affiliation.organization, + } + return render(request, template, context) diff --git a/src/cron/management/commands/send_publication_notifications.py b/src/cron/management/commands/send_publication_notifications.py index 3969b4f976..b9d3977d8c 100644 --- a/src/cron/management/commands/send_publication_notifications.py +++ b/src/cron/management/commands/send_publication_notifications.py @@ -8,6 +8,9 @@ from journal import models as journal_models from submission import models as submission_models from utils import notify_helpers, setting_handler, render_template +from utils.logger import get_logger + +logger = get_logger(__name__) def create_fake_request(journal): @@ -47,12 +50,12 @@ def handle(self, *args, **options): setting_name='send_reader_notifications', journal=journal, ).value: - print('Sending notification for {}'.format(journal.name)) + logger.info('Sending notification for {}'.format(journal.name)) readers = journal.users_with_role('reader') bcc_list = [reader.email for reader in readers] if bcc_list: - print("Sending notifications to {}".format( + logger.info("Sending notifications to {}".format( ", ".join(bcc_list) )) @@ -98,8 +101,8 @@ def handle(self, *args, **options): } ) else: - print("No articles were published today.") + logger.info("No articles were published today.") else: - print('Reader publication notifications are not enabled for {}'.format(journal.name)) + logger.info('Reader publication notifications are not enabled for {}'.format(journal.name)) diff --git a/src/cron/management/commands/send_reminders.py b/src/cron/management/commands/send_reminders.py index 112d3c45bf..c2c36fcf81 100755 --- a/src/cron/management/commands/send_reminders.py +++ b/src/cron/management/commands/send_reminders.py @@ -2,6 +2,9 @@ from cron import models from journal import models as journal_models +from utils.logger import get_logger + +logger = get_logger(__name__) class Command(BaseCommand): @@ -19,12 +22,12 @@ def handle(self, *args, **options): journals = journal_models.Journal.objects.all() for journal in journals: - print("Processing reminders for journal {0}: {1}".format(journal.pk, journal.name)) + logger.info("Processing reminders for journal {0}: {1}".format(journal.pk, journal.name)) reminders = models.Reminder.objects.filter(journal=journal) for reminder in reminders: - print("Reminder {0}, target date: {1}".format(reminder, reminder.target_date())) + logger.info("Reminder {0}, target date: {1}".format(reminder, reminder.target_date())) if action == 'test': reminder.send_reminder(test=True) else: diff --git a/src/repository/admin.py b/src/repository/admin.py index 676d38840b..be262e662c 100755 --- a/src/repository/admin.py +++ b/src/repository/admin.py @@ -128,6 +128,10 @@ class PreprintAuthorAdmin(admin_utils.PreprintFKModelAdmin): 'account__email', 'account__orcid', 'account__first_name', 'account__last_name') + inlines = [ + admin_utils.AffiliationInline, + ] + class PreprintVersionAdmin(admin_utils.PreprintFKModelAdmin): list_display = ('pk', '_preprint', 'title', 'version', 'date_time', diff --git a/src/repository/migrations/0044_remove_preprintauthor_affiliation_and_more.py b/src/repository/migrations/0044_remove_preprintauthor_affiliation_and_more.py new file mode 100644 index 0000000000..bd7ef28399 --- /dev/null +++ b/src/repository/migrations/0044_remove_preprintauthor_affiliation_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.14 on 2024-07-26 13:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0101_migrate_affiliation_institution'), + ('repository', '0045_historicalrepository_display_public_metrics_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='preprintauthor', + name='affiliation', + ), + migrations.AlterField( + model_name='author', + name='affiliation', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/src/repository/models.py b/src/repository/models.py index 0166f207b7..dc0bd492b6 100755 --- a/src/repository/models.py +++ b/src/repository/models.py @@ -589,10 +589,18 @@ def display_authors(self): def add_user_as_author(self, user): preprint_author, created = PreprintAuthor.objects.get_or_create( account=user, - affiliation=user.institution, preprint=self, defaults={'order': self.next_author_order()}, ) + for affiliation in user.affiliation_set.all(): + core_models.Affiliation.objects.get_or_create( + preprint_author=preprint_author, + title=affiliation.title, + department=affiliation.department, + organization=affiliation.is_primary, + start=affiliation.start, + end=affiliation.end, + ) return created @@ -888,6 +896,25 @@ def get_queryset(self): return super().get_queryset().select_related('account') + def get_or_create(self, defaults=None, **kwargs): + """ + Backwards-compatible override for affiliation-related kwargs + """ + # check for deprecated fields related to affiliation + affiliation = kwargs.pop('affiliation', '') + + preprint_author, created = super().get_or_create(defaults, **kwargs) + + # create or update affiliation + if affiliation: + Affiliation.naive_get_or_create( + institution=affiliation, + preprint_author=preprint_author, + ) + + return preprint_author, created + + class PreprintAuthor(models.Model): preprint = models.ForeignKey( 'Preprint', @@ -899,7 +926,6 @@ class PreprintAuthor(models.Model): on_delete=models.SET_NULL, ) order = models.PositiveIntegerField(default=0) - affiliation = models.TextField(blank=True, null=True) objects = PreprintAuthorManager() @@ -913,6 +939,17 @@ def __str__(self): preprint=self.preprint.title, ) + @property + def affiliation(self): + return core_models.Affiliation.get_primary(preprint_author=self) + + @affiliation.setter + def affiliation(self, value): + core_models.Affiliation.naive_get_or_create( + institution=value, + preprint_author=self, + ) + @property def full_name(self): if not self.account.middle_name: @@ -947,6 +984,9 @@ def display_affiliation(self): class Author(models.Model): + """ + Deprecated. Please use PreprintAuthor instead. + """ email_address = models.EmailField(unique=True) first_name = models.CharField(max_length=255) middle_name = models.CharField(max_length=255, blank=True, null=True) @@ -959,6 +999,10 @@ class Author(models.Model): verbose_name=_('ORCID') ) + def __init__(self, *args, **kwargs): + raise DeprecationWarning('Use PreprintAuthor instead.') + super().__init__(*args, **kwargs) + @property def full_name(self): if not self.middle_name: diff --git a/src/repository/views.py b/src/repository/views.py index 9e9fbcefc5..2636ba48ee 100644 --- a/src/repository/views.py +++ b/src/repository/views.py @@ -377,7 +377,7 @@ def repository_search(request, search_term=None): Q(account__first_name__in=split_search_term) | Q(account__middle_name__in=split_search_term) | Q(account__last_name__in=split_search_term) | - Q(account__institution__icontains=search_term) + Q(account__affiliation__organization__labels__value__icontains=search_term) ) & ( diff --git a/src/review/logic.py b/src/review/logic.py index b9c1e32c56..c7ac727733 100755 --- a/src/review/logic.py +++ b/src/review/logic.py @@ -824,11 +824,6 @@ def process_reviewer_csv(path, request, article, form): reader = csv.DictReader(csv_file) reviewers = [] for row in reader: - try: - country = core_models.Country.objects.get(code=row.get('country')) - except core_models.Country.DoesNotExist: - country = None - reviewer, created = core_models.Account.objects.get_or_create( email=row.get('email_address'), defaults={ @@ -836,12 +831,15 @@ def process_reviewer_csv(path, request, article, form): 'first_name': row.get('firstname', ''), 'middle_name': row.get('middlename', ''), 'last_name': row.get('lastname', ''), - 'department': row.get('department', ''), - 'institution': row.get('institution', ''), - 'country': country, 'is_active': True, } ) + if row.get('institution', '') or row.get('department', ''): + core_models.Affiliation.naive_get_or_create( + institution=row.get('institution', ''), + department=row.get('department', ''), + account=reviewer, + ) try: review_interests = row.get('interests') diff --git a/src/static/admin/img/icons/ror-icon-rgb.svg b/src/static/admin/img/icons/ror-icon-rgb.svg new file mode 100644 index 0000000000..4cabc057df --- /dev/null +++ b/src/static/admin/img/icons/ror-icon-rgb.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + diff --git a/src/static/common/css/utilities.css b/src/static/common/css/utilities.css index 321d1bc2d3..bcb09dada4 100644 --- a/src/static/common/css/utilities.css +++ b/src/static/common/css/utilities.css @@ -12,6 +12,10 @@ &.items-start { align-items: flex-start; } &.items-center { align-items: center; } + /* Flex direction */ + &.direction-row { flex-direction: row; } + &.direction-column { flex-direction: column; } + /* Responsive flex direction */ @media (orientation: portrait) { &.portrait-row { flex-direction: row; } diff --git a/src/submission/admin.py b/src/submission/admin.py index 239b05f5a3..3a8a33e57b 100755 --- a/src/submission/admin.py +++ b/src/submission/admin.py @@ -25,13 +25,17 @@ class ArticleFundingAdmin(admin.ModelAdmin): class FrozenAuthorAdmin(admin_utils.ArticleFKModelAdmin): list_display = ('pk', 'first_name', 'last_name', - 'frozen_email', 'frozen_orcid', 'institution', '_journal') + 'frozen_email', 'frozen_orcid', '_journal') list_filter = ('article__journal',) search_fields = ('frozen_email', 'frozen_orcid', 'first_name', 'last_name', - 'institution', 'frozen_biography', ) + 'frozen_biography', ) raw_id_fields = ('article', 'author',) + inlines = [ + admin_utils.AffiliationInline, + ] + class ArticleAdmin(admin_utils.JanewayModelAdmin): list_display = ('pk', 'title', 'correspondence_author', diff --git a/src/submission/forms.py b/src/submission/forms.py index 7c90a08f80..e5601f3069 100755 --- a/src/submission/forms.py +++ b/src/submission/forms.py @@ -373,10 +373,7 @@ class Meta: 'middle_name', 'last_name', 'name_suffix', - 'institution', - 'department', 'frozen_biography', - 'country', 'is_corporate', 'frozen_email', 'frozen_orcid', diff --git a/src/submission/migrations/0080_remove_frozenauthor_country_and_more.py b/src/submission/migrations/0080_remove_frozenauthor_country_and_more.py new file mode 100644 index 0000000000..e2c02de91f --- /dev/null +++ b/src/submission/migrations/0080_remove_frozenauthor_country_and_more.py @@ -0,0 +1,26 @@ +# Generated by Django 4.2.14 on 2024-07-26 13:26 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0101_migrate_affiliation_institution'), + ('submission', '0084_remove_article_jats_article_type_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='frozenauthor', + name='country', + ), + migrations.RemoveField( + model_name='frozenauthor', + name='department', + ), + migrations.RemoveField( + model_name='frozenauthor', + name='institution', + ), + ] diff --git a/src/submission/models.py b/src/submission/models.py index 9a544f929c..6d5a0f85c3 100755 --- a/src/submission/models.py +++ b/src/submission/models.py @@ -1928,6 +1928,31 @@ def pinned(self): return True +class FrozenAuthorManager(models.Manager): + + def get_or_create(self, defaults=None, **kwargs): + """ + Backwards-compatible override for affiliation-related kwargs + """ + # check for deprecated fields related to affiliation + institution = kwargs.pop('institution', '') + department = kwargs.pop('department', '') + country = kwargs.pop('country', '') + + frozen_author, created = super().get_or_create(defaults, **kwargs) + + # create or update affiliation + if institution or department or country: + Affiliation.naive_get_or_create( + institution=institution, + department=department, + country=country, + frozen_author=frozen_author, + ) + + return frozen_author, created + + class FrozenAuthor(AbstractLastModifiedModel): article = models.ForeignKey( 'submission.Article', @@ -1970,16 +1995,6 @@ class FrozenAuthor(AbstractLastModifiedModel): validators=[plain_text_validator], ) - institution = models.CharField( - max_length=1000, - blank=True, - validators=[plain_text_validator], -) - department = models.CharField( - max_length=300, - blank=True, - validators=[plain_text_validator], - ) frozen_biography = JanewayBleachField( blank=True, verbose_name=_('Frozen Biography'), @@ -1990,13 +2005,6 @@ class FrozenAuthor(AbstractLastModifiedModel): " for the account will be populated instead." ), ) - country = models.ForeignKey( - 'core.Country', - null=True, - blank=True, - on_delete=models.SET_NULL, - ) - order = models.PositiveIntegerField(default=1) is_corporate = models.BooleanField( @@ -2022,12 +2030,51 @@ class FrozenAuthor(AbstractLastModifiedModel): help_text=_("If checked, this authors email address link will be displayed on the article page.") ) + objects = FrozenAuthorManager() + class Meta: ordering = ('order', 'pk') def __str__(self): return self.full_name() + @property + def institution(self): + affil = self.affiliation(obj=True) + return str(affil.organization) if affil else '' + + @institution.setter + def institution(self, value): + core_models.Affiliation.naive_get_or_create( + institution=value, + frozen_author=self + ) + + @property + def department(self): + affil = self.affiliation(obj=True) + return str(affil.department) if affil else '' + + @department.setter + def department(self, value): + core_models.Affiliation.naive_set_primary_department( + value, + frozen_author=self, + ) + + @property + def country(self): + affil = self.affiliation(obj=True) + organization = affil.organization if affil else None + return str(organization.country) if organization else None + + @country.setter + def country(self, value): + core_models.Affiliation.naive_get_or_create( + country=value, + frozen_author=self + ) + def full_name(self): if self.is_corporate: return self.corporate_name @@ -2111,13 +2158,18 @@ def given_names(self): else: return self.first_name - def affiliation(self): - if self.institution and self.department: - return "{}, {}".format(self.department, self.institution) - elif self.institution: - return self.institution - else: - return '' + def affiliation(self, obj=False, date=None): + return core_models.Affiliation.get_primary( + frozen_author=self, + obj=obj, + date=date, + ) + + @property + def affiliations(self): + return core_models.Affiliation.objects.filter( + frozen_author=self + ).order_by('-is_primary') @property def is_correspondence_author(self): diff --git a/src/submission/tests.py b/src/submission/tests.py index de565fe4ba..622f9267c4 100644 --- a/src/submission/tests.py +++ b/src/submission/tests.py @@ -51,8 +51,8 @@ def test_new_journals_has_submission_configuration(self): if not self.journal_one.submissionconfiguration: self.fail('Journal does not have a submissionconfiguration object.') - @staticmethod - def create_journal(): + @classmethod + def create_journal(cls): """ Creates a dummy journal for testing :return: a journal @@ -66,54 +66,60 @@ def create_journal(): return journal_one - @staticmethod - def create_authors(): + @classmethod + def create_authors(cls): author_1_data = { + 'email': 'one@example.org', 'is_active': True, 'password': 'this_is_a_password', 'salutation': 'Prof.', 'first_name': 'Martin', + 'middle_name': '', 'last_name': 'Eve', 'department': 'English & Humanities', 'institution': 'Birkbeck, University of London', } author_2_data = { + 'email': 'two@example.org', 'is_active': True, 'password': 'this_is_a_password', 'salutation': 'Sr.', 'first_name': 'Mauro', + 'middle_name': '', 'last_name': 'Sanchez', 'department': 'English & Humanities', 'institution': 'Birkbeck, University of London', } - author_1 = Account.objects.create(email="1@t.t", **author_1_data) - author_2 = Account.objects.create(email="2@t.t", **author_2_data) + author_1 = helpers.create_author(cls.journal_one, **author_1_data) + author_2 = helpers.create_author(cls.journal_one, **author_2_data) return author_1, author_2 - def create_sections(self): - self.section_1 = models.Section.objects.create( + @classmethod + def create_sections(cls): + cls.section_1 = models.Section.objects.create( name='Test Public Section', - journal=self.journal_one, + journal=cls.journal_one, ) - self.section_2 = models.Section.objects.create( + cls.section_2 = models.Section.objects.create( name='Test Private Section', public_submissions=False, - journal=self.journal_one + journal=cls.journal_one ) - self.section_3 = models.Section.objects.create( - journal=self.journal_one, + cls.section_3 = models.Section.objects.create( + journal=cls.journal_one, ) - def setUp(self): + @classmethod + def setUpTestData(cls): """ Setup the test environment. :return: None """ - self.journal_one = self.create_journal() - self.editor = helpers.create_editor(self.journal_one) - self.press = helpers.create_press() - self.create_sections() + cls.journal_one = cls.create_journal() + cls.editor = helpers.create_editor(cls.journal_one) + cls.press = helpers.create_press() + cls.create_sections() def test_article_image_galley(self): article = models.Article.objects.create( @@ -273,7 +279,9 @@ def test_snapshot_author_metadata_override(self): article.snapshot_authors() new_department = "New department" - article.frozen_authors().update(department=new_department) + for frozen_author in article.frozen_authors(): + frozen_author.department = new_department + frozen_author.save() article.snapshot_authors(force_update=True) frozen = article.frozen_authors().all()[0] @@ -576,15 +584,12 @@ def test_author_form_harmful_inputs(self): "middle_name", "name_prefix", "suffix", - "institution", - "department", }): form = forms.AuthorForm( { 'first_name': 'Andy', 'last_name': 'Byers', 'biography': 'Andy', - 'institution': 'Birkbeck, University of London', 'email': f'andy{i}@janeway.systems', 'orcid': 'https://orcid.org/0000-0003-2126-266X', **{attr: harmful_string}, @@ -783,28 +788,31 @@ def create_journal(): return journal_one - @staticmethod - def create_authors(): + def create_authors(self): author_1_data = { + 'email': 'one@example.org', 'is_active': True, 'password': 'this_is_a_password', 'salutation': 'Prof.', 'first_name': 'Martin', + 'middle_name': '', 'last_name': 'Eve', 'department': 'English & Humanities', 'institution': 'Birkbeck, University of London', } author_2_data = { + 'email': 'two@example.org', 'is_active': True, 'password': 'this_is_a_password', 'salutation': 'Sr.', 'first_name': 'Mauro', + 'middle_name': '', 'last_name': 'Sanchez', 'department': 'English & Humanities', 'institution': 'Birkbeck, University of London', } - author_1 = Account.objects.create(email="1@t.t", **author_1_data) - author_2 = Account.objects.create(email="2@t.t", **author_1_data) + author_1 = helpers.create_author(self.journal_one, **author_1_data) + author_2 = helpers.create_author(self.journal_one, **author_2_data) return author_1, author_2 diff --git a/src/templates/admin/core/accounts/edit_profile.html b/src/templates/admin/core/accounts/edit_profile.html index f647f66f81..96b070d849 100644 --- a/src/templates/admin/core/accounts/edit_profile.html +++ b/src/templates/admin/core/accounts/edit_profile.html @@ -2,7 +2,7 @@ {% load i18n foundation static %} {% block css %} - + {% endblock %} {% block contextual_title %} @@ -117,6 +117,29 @@

{% trans "Update Password" %}

{% include "admin/elements/forms/denotes_required.html" %} +
+
+

{% trans "Affiliations" %}

+
+
+ {% for affiliation in request.user.affiliations %} +
+ {% include "admin/core/affiliation_display.html" with affiliation=affiliation %} +
+ {% url 'core_affiliation_update' affiliation.pk as update_url %} + {% include "elements/a_edit.html" with href=update_url size="small" %} + {% url 'core_affiliation_delete' affiliation.pk as delete_url %} + {% include "elements/a_delete.html" with href=delete_url size="small" %} +
+
+ {% endfor %} +
+ {% url 'core_organization_search' as create_url %} + {% trans "Add affiliation" as add_affiliation %} + {% include "elements/a_create.html" with href=create_url size="small" label=add_affiliation %} +
+
+

{% trans 'Profile Details' %}

diff --git a/src/templates/admin/core/affiliation_confirm_delete.html b/src/templates/admin/core/affiliation_confirm_delete.html new file mode 100644 index 0000000000..d61c1293c7 --- /dev/null +++ b/src/templates/admin/core/affiliation_confirm_delete.html @@ -0,0 +1,62 @@ +{% extends "admin/core/large_form.html" %} + +{% load i18n foundation %} + +{% block contextual_title %} + {% blocktrans with organization=affiliation.organization.name %} + Delete affiliation: {{ organization }} + {% endblocktrans %} +{% endblock contextual_title %} + +{% block title-section %} + {% blocktrans with organization=affiliation.organization.name %} + Delete affiliation: {{ organization }} + {% endblocktrans %} +{% endblock title-section %} + +{% block breadcrumbs %} + {% if request.user == account %} +
  • + + {% trans "Edit Profile" %} + +
    • +
  • + + {% trans "Edit Affiliation" %} + +
    • +
  • + {% blocktrans with organization=affiliation.organization.name %} + Delete affiliation: {{ organization }} + {% endblocktrans %} +
    • + {% endif %} +{% endblock breadcrumbs %} + +{% block body %} +
    +
    + {% include "admin/core/affiliation_summary.html" %} +
    +
    +

    {% trans "Confirmation" %}

    +
    + {% include "admin/elements/forms/messages_in_callout.html" with form=form %} +
    + {% csrf_token %} + {% for field in form.hidden_fields %} + {{ field }} + {% endfor %} +

    {% blocktrans %} + Are you sure you want to delete the affiliation "{{ affiliation }}"? + {% endblocktrans %}

    + {{ form }} + {% include "elements/button_yes_delete.html" %} + {% url 'core_edit_profile' as go_back_url %} + {% include "elements/a_no_go_back.html" with href=go_back_url %} +
    +
    +
    +
    +{% endblock body %} diff --git a/src/templates/admin/core/affiliation_display.html b/src/templates/admin/core/affiliation_display.html new file mode 100644 index 0000000000..865bd36e7c --- /dev/null +++ b/src/templates/admin/core/affiliation_display.html @@ -0,0 +1,28 @@ +{% load static %} + +
    + {% if affiliation.title_department %} + {{ affiliation.title_department }}, + {% endif %} + {% if affiliation.organization.ror %} + + {{ affiliation.organization }} + (opens in new tab) + ROR logo + + {% else %} + {{ affiliation.organization }} + {% endif %} + {% if affiliation.start or affiliation.end %} + ({{ affiliation.start|date:"M Y" }}–{{ affiliation.end|date:"M Y" }}) + {% endif %} + {% if affiliation.is_primary %} + + Primary + + {% endif %} +
    diff --git a/src/templates/admin/core/affiliation_form.html b/src/templates/admin/core/affiliation_form.html new file mode 100644 index 0000000000..62f22c5c17 --- /dev/null +++ b/src/templates/admin/core/affiliation_form.html @@ -0,0 +1,83 @@ +{% extends "admin/core/large_form.html" %} + +{% load i18n static foundation %} + +{% block contextual_title %} + {% blocktrans with organization=organization.name %} + Affiliation details for {{ organization }} + {% endblocktrans %} +{% endblock contextual_title %} + +{% block title-section %} + {% blocktrans with organization=organization.name %} + Affiliation details for {{ organization }} + {% endblocktrans %} +{% endblock title-section %} + +{% block breadcrumbs %} + {% if request.user == account %} +
  • Edit Profile
    • + {% if not affiliation %} +
  • + + {% trans "Add Affiliation" %} + +
    • + {% endif %} +
  • {% blocktrans with organization=organization.name %} + Affiliation details for {{ organization }} + {% endblocktrans %}
    • + {% endif %} +{% endblock breadcrumbs %} + +{% block body %} +
    +
    + {% include "admin/core/affiliation_summary.html" %} +
    +
    +

    + {% trans "Affiliation details for" %} {{ organization.name }} +

    +
    + {% include "admin/elements/forms/messages_in_callout.html" with form=form %} + {% blocktrans with organization=organization.name %} +

    Enter optional affiliation details, and select Save to create the affiliation.

    + {% endblocktrans %} + {% if organization.custom_label %} +
    + {% include "admin/elements/layout/key_value_above.html" with key="Custom organization" value=organization.custom_label %} + {% if affiliation %} +
    + {% url 'core_organization_name_update' organization.custom_label.pk as edit_url %} + {% include "elements/a_edit.html" with href=edit_url size="small" %} +
    + {% endif %} +
    + {% endif %} +
    + {% csrf_token %} + {% for field in form.hidden_fields %} + {{ field }} + {% endfor %} +
    + {% include "admin/elements/forms/field.html" with field=form.title %} + {% include "admin/elements/forms/field.html" with field=form.department %} +
    +
    + {% include "admin/elements/forms/field.html" with field=form.start %} + {% include "admin/elements/forms/field.html" with field=form.end %} + {% include "admin/elements/forms/field.html" with field=form.is_primary %} +
    + {% include "elements/button_save.html" %} + {% if affiliation %} + {% url 'core_affiliation_delete' affiliation.pk as delete_url %} + {% include "elements/a_delete.html" with href=delete_url %} + {% endif %} + {% url 'core_edit_profile' as cancel_url %} + {% include "elements/a_cancel.html" with href=cancel_url %} +
    +
    +
    +
    +{% endblock body %} diff --git a/src/templates/admin/core/affiliation_summary.html b/src/templates/admin/core/affiliation_summary.html new file mode 100644 index 0000000000..466629e6cf --- /dev/null +++ b/src/templates/admin/core/affiliation_summary.html @@ -0,0 +1,12 @@ +
    +
    +
    +

    {% trans "Summary" %}

    +
    + {% include "admin/elements/layout/key_value_above.html" with key="Account Name" value=account %} + {% include "admin/elements/layout/key_value_above.html" with key="Email" value=account.email %} + {% if organization %} + {% include "admin/elements/layout/key_value_above.html" with key="Organization" value=organization %} + {% endif %} +
    +
    diff --git a/src/templates/admin/core/manager/users/edit.html b/src/templates/admin/core/manager/users/edit.html index 7093cc3e0d..43a3ec67f8 100644 --- a/src/templates/admin/core/manager/users/edit.html +++ b/src/templates/admin/core/manager/users/edit.html @@ -5,7 +5,7 @@ {% block title %}{{ active|capfirst }} Profile{% endblock title %} {% block css %} - + {% endblock %} {% block breadcrumbs %} diff --git a/src/templates/admin/core/organization_list.html b/src/templates/admin/core/organization_list.html new file mode 100644 index 0000000000..63f18d7461 --- /dev/null +++ b/src/templates/admin/core/organization_list.html @@ -0,0 +1,70 @@ +{% extends "admin/submission/submit_authors_base.html" %} + +{% load static %} +{% load i18n %} + +{% block manage_authors %} +
    +
    +

    {% trans "Add affiliation for " %}{{ account.full_name }}

    +
    +
    +
    + + +
    + +
    +
    +
    + {% for organization in organization_list %} +
    +
    +
    +

    {{ organization.ror_display }}

    +
    + {% include "admin/elements/layout/key_value_above.html" with key="Locations" value=organization.locations.all list=True %} + {% if organization.also_known_as %} + {% include "admin/elements/layout/key_value_above.html" with key="Also known as" value=organization.also_known_as list=True %} + {% endif %} + {% if organization.locations.count > 1 %} + {% include "admin/elements/layout/key_value_above.html" with key="Locations" value=organization.locations.all list=True %} + {% endif %} + {% include "admin/elements/layout/key_value_above.html" with key="ROR ID" value=organization.ror link=organization.ror %} + {% include "admin/elements/layout/key_value_above.html" with key="Website" value=organization.website link=organization.website %} +
    +
    +
    + {% url 'core_affiliation_create' organization.pk as create_url %} + {% trans "Add affiliation" as add_affiliation %} + {% include "elements/a_create.html" with href=create_url label=add_affiliation %} +
    +
    +
    + {% empty %} +

    {% trans 'No organizations to display.' %}

    + {% endfor %} +
    +

    Organization not found?

    + {% url 'core_organization_name_create' as create_url %} + {% trans "Create custom organization" as create_custom_organization %} + {% include "elements/a_create.html" with href=create_url label=create_custom_organization %} +
    + {% if organization_list %} + {% include "common/elements/pagination.html" with form_id=facet_form.id %} + {% endif %} +
    + {% include "admin/core/affiliation_summary.html" %} +{% endblock manage_authors %} diff --git a/src/templates/admin/core/organization_search.html b/src/templates/admin/core/organization_search.html new file mode 100644 index 0000000000..0facd3dc8e --- /dev/null +++ b/src/templates/admin/core/organization_search.html @@ -0,0 +1,84 @@ +{% extends "admin/core/large_form.html" %} + +{% load i18n static %} + +{% block contextual_title %} + {% trans "Add Affiliation" %} +{% endblock contextual_title %} + +{% block title-section %} + {% trans "Add Affiliation" %} +{% endblock title-section %} + +{% block breadcrumbs %} + {% if request.user == account %} +
  • Edit Profile
    • +
  • {% trans "Add Affiliation" %}
    • + {% endif %} +{% endblock breadcrumbs %} + +{% block body %} +
    +
    + {% include "admin/core/affiliation_summary.html" %} +
    + {% include "admin/elements/forms/messages_in_callout.html" with form=form %} +
    +
    +

    {% trans "Search for organization" %}

    +
    +
    + + +
    + +
    +
    +
    + {% for organization in organization_list %} +
    +
    +
    +

    {{ organization.ror_display }}

    + {% include "admin/elements/layout/key_value_above.html" with key="Locations" value=organization.locations.all list=True %} + {% if organization.also_known_as %} + {% include "admin/elements/layout/key_value_above.html" with key="Also known as" value=organization.also_known_as list=True %} + {% endif %} + {% include "admin/elements/layout/key_value_above.html" with key="ROR ID" value=organization.ror link=organization.ror %} + {% include "admin/elements/layout/key_value_above.html" with key="Website" value=organization.website link=organization.website %} +
    +
    + {% url 'core_affiliation_create' organization.pk as create_url %} + {% trans "Add affiliation" as add_affiliation %} + {% include "elements/a_create.html" with href=create_url label=add_affiliation %} +
    +
    +
    + {% empty %} +

    {% trans 'No organizations to display.' %}

    + {% endfor %} +
    +

    Organization not found?

    + {% url 'core_organization_name_create' as create_url %} + {% trans "Create custom organization" as create_custom_organization %} + {% include "elements/a_create.html" with href=create_url label=create_custom_organization %} +
    + {% if organization_list %} + {% include "common/elements/pagination.html" with form_id=facet_form.id %} + {% endif %} +
    +
    +
    +{% endblock body %} diff --git a/src/templates/admin/core/organizationname_form.html b/src/templates/admin/core/organizationname_form.html new file mode 100644 index 0000000000..319abf1e74 --- /dev/null +++ b/src/templates/admin/core/organizationname_form.html @@ -0,0 +1,47 @@ +{% extends "admin/core/large_form.html" %} + +{% load i18n static foundation %} + +{% block contextual_title %} + {% trans "Custom organization" %} +{% endblock contextual_title %} + +{% block title-section %} + {% trans "Custom organization" %} +{% endblock title-section %} + +{% block breadcrumbs %} + {% if request.user == account %} +
  • Edit Profile
    • +
  • + + {% trans "Add Affiliation" %} + +
    • +
  • {% trans "Custom organization" %}
    • + {% endif %} +{% endblock breadcrumbs %} + +{% block body %} +
    +
    + {% include "admin/core/affiliation_summary.html" %} +
    +
    +

    {% trans "Custom name" %}

    +
    +
    + {% include "admin/elements/forms/messages_in_callout.html" with form=form %} + {% csrf_token %} + {% for field in form.hidden_fields %} + {{ field }} + {% endfor %} + {% include "admin/elements/forms/field.html" with field=form.value %} + {% include "elements/button_save.html" %} + {% url 'core_edit_profile' as cancel_url %} + {% include "elements/a_cancel.html" with href=cancel_url %} +
    +
    +
    +
    +{% endblock body %} diff --git a/src/templates/admin/elements/a_cancel.html b/src/templates/admin/elements/a_cancel.html new file mode 100644 index 0000000000..7ef1537626 --- /dev/null +++ b/src/templates/admin/elements/a_cancel.html @@ -0,0 +1,12 @@ +{% comment %} +A generic dark blue cancel button with a cross icon +{% endcomment %} + +{% load i18n %} + + + + {% trans "Cancel" %} + diff --git a/src/templates/admin/elements/a_create.html b/src/templates/admin/elements/a_create.html new file mode 100644 index 0000000000..ace7f2e60d --- /dev/null +++ b/src/templates/admin/elements/a_create.html @@ -0,0 +1,16 @@ +{% comment %} +A generic green create button with a plus icon +{% endcomment %} + +{% load i18n %} + + + + {% if label %} + {{ label }} + {% else %} + {% trans "Create" %} + {% endif %} + diff --git a/src/templates/admin/elements/a_delete.html b/src/templates/admin/elements/a_delete.html new file mode 100644 index 0000000000..b5546718f1 --- /dev/null +++ b/src/templates/admin/elements/a_delete.html @@ -0,0 +1,12 @@ +{% comment %} +A generic red delete button with rubbish icon +{% endcomment %} + +{% load i18n %} + + + + {% trans "Delete" %} + diff --git a/src/templates/admin/elements/a_edit.html b/src/templates/admin/elements/a_edit.html new file mode 100644 index 0000000000..96ada68612 --- /dev/null +++ b/src/templates/admin/elements/a_edit.html @@ -0,0 +1,12 @@ +{% comment %} +A generic light blue edit button with a pencil icon +{% endcomment %} + +{% load i18n %} + + + + {% trans "Edit" %} + diff --git a/src/templates/admin/elements/a_no_go_back.html b/src/templates/admin/elements/a_no_go_back.html new file mode 100644 index 0000000000..473b852685 --- /dev/null +++ b/src/templates/admin/elements/a_no_go_back.html @@ -0,0 +1,12 @@ +{% comment %} +A generic dark blue cancel / go-back button with a cross icon +{% endcomment %} + +{% load i18n %} + + + + {% trans "No, go back" %} + diff --git a/src/templates/admin/elements/accounts/user_form.html b/src/templates/admin/elements/accounts/user_form.html index e2bc6119b5..a0fdfdb389 100644 --- a/src/templates/admin/elements/accounts/user_form.html +++ b/src/templates/admin/elements/accounts/user_form.html @@ -10,13 +10,6 @@

    {% trans "Name" %}

    {% include "admin/elements/forms/field.html" with field=form.suffix %}
    -

    {% trans "Affiliations" %}

    -
    - {% include "admin/elements/forms/field.html" with field=form.department %} - {% include "admin/elements/forms/field.html" with field=form.institution %} - {% include "admin/elements/forms/field.html" with field=form.country %} -
    -

    {% trans "Social Media and Accounts" %}

    {% include "admin/elements/forms/field.html" with field=form.twitter %} diff --git a/src/templates/admin/elements/button_save.html b/src/templates/admin/elements/button_save.html new file mode 100644 index 0000000000..b172cda4c6 --- /dev/null +++ b/src/templates/admin/elements/button_save.html @@ -0,0 +1,13 @@ +{% comment %} +A generic green save button with floppy disk icon +{% endcomment %} + +{% load i18n %} + + diff --git a/src/templates/admin/elements/button_submit_warning.html b/src/templates/admin/elements/button_submit_warning.html new file mode 100644 index 0000000000..70f6babe12 --- /dev/null +++ b/src/templates/admin/elements/button_submit_warning.html @@ -0,0 +1,16 @@ +{% comment %} +A generic yellow submit button with a forward arrow icon +{% endcomment %} + +{% load i18n %} + + diff --git a/src/templates/admin/elements/button_yes_delete.html b/src/templates/admin/elements/button_yes_delete.html new file mode 100644 index 0000000000..640fa97854 --- /dev/null +++ b/src/templates/admin/elements/button_yes_delete.html @@ -0,0 +1,13 @@ +{% comment %} +A generic red confirm delete button with a rubbish icon +{% endcomment %} + +{% load i18n %} + + diff --git a/src/templates/admin/elements/layout/key_value_above.html b/src/templates/admin/elements/layout/key_value_above.html index ce8f663829..10fafcf3aa 100644 --- a/src/templates/admin/elements/layout/key_value_above.html +++ b/src/templates/admin/elements/layout/key_value_above.html @@ -17,10 +17,16 @@
  • {{ item }}
    • {% endfor %} + {% elif link %} + + {{ value }} + (opens in new tab) + + {% elif render_line_breaks %} {{ value|linebreaksbr|default:"No value supplied" }} {% else %} {{ value|default:"No value supplied" }} {% endif %} -
    \ No newline at end of file + diff --git a/src/templates/admin/elements/list_filters.html b/src/templates/admin/elements/list_filters.html index ac2f5f4835..6252d73075 100644 --- a/src/templates/admin/elements/list_filters.html +++ b/src/templates/admin/elements/list_filters.html @@ -1,7 +1,7 @@ {% load foundation %} {% if facet_form.fields %} - +
    {% endif %} diff --git a/src/templates/admin/identifiers/manage_identifier.html b/src/templates/admin/identifiers/manage_identifier.html index 736ff765c4..7fda63bdb3 100644 --- a/src/templates/admin/identifiers/manage_identifier.html +++ b/src/templates/admin/identifiers/manage_identifier.html @@ -28,7 +28,7 @@

    {% if identifier %}Edit Identifier - {{ identifier.pk }}{% else %}Add New Id {% csrf_token %} {% include "elements/forms/errors.html" %} {{ form|foundation }} - + {% include "elements/button_save.html" %} {% endblock body %} diff --git a/src/themes/OLH/templates/elements/accounts/edit_profile_body_block.html b/src/themes/OLH/templates/elements/accounts/edit_profile_body_block.html index 9781ac546b..388073b332 100644 --- a/src/themes/OLH/templates/elements/accounts/edit_profile_body_block.html +++ b/src/themes/OLH/templates/elements/accounts/edit_profile_body_block.html @@ -1,3 +1,8 @@ +{% comment %} + This template is deprecated. Use admin/elements/accounts/user_form.html instead. + +{% endcomment %} + {% load i18n roles %} {% user_has_role request 'reader' staff_override=False as reader %} diff --git a/src/themes/OLH/templates/elements/accounts/user_form.html b/src/themes/OLH/templates/elements/accounts/user_form.html index 98f3133d56..f3cc5cfd15 100644 --- a/src/themes/OLH/templates/elements/accounts/user_form.html +++ b/src/themes/OLH/templates/elements/accounts/user_form.html @@ -1,3 +1,7 @@ +{% comment %} + This template is deprecated. Use admin/elements/accounts/user_form.html instead. +{% endcomment %} + {% load foundation %} {% load static %} {% load i18n %} diff --git a/src/themes/clean/templates/elements/accounts/edit_profile_body_block.html b/src/themes/clean/templates/elements/accounts/edit_profile_body_block.html index 6149e150c6..de50b2fa88 100644 --- a/src/themes/clean/templates/elements/accounts/edit_profile_body_block.html +++ b/src/themes/clean/templates/elements/accounts/edit_profile_body_block.html @@ -1,3 +1,7 @@ +{% comment %} + This template is deprecated. Use admin/elements/accounts/user_form.html instead. +{% endcomment %} + {% load i18n roles %} {% user_has_role request 'reader' staff_override=False as reader %} diff --git a/src/themes/clean/templates/elements/accounts/user_form.html b/src/themes/clean/templates/elements/accounts/user_form.html index 88c03c13b2..98f5c97de1 100644 --- a/src/themes/clean/templates/elements/accounts/user_form.html +++ b/src/themes/clean/templates/elements/accounts/user_form.html @@ -1,3 +1,7 @@ +{% comment %} + This template is deprecated. Use admin/elements/accounts/user_form.html instead. +{% endcomment %} + {% load foundation %} {% load bootstrap4 %} {% load static %} diff --git a/src/themes/material/templates/elements/accounts/edit_profile_body_block.html b/src/themes/material/templates/elements/accounts/edit_profile_body_block.html index 882786f695..08650494d5 100644 --- a/src/themes/material/templates/elements/accounts/edit_profile_body_block.html +++ b/src/themes/material/templates/elements/accounts/edit_profile_body_block.html @@ -1,3 +1,7 @@ +{% comment %} + This template is deprecated. Use admin/elements/accounts/user_form.html instead. +{% endcomment %} + {% load i18n roles %} {% user_has_role request 'reader' staff_override=False as reader %} diff --git a/src/utils/admin.py b/src/utils/admin.py index 4c3f682485..4e42fdb734 100755 --- a/src/utils/admin.py +++ b/src/utils/admin.py @@ -53,11 +53,34 @@ class VersionAdmin(admin.ModelAdmin): date_hierarchy = ('date') +class RORImportAdmin(admin.ModelAdmin): + list_display = ('pk', 'status', 'started', 'stopped') + list_filter = ('status', 'started', 'stopped') + search_fields = ('rorimporterror__message', 'records',) + date_hierarchy = ('started') + readonly_fields = ('started', 'stopped', 'status', 'records') + inlines = [ + admin_utils.RORImportErrorInline, + ] + + +class RORImportErrorAdmin(admin.ModelAdmin): + list_display = ('pk', '_first_line') + search_fields = ('message',) + date_hierarchy = ('ror_import__started') + raw_id_fields = ('ror_import', ) + + def _first_line(self, obj): + return obj.message.split('\n')[0] if obj and obj.message else '' + + admin_list = [ (models.LogEntry, LogAdmin), (models.Plugin, PluginAdmin), (models.ImportCacheEntry, ImportCacheAdmin), - (models.Version, VersionAdmin) + (models.Version, VersionAdmin), + (models.RORImport, RORImportAdmin), + (models.RORImportError, RORImportErrorAdmin), ] [admin.site.register(*t) for t in admin_list] diff --git a/src/utils/admin_utils.py b/src/utils/admin_utils.py index e7541d5416..093ca828b8 100644 --- a/src/utils/admin_utils.py +++ b/src/utils/admin_utils.py @@ -297,6 +297,19 @@ class NewsItemInline(admin.TabularInline): raw_id_fields = ('newsitem',) +class RORImportErrorInline(admin.TabularInline): + model = core_models.RORImportError + extra = 0 + readonly_fields = ('message',) + + +class AffiliationInline(admin.TabularInline): + model = core_models.Affiliation + extra = 0 + fields = ('title', 'department', 'organization', + 'is_primary', 'start', 'end') + + class JournalFilterBase(admin.SimpleListFilter): """ A base class for other journal filters diff --git a/src/utils/install.py b/src/utils/install.py index 0ced46e651..4ec8575c26 100755 --- a/src/utils/install.py +++ b/src/utils/install.py @@ -13,9 +13,12 @@ from journal import models from press import models as press_models from utils import setting_handler +from utils.logger import get_logger from submission import models as submission_models from cms import models as cms_models +logger = get_logger(__name__) + def update_settings(journal_object=None, management_command=False, overwrite_with_defaults=False, @@ -83,7 +86,7 @@ def update_settings(journal_object=None, management_command=False, setting.editable_by.add(*roles) if management_command: - print('Parsed setting {0}'.format(item['setting'].get('name'))) + logger.info('Parsed setting {0}'.format(item['setting'].get('name'))) def load_permissions(file_path='utils/install/journal_defaults.json'): diff --git a/src/utils/management/commands/import_ror_data.py b/src/utils/management/commands/import_ror_data.py new file mode 100644 index 0000000000..d92f06bae1 --- /dev/null +++ b/src/utils/management/commands/import_ror_data.py @@ -0,0 +1,63 @@ +from django.conf import settings +from django.core.management.base import BaseCommand +from django.utils import timezone + +from utils.models import RORImport +from core.models import Organization +from utils.logger import get_logger + + +logger = get_logger(__name__) + + +class Command(BaseCommand): + """ + Fetches ROR data and generates Organization records. + """ + + help = "Fetches ROR data and generates Organization records." + + def add_arguments(self, parser): + parser.add_argument( + "--limit", + help="The cap on the number of ROR records to process.", + default=0, + type=int, + ) + return super().add_arguments(parser) + + def handle(self, *args, **options): + limit = options.get("limit", 0) + if not limit and settings.DEBUG: + limit = 100 + logger.info( + f"Setting ROR import limit to {limit} while settings.DEBUG." + "Override by passing --limit to import_ror_data." + ) + + ror_import = RORImport.objects.create() + ror_import.get_records() + + # The import is necessary. + # Check we have the right copy of the data dump. + if ror_import.ongoing: + if not ror_import.previous_import: + ror_import.download_data() + elif ror_import.previous_import.zip_path != ror_import.zip_path: + ror_import.download_data() + + # The data is all downloaded and ready to import. + if ror_import.ongoing: + Organization.import_ror_batch( + ror_import, + limit=limit, + ) + + ror_import.stopped = timezone.now() + ror_import.save() + # The process did not error out, so it can be considered a success. + if ror_import.ongoing: + ror_import.status = ror_import.RORImportStatus.SUCCESSFUL + ror_import.save() + + logger.info(ror_import.status) diff --git a/src/utils/management/commands/install_plugins.py b/src/utils/management/commands/install_plugins.py index 7a8725e987..11789f14b6 100755 --- a/src/utils/management/commands/install_plugins.py +++ b/src/utils/management/commands/install_plugins.py @@ -3,9 +3,12 @@ from django.core.management.base import BaseCommand from core import plugin_loader +from utils.logger import get_logger from importlib import import_module +logger = get_logger(__name__) + class Command(BaseCommand): """A management command to check for and install new plugins.""" @@ -40,13 +43,13 @@ def handle(self, *args, **options): ) for plugin in plugin_dirs: - print('Checking plugin {0}'.format(plugin)) + logger.debug('Checking plugin {0}'.format(plugin)) plugin_module_name = "plugins.{0}.plugin_settings".format(plugin) plugin_settings = import_module(plugin_module_name) plugin_settings.install() for plugin in homepage_dirs: - print('Checking plugin {0}'.format(plugin)) + logger.debug('Checking plugin {0}'.format(plugin)) plugin_module_name = "core.homepage_elements.{0}.plugin_settings".format(plugin) plugin_settings = import_module(plugin_module_name) plugin_settings.install() diff --git a/src/utils/management/commands/match_ror_ids.py b/src/utils/management/commands/match_ror_ids.py new file mode 100644 index 0000000000..c98fb494de --- /dev/null +++ b/src/utils/management/commands/match_ror_ids.py @@ -0,0 +1,44 @@ +from django.conf import settings +from django.core.management.base import BaseCommand + +from utils.models import RORImport +from core.models import Affiliation, Organization +from utils.logger import get_logger +from django.utils import timezone +from datetime import timedelta + + +logger = get_logger(__name__) + + +class Command(BaseCommand): + """ + Matches organization records with ROR data. + """ + + help = "Matches organization records with ROR data." + + def add_arguments(self, parser): + parser.add_argument( + '--ignore_missing_import', + help='Run the match even if no ROR import is found', + action='store_true', + ) + return super().add_arguments(parser) + + def handle(self, *args, **options): + + ror_import = RORImport.objects.filter( + started__gte=timezone.now() - timedelta(days=90) + ).exists() + ignore_missing_import = options.get("ignore_missing_import") + if not ror_import and not ignore_missing_import: + logger.warning( + "There was no ROR import in the last 90 days." + "Run import_ror_data before continuing," + "or pass --ignore_missing_import." + ) + return + uncontrolled_organizations = Organization.objects.filter(ror='') + for organization in uncontrolled_organizations: + organization.deduplicate_to_ror_record() diff --git a/src/utils/migrations/0035_rorimport_rorimporterror.py b/src/utils/migrations/0035_rorimport_rorimporterror.py new file mode 100644 index 0000000000..a4e9a2c86d --- /dev/null +++ b/src/utils/migrations/0035_rorimport_rorimporterror.py @@ -0,0 +1,37 @@ +# Generated by Django 4.2.14 on 2024-07-26 20:51 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('utils', '0038_upgrade_1_7_2'), + ] + + operations = [ + migrations.CreateModel( + name='RORImport', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('started', models.DateTimeField(auto_now_add=True)), + ('stopped', models.DateTimeField(blank=True, null=True)), + ('status', models.CharField(choices=[('ongoing', 'Ongoing'), ('unnecessary', 'Unnecessary'), ('successful', 'Successful'), ('failed', 'Failed')], default='ongoing')), + ('records', models.JSONField(default=dict)), + ], + options={ + 'verbose_name': 'ROR import', + 'verbose_name_plural': 'ROR imports', + 'get_latest_by': 'started', + }, + ), + migrations.CreateModel( + name='RORImportError', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('message', models.TextField(blank=True)), + ('ror_import', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='utils.rorimport')), + ], + ), + ] diff --git a/src/utils/models.py b/src/utils/models.py index 0ad6918794..db57e6407a 100755 --- a/src/utils/models.py +++ b/src/utils/models.py @@ -3,10 +3,12 @@ __license__ = "AGPL v3" __maintainer__ = "Birkbeck Centre for Technology and Publishing" -import json as jason +import json import os from uuid import uuid4 import requests +import tqdm + from requests.packages.urllib3.exceptions import InsecureRequestWarning from django.utils import timezone @@ -16,10 +18,14 @@ from django.conf import settings from django.utils.text import slugify -from utils.shared import get_ip_address, join_lists +from utils.logger import get_logger +from utils.shared import get_ip_address from utils.importers.up import get_input_value_by_name +logger = get_logger(__name__) + + LOG_TYPES = [ ('Email', 'Email'), ('PageView', 'PageView'), @@ -295,7 +301,7 @@ def fetch(url, up_auth_file='', up_base_url='', ojs_auth_file=''): # first, check whether there's an auth file if up_auth_file != '': with open(up_auth_file, 'r', encoding="utf-8") as auth_in: - auth_dict = jason.loads(auth_in.read()) + auth_dict = json.loads(auth_in.read()) do_auth = True username = auth_dict['username'] password = auth_dict['password'] @@ -348,3 +354,170 @@ def fetch(url, up_auth_file='', up_base_url='', ojs_auth_file=''): def __str__(self): return self.url + + +class RORImport(models.Model): + """ + An record of an import of ROR organization data into Janeway. + """ + class RORImportStatus(models.TextChoices): + ONGOING = 'ongoing', 'Ongoing' + UNNECESSARY = 'unnecessary', 'Unnecessary' + SUCCESSFUL = 'successful', 'Successful' + FAILED = 'failed', 'Failed' + + started = models.DateTimeField( + auto_now_add=True, + ) + stopped = models.DateTimeField( + blank=True, + null=True, + ) + status = models.CharField( + choices=RORImportStatus.choices, + default=RORImportStatus.ONGOING, + ) + records = models.JSONField( + default=dict, + ) + + class Meta: + get_latest_by = 'started' + verbose_name = 'ROR import' + verbose_name_plural = 'ROR imports' + + def __str__(self): + return f'{self.status} RORImport started { self.started }' + + @property + def previous_import(self): + try: + return RORImport.objects.exclude(pk=self.pk).latest() + except RORImport.DoesNotExist: + return None + + @property + def new_download_needed(self): + if not self.previous_import: + return True + elif self.previous_import.status == self.RORImportStatus.FAILED: + return True + elif not self.source_data_created or self.source_data_created > self.previous_import.started: + return True + else: + return False + + @property + def zip_path(self): + temp_dir = os.path.join(settings.BASE_DIR, 'files', 'temp') + if not os.path.exists(temp_dir): + os.makedirs(temp_dir) + try: + file_id = self.records['hits']['hits'][0]['files'][-1]['id'] + except (KeyError, AttributeError) as error: + self.fail(error) + return '' + zip_name = f'ror-download-{file_id}.zip' + return os.path.join(temp_dir, zip_name) + + @property + def download_link(self): + try: + return self.records['hits']['hits'][0]['files'][-1]['links']['self'] + except (KeyError, AttributeError) as error: + self.fail(error) + return '' + + @property + def source_data_created(self): + try: + timestamp = self.records['hits']['hits'][0]['created'] + return timezone.datetime.fromisoformat(timestamp) + except (KeyError, AttributeError) as error: + self.fail(error) + return None + + def fail(self, error): + self.stopped = timezone.datetime.now() + self.status = self.RORImportStatus.FAILED + self.save() + logger.error(error) + RORImportError.objects.create(ror_import=self, messsage=error) + + @property + def ongoing(self): + return self.status == self.RORImportStatus.ONGOING + + def get_records(self): + """ + Gets the manifest of available data and checks if it contains + anything new. If there is no new data, or if the previous import failed, + the import is marked as unnecessary. + """ + records_url = 'https://zenodo.org/api/communities/ror-data/records?sort=newest' + try: + response = requests.get(records_url, timeout=settings.HTTP_TIMEOUT_SECONDS) + response.raise_for_status() + self.records = response.json() + self.save() + if not self.new_download_needed: + self.status = self.RORImportStatus.ONGOING + self.save() + except requests.RequestException as error: + self.fail(error) + + def delete_previous_download(self): + if not self.previous_import: + logger.debug('No previous import to remove.') + return + try: + os.unlink(self.previous_import.zip_path) + except FileNotFoundError: + logger.debug('Previous import had no zip file.') + + def download_data(self): + """ + Downloads the current data dump from Zenodo. + Then removes previous files to save space. + """ + try: + response = requests.get( + self.download_link, + timeout=settings.HTTP_TIMEOUT_SECONDS, + stream=True, + ) + response.raise_for_status() + with open(self.zip_path, 'wb') as zip_ref: + for chunk in response.iter_content(chunk_size=128): + zip_ref.write(chunk) + if os.path.exists(self.zip_path): + self.delete_previous_download() + except requests.RequestException as error: + self.fail(error) + + def filter_new_records(self, ror_data, organizations): + """ + Finds new records from the data dump based on + ROR's last_modified time stamp. + """ + timestamps = {} + for ror, ts in organizations.values_list("ror", "ror_record_timestamp"): + timestamps[ror] = ts + filtered_data = [] + for record in tqdm.tqdm(ror_data, desc="Finding new or updated ROR records"): + ror = record.get("id", {}) + last_modified = record.get("admin", {}).get("last_modified", {}) + timestamp = last_modified.get("date", "") + if ror and timestamp and timestamp > timestamps.get(ror, ''): + filtered_data.append(record) + return filtered_data + + +class RORImportError(models.Model): + ror_import = models.ForeignKey( + RORImport, + on_delete=models.CASCADE, + ) + message = models.TextField( + blank=True, + ) diff --git a/src/utils/testing/helpers.py b/src/utils/testing/helpers.py index 0b39f7bceb..67f15d67a3 100755 --- a/src/utils/testing/helpers.py +++ b/src/utils/testing/helpers.py @@ -62,6 +62,7 @@ def create_user(username, roles=None, journal=None, **attrs): journal=journal ) + user.save() for attr, value in attrs.items(): setattr(user, attr, value) @@ -172,6 +173,28 @@ def create_peer_reviewer(journal, **kwargs): return reviewer +def create_affiliation( + institution='', + department='', + account=None, + frozen_author=None, + preprint_author=None, +): + organization = core_models.Organization.objects.create() + core_models.OrganizationName.objects.create( + value=institution, + custom_label_for=organization, + ) + affiliation = core_models.Affiliation.objects.create( + organization=organization, + department=department, + account=account, + frozen_author=frozen_author, + preprint_author=preprint_author, + ) + return affiliation + + def create_author(journal, **kwargs): roles = kwargs.pop('roles', ['author']) email = kwargs.pop('email', "authoruser@martineve.com") @@ -179,8 +202,6 @@ def create_author(journal, **kwargs): "first_name": "Author", "middle_name": "A", "last_name": "User", - "institution": "Author institution", - "department": "Author Department", "biography": "Author test biography" } attrs.update(kwargs) @@ -192,6 +213,11 @@ def create_author(journal, **kwargs): ) author.is_active = True author.save() + create_affiliation( + institution="Author institution", + department="Author Department", + account=author, + ) return author @@ -348,11 +374,15 @@ def create_preprint(repository, author, subject, title='This is a Test Preprint' size=100, ) preprint.submission_file = file - repo_models.PreprintAuthor.objects.create( + preprint_author = repo_models.PreprintAuthor.objects.create( preprint=preprint, account=author, order=1, - affiliation='Made Up University', + ) + preprint_author.save() + create_affiliation( + institution="Made Up University", + preprint_author=preprint_author, ) return preprint