From 7ef83cf4eb8759803ffe4afeda08be7daa53200b Mon Sep 17 00:00:00 2001 From: "Stephen A. Fuqua" Date: Fri, 28 Aug 2020 16:41:49 -0500 Subject: [PATCH] ODS/API 3.1 and Change Queries --- .gitignore | 4 +- Administration/Azure Test Lab.md | 14 +- Administration/ChangeQueryDataSet-Config.json | 8 + Administration/ChangeQueryDataSet.ps1 | 247 +++++++++++++++ ...tabases.sql => RestoreSqlDatabases-v3.sql} | 8 +- Administration/RestoreSqlDatabases-v31.sql | 33 ++ TestRunner.ps1 | 129 ++++++-- docs/generating-change-queries-data-sets.md | 173 +++++++++++ docs/how-to-create-tests.md | 106 ++++++- edfi_performance/api/basic_client/__init__.py | 114 +++++++ edfi_performance/api/client/__init__.py | 64 +++- edfi_performance/api/client/account.py | 38 +-- edfi_performance/api/client/assessment.py | 54 +--- edfi_performance/api/client/bell_schedule.py | 16 +- edfi_performance/api/client/calendar_date.py | 19 +- edfi_performance/api/client/community.py | 32 +- edfi_performance/api/client/composite.py | 4 +- edfi_performance/api/client/course.py | 16 +- .../api/client/course_offering.py | 16 +- edfi_performance/api/client/discipline.py | 32 +- edfi_performance/api/client/education.py | 95 +----- edfi_performance/api/client/grade.py | 17 +- .../api/client/gradebook_entries.py | 17 +- .../api/client/graduation_plan.py | 16 +- edfi_performance/api/client/intervention.py | 32 +- edfi_performance/api/client/post_secondary.py | 16 +- edfi_performance/api/client/report_card.py | 16 +- edfi_performance/api/client/section.py | 20 +- edfi_performance/api/client/session.py | 13 +- edfi_performance/api/client/staff.py | 63 +--- edfi_performance/api/client/student.py | 288 +++--------------- edfi_performance/config/__init__.py | 38 ++- .../factories/resources/assessment.py | 64 ++-- .../factories/resources/student.py | 5 +- edfi_performance/tasks/__init__.py | 4 + .../tasks/change_query/__init__.py | 128 ++++++++ edfi_performance/tasks/change_query/course.py | 10 + .../tasks/change_query/course_offering.py | 10 + .../tasks/change_query/education.py | 10 + edfi_performance/tasks/change_query/school.py | 10 + .../tasks/change_query/section.py | 10 + .../tasks/change_query/session.py | 10 + edfi_performance/tasks/change_query/staff.py | 14 + .../tasks/change_query/student.py | 22 ++ .../tasks/pipeclean/descriptors.py | 1 + locust-config.json | 1 + package.ps1 | 1 + pipeclean_tests.py | 4 +- 48 files changed, 1265 insertions(+), 797 deletions(-) create mode 100644 Administration/ChangeQueryDataSet-Config.json create mode 100644 Administration/ChangeQueryDataSet.ps1 rename Administration/{RestoreSqlDatabases.sql => RestoreSqlDatabases-v3.sql} (75%) create mode 100644 Administration/RestoreSqlDatabases-v31.sql create mode 100644 docs/generating-change-queries-data-sets.md create mode 100644 edfi_performance/api/basic_client/__init__.py create mode 100644 edfi_performance/tasks/change_query/__init__.py create mode 100644 edfi_performance/tasks/change_query/course.py create mode 100644 edfi_performance/tasks/change_query/course_offering.py create mode 100644 edfi_performance/tasks/change_query/education.py create mode 100644 edfi_performance/tasks/change_query/school.py create mode 100644 edfi_performance/tasks/change_query/section.py create mode 100644 edfi_performance/tasks/change_query/session.py create mode 100644 edfi_performance/tasks/change_query/staff.py create mode 100644 edfi_performance/tasks/change_query/student.py diff --git a/.gitignore b/.gitignore index 08325efb..05785962 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,5 @@ /artifacts/ /TestResults/ *.pdf - -node_modules/ \ No newline at end of file +change_version_tracker.json +node_modules/ diff --git a/Administration/Azure Test Lab.md b/Administration/Azure Test Lab.md index d7241a42..c0d7fd44 100644 --- a/Administration/Azure Test Lab.md +++ b/Administration/Azure Test Lab.md @@ -93,7 +93,7 @@ Steps taken in the Azure Portal: Initial database state is that of a development environment after 'initdev'. Databases BAK created on developer machine, copied to VM over RDP. - See RestoreSqlDatabases.sql for the restore script. Note how the Data Disk is deliberately used for both backups and restored databases. + See RestoreSqlDatabases-v3.sql for the restore script. Note how the Data Disk is deliberately used for both backups and restored databases. Then, copy extract the Northridge 3.0 Dataset zip to the VM, extracting its BAK to F:\Database Backups\ See https://s3.amazonaws.com/edfi_ods_samples/v3.0/EdFi_Ods_Northridge.7z @@ -387,22 +387,26 @@ Steps taken in the Azure Portal: Create Desktop Shortcuts: - Target: C:\Windows\System32\cmd.exe /k "C:\Users\edFiAdmin\run-deployed-tests.bat volume" + Target: C:\Windows\System32\cmd.exe /k "C:\Users\edFiAdmin\run-deployed-tests.bat Volume" Start in: C:\Users\edFiAdmin Shortcut icon text: Run Volume Tests - Target: C:\Windows\System32\cmd.exe /k "C:\Users\edFiAdmin\run-deployed-tests.bat pipeclean" + Target: C:\Windows\System32\cmd.exe /k "C:\Users\edFiAdmin\run-deployed-tests.bat Pipeclean" Start in: C:\Users\edFiAdmin Shortcut icon text: Run Pipeclean Tests - Target: C:\Windows\System32\cmd.exe /k "C:\Users\edFiAdmin\run-deployed-tests.bat stress" + Target: C:\Windows\System32\cmd.exe /k "C:\Users\edFiAdmin\run-deployed-tests.bat Stress" Start in: C:\Users\edFiAdmin Shortcut icon text: Run Stress Tests - Target: C:\Windows\System32\cmd.exe /k "C:\Users\edFiAdmin\run-deployed-tests.bat soak" + Target: C:\Windows\System32\cmd.exe /k "C:\Users\edFiAdmin\run-deployed-tests.bat Soak" Start in: C:\Users\edFiAdmin Shortcut icon text: Run Soak Tests + Target: C:\Windows\System32\cmd.exe /k "C:\Users\edFiAdmin\run-deployed-tests.bat ChangeQuery" + Start in: C:\Users\edFiAdmin + Shortcut icon text: Run Change Query Tests + Register Credentials for the SQL and Web VMs Copy AzureTestLab.ps1 to C:\Users\edFiAdmin. diff --git a/Administration/ChangeQueryDataSet-Config.json b/Administration/ChangeQueryDataSet-Config.json new file mode 100644 index 00000000..37b9160c --- /dev/null +++ b/Administration/ChangeQueryDataSet-Config.json @@ -0,0 +1,8 @@ +{ + "apiHost": "http://localhost:54746", + "apiVersion": "3", + "clientId": "minimalSandbox", + "clientSecret": "minimalSandboxSecret", + "databaseName": "EdFi_Ods_Sandbox_minimalSandbox", + "schoolYear": 2018 +} diff --git a/Administration/ChangeQueryDataSet.ps1 b/Administration/ChangeQueryDataSet.ps1 new file mode 100644 index 00000000..438d3b9e --- /dev/null +++ b/Administration/ChangeQueryDataSet.ps1 @@ -0,0 +1,247 @@ +function task($heading, $command, $path) { + write-host + write-host $heading -fore CYAN + execute $command $path +} + +function execute($command, $path) { + if ($path -eq $null) { + $global:lastexitcode = 0 + & $command + } else { + Push-Location $path + $global:lastexitcode = 0 + & $command + Pop-Location + } + if ($lastexitcode -ne 0) { + throw "Error executing command: $command" + } +} + +function Get-Config { + $configJson = Get-Content 'ChangeQueryDataSet-Config.json' | Out-String + $global:config = $configJson | ConvertFrom-Json +} + +function Get-LocustConfig { + $configJson = Get-Content '../locust-config.json' | Out-String + $global:locustConfig = $configJson | ConvertFrom-Json + $locustConfig.change_query_backup_filenames = New-Object System.Collections.ArrayList + $locustConfig.restore_database = "true" +} + +function Update-LocustConfig($currentDataPeriod) { + $locustConfig.change_query_backup_filenames.Add("$($config.databaseName)_DataPeriod_$currentDataPeriod.bak") + $locustConfig | ConvertTo-Json | Set-Content -Path '../locust-config.json' +} + +function Create-Backup { + param( + [Int]$dataPeriodNumber=$(throw "-dataPeriodNumber argument is required. Must provide the data period number."), + [string]$sqlBackupPath="", + [string]$databaseName="") + if ($sqlBackupPath -eq "") { + Get-LocustConfig + $sqlBackupPath = $locustConfig.sql_backup_path + } + if ($databaseName -eq "") { + Get-Config + $databaseName = $config.databaseName + } + $filePath = Join-Path $sqlBackupPath "$($databaseName)_DataPeriod_$dataPeriodNumber.bak" + Invoke-SqlCmd -Database "master" ` + -Query "ALTER DATABASE $($databaseName) SET SINGLE_USER WITH ROLLBACK IMMEDIATE; + BACKUP DATABASE $($databaseName) TO DISK = '$($filePath)'; + ALTER DATABASE $($databaseName) SET MULTI_USER;" -QueryTimeout 0 + write-host "Backup file created for Data Period: $($dataPeriodNumber)" +} + +function ApiClientLoader { + param( + [Int]$dataPeriodNumber=$(throw "-dataPeriodNumber argument is required. Must provide the data period number."), + [string]$apiLoader=$(throw "-apiLoader argument is required. Must provide the full file path to the folder containing the EdFi.ApiLoader.Console executable."), + [string]$working=$(throw "-working argument is required. Must provide the full file path to a writable folder containing the working files for the Ed-Fi ApiLoader."), + [string]$xml=$(throw "-xml argument is required. Must provide the full file path to the folder containing the xml files created by the SDG."), + [string]$xsd=$(throw "-xsd argument is required. Must provide the full file path to the folder containing the Ed-Fi Xsd Schema files e.g. 'C:\dev\Ed-Fi-Standard\v3.1\Schemas\Bulk'.")) + # Useful when using a 31 (assessments) build of the api client loader against a corresponding ODS. + # Note how this is using a specially-built api loader from a branch that claimed to know those Assessments changes. + # Specifying /c 1 and /l 1 are maybe useful during troubleshooting but should be removed in general. + Get-Config + $outputPath = Resolve-Path . + task $config.databaseName { + .\EdFi.ApiLoader.Console.exe ` + /a "$($config.apiHost)/data/v$($config.apiVersion)/" ` + /m "$($config.apiHost)/metadata" ` + /o $config.apiHost ` + /k $config.clientId ` + /s $config.clientSecret ` + /d "$($xml)\Change Events Performance Test - Data Period $dataPeriodNumber" ` + /x $xsd ` + /w $working ` + /y $config.schoolYear ` + /c 1 ` + /l 1 ` + /f ` + 2>&1 | tee "$($outputPath)\DataPeriod_$dataPeriodNumber.txt" + } $apiLoader +} + +function Run-ApiClientLoader { + param( + [Int]$lastDataPeriod=$(throw "-lastDataPeriod argument is required. Must provide the number for the last data period."), + [string]$apiLoader=$(throw "-apiLoader argument is required. Must provide the full file path to the folder containing the EdFi.ApiLoader.Console executable."), + [string]$working=$(throw "-working argument is required. Must provide the full file path to a writable folder containing the working files for the Ed-Fi ApiLoader."), + [string]$xml=$(throw "-xml argument is required. Must provide the full file path to the folder containing the xml files created by the SDG."), + [string]$xsd=$(throw "-xsd argument is required. Must provide the full file path to the folder containing the Ed-Fi Xsd Schema files e.g. 'C:\dev\Ed-Fi-Standard\v3.1\Schemas\Bulk'."), + [Int]$firstDataPeriod=1, + [Bool]$updateConfig=$FALSE) + # The $lastDataPeriod argument and the optional $firstDataPeriod argument allow the user to run the ApiClientLoader + # for a particular range of data periods (e.g. data periods 2 to 5). + Initialize-Folder $working + Get-LocustConfig + for ($i = $firstDataPeriod; $i -le $lastDataPeriod; $i++) { + ApiClientLoader ` + -DataPeriodNumber $i ` + -ApiLoader $apiLoader ` + -Working $working ` + -Xml $xml ` + -Xsd $xsd + Create-Backup ` + -DataPeriodNumber $i ` + -SqlBackupPath $locustConfig.sql_backup_path ` + -DatabaseName $config.databaseName + if ($updateConfig) { + Update-LocustConfig $i + } + } +} + +function Initialize-Folder($path) { + if (!(Test-Path $path)) { + New-Item -ItemType Directory -Force -Path $path | Out-Null + } else { + Get-ChildItem -Path $path -Recurse | Remove-Item -Recurse + } +} + +function Sort-XmlFiles { + param($xml=$(throw "-xml argument is required. Must provide the full file path to the folder containing the xml files created by the SDG.")) + $dataPeriodNumber = 1 + $isDataPresent = $TRUE + while ($isDataPresent -eq $TRUE) { + $searchTerm = "Change Events Performance Test - Data Period $dataPeriodNumber" + $dataPeriodFolder = Join-Path $xml $searchTerm + $dataPeriodFiles = Get-ChildItem -File -Path $xml -Filter "*$searchTerm*" + if ($dataPeriodFiles.Count -eq 0 -and !(Test-Path -Path $dataPeriodFolder)) { + $isDataPresent = $FALSE + } elseif ($dataPeriodFiles.Count -eq 0 -and (Test-Path -Path $dataPeriodFolder)) { + $dataPeriodNumber = $dataPeriodNumber + 1 + } else { + New-Item -ItemType Directory -Force -Path $dataPeriodFolder | Out-Null + foreach ($file in $dataPeriodFiles) { + $file | Move-Item -Destination $dataPeriodFolder + } + $dataPeriodNumber = $dataPeriodNumber + 1 + } + } + $dataPeriodCount = $dataPeriodNumber - 1 + write-host "$dataPeriodCount Data Periods were sorted." + return $dataPeriodCount +} + +function GenerateAllSQLBackups { + param( + [string]$apiLoader=$(throw "-apiLoader argument is required. Must provide the full file path to the folder containing the EdFi.ApiLoader.Console executable."), + [string]$working=$(throw "-working argument is required. Must provide the full file path to a writable folder containing the working files for the Ed-Fi ApiLoader."), + [string]$xml=$(throw "-xml argument is required. Must provide the full file path to the folder containing the xml files created by the SDG."), + [string]$xsd=$(throw "-xsd argument is required. Must provide the full file path to the folder containing the Ed-Fi Xsd Schema files e.g. 'C:\dev\Ed-Fi-Standard\v3.1\Schemas\Bulk'."), + [Bool]$updateConfig=$FALSE) + $dataPeriodCount = Sort-XmlFiles $xml + Run-ApiClientLoader ` + -LastDataPeriod $dataPeriodCount ` + -ApiLoader $apiLoader ` + -Working $working ` + -Xml $xml ` + -Xsd $xsd ` + -UpdateConfig $updateConfig +} + +function Info { + write-host + write-host "FUNCTION LIST" -fore YELLOW + write-host "-------------" + write-host + write-host "GenerateALLSQLBackups" -fore GREEN + write-host "Purpose:" -fore YELLOW -NoNewLine + write-host " Sort xml files, run ApiLoader for each data period and create backups for each data period." + write-host "Required Arguments:" -fore YELLOW + write-host "-apiLoader" -fore CYAN -NoNewLine + write-host " Full file path to the folder containing the EdFi.ApiLoader.Console executable." + write-host "-working" -fore CYAN -NoNewLine + write-host " Full file path to a writable folder containing the working files for the Ed-Fi ApiLoader." + write-host "-xml" -fore CYAN -NoNewLine + write-host " Full file path to the folder containing the xml files created by the SDG." + write-host "-xsd" -fore CYAN -NoNewLine + write-host " Full file path to the folder containing the Ed-Fi Xsd Schema files e.g. 'C:\dev\Ed-Fi-Standard\v3.1\Schemas\Bulk'." + write-host "Optional Argument:" -fore YELLOW + write-host "-updateConfig" -fore CYAN -NoNewLine + write-host " If set to $TRUE, the locust-config.json file will be updated automatically so that the ChangeQuery test can be run without needing to update the config file. Default value is $FALSE." + write-host + write-host "Sort-XmlFiles" -fore GREEN + write-host "Purpose:" -fore YELLOW -NoNewLine + write-host " Sort xml files created by the SDG." + write-host "Required Argument:" -fore YELLOW + write-host "-xml" -fore CYAN -NoNewLine + write-host " Full file path to the folder containing the xml files created by the SDG." + write-host + write-host "Run-ApiClientLoader" -fore GREEN + write-host "Purpose:" -fore YELLOW -NoNewLine + write-host " Run the ApiLoader for a particular range of data periods and create backups for each data period. It does not sort the xml files and assumes that the xml files have been sorted previously." + write-host "Required Arguments:" -fore YELLOW + write-host "-lastDataPeriod" -fore CYAN -NoNewLine + write-host " In the normal case of 7 data periods, this should be set to 7. To run the ApiLoader for a particular range of data periods, this argument should be set to the max of the range." + write-host "-apiLoader" -fore CYAN -NoNewLine + write-host " Full file path to the folder containing the EdFi.ApiLoader.Console executable." + write-host "-working" -fore CYAN -NoNewLine + write-host " Full file path to a writable folder containing the working files for the Ed-Fi ApiLoader." + write-host "-xml" -fore CYAN -NoNewLine + write-host " Full file path to the folder containing the xml files created by the SDG." + write-host "-xsd" -fore CYAN -NoNewLine + write-host " Full file path to the folder containing the Ed-Fi Xsd Schema files e.g. 'C:\dev\Ed-Fi-Standard\v3.1\Schemas\Bulk'." + write-host "Optional Arguments:" -fore YELLOW + write-host "-firstDataPeriod" -fore CYAN -NoNewLine + write-host " Its default value is 1. To run the ApiLoader for a particular range of data periods, this argument should be set to the min of the range." + write-host "-updateConfig" -fore CYAN -NoNewLine + write-host " If set to $TRUE, the locust-config.json file will be updated automatically so that the ChangeQuery test can be run without needing to update the config file. Default value is $FALSE." + write-host + write-host "Create-Backup" -fore GREEN + write-host "Purpose:" -fore YELLOW -NoNewLine + write-host " Create database backups." + write-host "Required Argument:" -fore YELLOW + write-host "-dataPeriodNumber" -fore CYAN -NoNewLine + write-host " Integer value used to name the database backup file correctly." + write-host "Optional Arguments:" -fore YELLOW + write-host "-sqlBackupPath" -fore CYAN -NoNewLine + write-host " Full file path to the folder where the SQL backup file should be stored. If not set, it defaults to the value provided in the locust-config.json file." + write-host "-databaseName" -fore CYAN -NoNewLine + write-host " Name of database that the backup is being generated for e.g. 'EdFi_Ods_Sandbox_minimalSandbox'. If not set, it defaults to the database name provided in the ChangeQueryDataSet.json file." + write-host + write-host "ApiClientLoader" -fore GREEN + write-host "Purpose:" -fore YELLOW -NoNewLine + write-host " Run ApiLoader for only one data period. Does not create database backup. This function also requires that the xml files have already been sorted." + write-host "Required Arguments:" -fore YELLOW + write-host "-dataPeriodNumber" -fore CYAN -NoNewLine + write-host " Integer value used to grab the xml files for the correct data period." + write-host "-apiLoader" -fore CYAN -NoNewLine + write-host " Full file path to the folder containing the EdFi.ApiLoader.Console executable." + write-host "-working" -fore CYAN -NoNewLine + write-host " Full file path to a writable folder containing the working files for the Ed-Fi ApiLoader." + write-host "-xml" -fore CYAN -NoNewLine + write-host " Full file path to the folder containing the xml files created by the SDG." + write-host "-xsd" -fore CYAN -NoNewLine + write-host " Full file path to the folder containing the Ed-Fi Xsd Schema files e.g. 'C:\dev\Ed-Fi-Standard\v3.1\Schemas\Bulk'." + write-host +} + +Info diff --git a/Administration/RestoreSqlDatabases.sql b/Administration/RestoreSqlDatabases-v3.sql similarity index 75% rename from Administration/RestoreSqlDatabases.sql rename to Administration/RestoreSqlDatabases-v3.sql index 8d2136cc..7b783eb8 100644 --- a/Administration/RestoreSqlDatabases.sql +++ b/Administration/RestoreSqlDatabases-v3.sql @@ -1,9 +1,5 @@ --- SPDX-License-Identifier: Apache-2.0 --- Licensed to the Ed-Fi Alliance under one or more agreements. --- The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. --- See the LICENSE and NOTICES files in the project root for more information. - --- This script restores databases on the Azure SQL Server VM. It assumes the *.bak files have been copied into the VM under F:\Database Backups\ +-- This script restores initial-state databases on the Azure SQL Server VM. It assumes the *.bak files have been copied into the VM under +-- F:\Database Backups\ -- and restores them to the data disk at F:\Data. RESTORE DATABASE EdFi_Admin FROM DISK = 'F:\Database Backups\EdFi_Admin.bak' diff --git a/Administration/RestoreSqlDatabases-v31.sql b/Administration/RestoreSqlDatabases-v31.sql new file mode 100644 index 00000000..0525f065 --- /dev/null +++ b/Administration/RestoreSqlDatabases-v31.sql @@ -0,0 +1,33 @@ +-- This script restores initial-state databases on the Azure SQL Server VM. It assumes the *.bak files have been copied into the VM under +-- F:\Database Backups\2018-11-25 v3 Change Query Backups\Initial State\ +-- and restores them to the data disk at F:\Data. + +RESTORE DATABASE EdFi_Admin FROM DISK = 'F:\Database Backups\2018-11-25 v3 Change Query Backups\Initial State\EdFi_Admin.bak' +WITH + MOVE 'EdFi_Admin' TO 'F:\DATA\EdFi_Admin.mdf', + MOVE 'EdFi_Admin_log' TO 'F:\DATA\EdFi_Admin_log.ldf', + REPLACE; + +RESTORE DATABASE EdFi_Bulk FROM DISK = 'F:\Database Backups\2018-11-25 v3 Change Query Backups\Initial State\EdFi_Bulk.bak' +WITH + MOVE 'EdFi_Bulk' TO 'F:\DATA\EdFi_Bulk.mdf', + MOVE 'EdFi_Bulk_log' TO 'F:\DATA\EdFi_Bulk_log.ldf', + REPLACE; + +RESTORE DATABASE EdFi_Security FROM DISK = 'F:\Database Backups\2018-11-25 v3 Change Query Backups\Initial State\EdFi_Security.bak' +WITH + MOVE 'EdFi_Security' TO 'F:\DATA\EdFi_Security.mdf', + MOVE 'EdFi_Security_log' TO 'F:\DATA\EdFi_Security_log.ldf', + REPLACE; + +RESTORE DATABASE EdFi_Ods_Sandbox_populatedSandbox FROM DISK = 'F:\Database Backups\2018-11-25 v3 Change Query Backups\Initial State\EdFi_Ods_Sandbox_populatedSandbox.bak' +WITH + MOVE 'EdFi_Ods_Populated_Template' TO 'F:\DATA\EdFi_Ods_Sandbox_populatedSandbox.mdf', + MOVE 'EdFi_Ods_Populated_Template_log' TO 'F:\DATA\EdFi_Ods_Sandbox_populatedSandbox_log.ldf', + REPLACE; + +RESTORE DATABASE EdFi_Ods_Sandbox_minimalSandbox FROM DISK = 'F:\Database Backups\2018-11-25 v3 Change Query Backups\Initial State\EdFi_Ods_Sandbox_minimalSandbox.bak' +WITH + MOVE 'EdFi_Ods_Minimal_Template' TO 'F:\DATA\EdFi_Ods_Sandbox_minimalSandbox.mdf', + MOVE 'EdFi_Ods_Minimal_Template_log' TO 'F:\DATA\EdFi_Ods_Sandbox_minimalSandbox_log.ldf', + REPLACE; diff --git a/TestRunner.ps1 b/TestRunner.ps1 index e1a814ea..9b6d143e 100644 --- a/TestRunner.ps1 +++ b/TestRunner.ps1 @@ -53,9 +53,9 @@ function Duplicate-OdsLogs { } } -function Invoke-RemoteCommand($server, [PSCredential] $credential, [ScriptBlock] $scriptBlock) { +function Invoke-RemoteCommand($server, [PSCredential] $credential, $argumentList, [ScriptBlock] $scriptBlock) { if ($server -eq 'localhost') { - return Invoke-Command -ScriptBlock $scriptBlock + return Invoke-Command -ArgumentList $argumentList -ScriptBlock $scriptBlock } $thisFolder = $PSScriptRoot @@ -72,7 +72,7 @@ function Invoke-RemoteCommand($server, [PSCredential] $credential, [ScriptBlock] Set-GlobalsFromConfig $configJson } - return Invoke-Command -Session $psSession -ScriptBlock $scriptBlock + return Invoke-Command -Session $psSession -ArgumentList $argumentList -ScriptBlock $scriptBlock } finally { if ($null -ne $psSession) { @@ -85,19 +85,24 @@ function Log($message) { $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss,fff" $output = "[$timestamp] $message" Write-Host $output - $output | Out-File -Encoding UTF8 TestResults\PerformanceTesterLog.txt -Append + $output | Out-File -Encoding UTF8 $testResultsPath\PerformanceTesterLog.txt -Append } -function Reset-OdsDatabase { +function Reset-OdsDatabase($backupFilename) { Import-Module SQLPS + $logicalName = "EdFi_Ods_Populated_Template" + if ($backupFilename -Match "Minimal") { + $logicalName = "EdFi_Ods_Minimal_Template" + } + Invoke-SqlCmd -Database "master" ` -Query "ALTER DATABASE [$databaseName] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; RESTORE DATABASE [$databaseName] FROM DISK = '$sqlBackupPath\$backupFilename' WITH - MOVE 'EdFi_Ods_Empty' TO '$sqlDataPath\$($databaseName).mdf', - MOVE 'EdFi_Ods_Empty_log' TO '$sqlDataPath\$($databaseName)_log.ldf', + MOVE '$($logicalName)' TO '$sqlDataPath\$($databaseName).mdf', + MOVE '$($logicalName)_log' TO '$sqlDataPath\$($databaseName)_log.ldf', REPLACE; ALTER DATABASE [$databaseName] SET MULTI_USER;" -QueryTimeout 0 @@ -140,10 +145,12 @@ function Reset-OdsDatabase { } # Runs the specified test suite for the specified run time. -function Invoke-TestRunner($testSuite, $clientCount, $hatchRate, $runTime, $testType) { - Get-ChildItem -Path TestResults -Recurse | Remove-Item -Recurse - if (!(Test-Path TestResults)) { New-Item -ItemType Directory -Force -Path TestResults | Out-Null} - Log "Writing to TestResults Folder" +function Invoke-TestRunner($testSuite, $clientCount, $hatchRate, $testType, $backupFilename, $runTime) { + Log "Writing to $testResultsPath" + + if (($testSuite -eq "volume") -and ($null -eq $runTime)) { + throw "The -RunTime parameter must be provided when running the 'volume' suite of tests, so that the test run can exit gracefully." + } $databaseCredential = Get-CredentialOrDefault $databaseServer $webCredential = Get-CredentialOrDefault $webServer @@ -151,7 +158,7 @@ function Invoke-TestRunner($testSuite, $clientCount, $hatchRate, $runTime, $test if($restoreDatabase) { Log "Resetting ODS database from backup, and resetting indexes" Log "Restoring $databaseName from $sqlBackupPath\$backupFilename" - Invoke-RemoteCommand $databaseServer $databaseCredential { Reset-OdsDatabase } + Invoke-RemoteCommand $databaseServer $databaseCredential -ArgumentList @($backupFilename) -ScriptBlock { param($backupFilename) Reset-OdsDatabase $backupFilename } } else { Log "Skipping Database Restore" } @@ -177,7 +184,7 @@ function Invoke-TestRunner($testSuite, $clientCount, $hatchRate, $runTime, $test $commonFunctions = (Get-Command $Using:thisFolder\TestRunner.ps1).ScriptContents try { - $csvPath = "$Using:thisFolder\TestResults\$Using:testType.$server.csv" + $csvPath = "$Using:thisFolder\$Using:testResultsPath\$Using:testType.$server.csv" if ($server -eq 'localhost') { $actualPerformanceCounters = Compare-Object (typeperf -q) $expectedPerformanceCounters -PassThru -IncludeEqual -ExcludeDifferent @@ -245,10 +252,17 @@ function Invoke-TestRunner($testSuite, $clientCount, $hatchRate, $runTime, $test } } - Log "Running $testSuite tests with $clientCount clients for $runTime..." + $runTimeArgument = "" + if ($null -eq $runTime) { + Log "Running $testSuite tests with $clientCount clients..." + } else { + Log "Running $testSuite tests with $clientCount clients for $runTime..." + $runTimeArgument = "--run-time $runTime" + } + $locustProcess = Start-Process "locust" -PassThru -NoNewWindow -ArgumentList ` - "-f $($testSuite)_tests.py -c $clientCount -r $hatchRate --no-web --csv TestResults\$testType --run-time $runTime --only-summary" ` - -RedirectStandardError TestResults\Summary.txt + "-f $($testSuite)_tests.py -c $clientCount -r $hatchRate --no-web --csv $testResultsPath\$testType $runTimeArgument --only-summary" ` + -RedirectStandardError $testResultsPath\Summary.txt Wait-Process -Id $locustProcess.Id Log "Test runner process complete" @@ -276,15 +290,15 @@ function Invoke-TestRunner($testSuite, $clientCount, $hatchRate, $runTime, $test } Log "Fetching new ODS log file content from $webServer" - Invoke-RemoteCommand $webServer $webCredential { Duplicate-OdsLogs } + Invoke-RemoteCommand $webServer $webCredential -ScriptBlock { Duplicate-OdsLogs } $logFiles = (Join-Path $logFilePath "OdsLogs") if ($webServer -eq 'localhost') { - Copy-Item $logFiles -Destination TestResults -Recurse + Copy-Item $logFiles -Destination $testResultsPath -Recurse } else { $sessionOptions = New-PSSessionOption -SkipCACheck -SkipCNCheck $psSession = New-PSSession -ComputerName $webServer -Credential $webCredential -UseSSL -SessionOption $sessionOptions -ErrorAction Stop - Copy-Item $logFiles -Destination TestResults -FromSession $psSession -Recurse + Copy-Item $logFiles -Destination $testResultsPath -FromSession $psSession -Recurse } } @@ -297,6 +311,7 @@ function Set-GlobalsFromConfig($configJson) { $global:sqlDataPath = $configValues.sql_data_path $global:databaseName = $configValues.database_name $global:backupFilename = $configValues.backup_filename + $global:changeQueryBackupFilenames = $configValues.change_query_backup_filenames $global:restoreDatabase = [Bool]::Parse($configValues.restore_database) } @@ -304,6 +319,17 @@ function Read-TestRunnerConfig { $global:configJson = Get-Content 'locust-config.json' | Out-String } +function Set-ChangeVersionTracker { + $changeVersion = @{ + newest_change_version = 0 + } + $changeVersion | ConvertTo-Json | Set-Content -Path 'change_version_tracker.json' +} + +function Duplicate-ChangeVersionTracker { + Copy-Item 'change_version_tracker.json' -Destination $testResultsPath +} + function Get-CredentialOrDefault($server) { if ($server -eq 'localhost') { return $null @@ -323,46 +349,85 @@ function Get-CredentialOrDefault($server) { } } -function Invoke-VolumeTests { +function Initialize-Folder($path) { + Get-ChildItem -Path $path -Recurse | Remove-Item -Recurse + if (!(Test-Path $path)) { + New-Item -ItemType Directory -Force -Path $path | Out-Null + } +} + +function Initialize-TestRunner { Read-TestRunnerConfig Set-GlobalsFromConfig $configJson + + # Default to a single TestResults folder. This may be overridden, + # such as for Change Query tests, which write to multiple subfolders + # underneath TestResults. + $global:testResultsPath = "TestResults" + Initialize-Folder TestResults +} + +function Invoke-VolumeTests { + Initialize-TestRunner Invoke-TestRunner ` -TestSuite volume ` -ClientCount 50 ` -HatchRate 1 ` -RunTime 30m ` - -TestType volume + -TestType volume ` + -BackupFilename $backupFilename } function Invoke-PipecleanTests { - Read-TestRunnerConfig - Set-GlobalsFromConfig $configJson + Initialize-TestRunner Invoke-TestRunner ` -TestSuite pipeclean ` -ClientCount 1 ` -HatchRate 1 ` - -RunTime 30m ` - -TestType pipeclean + -TestType pipeclean ` + -BackupFilename $backupFilename } function Invoke-StressTests { - Read-TestRunnerConfig - Set-GlobalsFromConfig $configJson + Initialize-TestRunner Invoke-TestRunner ` -TestSuite volume ` -ClientCount 1000 ` -HatchRate 25 ` -RunTime 30m ` - -TestType stress + -TestType stress ` + -BackupFilename $backupFilename } function Invoke-SoakTests { - Read-TestRunnerConfig - Set-GlobalsFromConfig $configJson + Initialize-TestRunner Invoke-TestRunner ` -TestSuite volume ` -ClientCount 500 ` -HatchRate 1 ` -RunTime 48h ` - -TestType soak -} \ No newline at end of file + -TestType soak ` + -BackupFilename $backupFilename +} + +function Invoke-ChangeQueryTests { + Initialize-TestRunner + + Set-ChangeVersionTracker + + $iteration = 1 + foreach ($changeQueryBackupFilename in $changeQueryBackupFilenames) { + $global:testResultsPath = "TestResults\TestResult_$iteration" + $iteration = $iteration + 1 + + Initialize-Folder $testResultsPath + + Invoke-TestRunner ` + -TestSuite change_query ` + -ClientCount 1 ` + -HatchRate 1 ` + -TestType change_query ` + -BackupFilename $changeQueryBackupFilename + Duplicate-ChangeVersionTracker + } +} diff --git a/docs/generating-change-queries-data-sets.md b/docs/generating-change-queries-data-sets.md new file mode 100644 index 00000000..414d4957 --- /dev/null +++ b/docs/generating-change-queries-data-sets.md @@ -0,0 +1,173 @@ +# Generating Change Queries Data Sets + +## How to Generate Database Backup Files + +### 1. Use the Ed-Fi SDG to create Xml files + +Configure the Ed-Fi Sample Data Generator (SDG) using multiple Data Periods like so: + + + 2016-08-22 + 2016-09-29 + + + + 2016-09-30 + 2016-11-08 + + + + 2016-11-09 + 2016-12-17 + + + + 2016-12-18 + 2017-01-26 + + + + 2017-01-27 + 2017-03-07 + + + + 2017-03-08 + 2017-04-16 + + + + 2017-04-17 + 2017-05-26 + + +Make note of the full file path of wherever the resulting xml files are stored. This file path is a required argument +in future steps. + +### 2. Ensure that you have Ed-Fi-Standard + +The ChangeQueryDataSet script requires that the user has the Ed-Fi-Standard repository downloaded on their machine. +Make sure you are on the correct branch for the version you are trying to test and make note of the full file path of +wherever the schema files are stored for the particular version you are testing e.g. +"C:\dev\Ed-Fi-Standard\v3.1\Schemas\Bulk". + +### 3. Ensure that you have EdFi.ApiLoader.Console + +The EdFi.ApiLoader.Console is required to run the ChangeQueryDataSet script. Make note of the file path of the folder +that has the EdFi.ApiLoader.Console executable as this file path is a required argument in future steps. + +### 4. Start running the ODS API + +The Ed-Fi-ODS documentation provides instructions on setting up and running the ODS API. + +### 5. Review values in Config + +Review the values in the ChangeQueryDataSet-Config.json file and confirm that the values are correct for your setup and +tests that you are trying to run. + +### 6. Run the ChangeQueryDataSet PowerShell script and associated function(s) + +In PowerShell navigate to the Administration folder and run: + + ``` + . .\ChangeQueryDataSet.ps1 + ``` + +This ensures that you can access all of the functions within the ChangeQueryDataSet script. There are five functions in +the ChangeQueryDataSet script that are useful for creating the Change Query data sets but the GenerateALLSQLBackups +function is all that is needed in most circumstances: + +1. GenerateAllSQLBackups + * This function is used to sort the xml files, run the ApiLoader for each data period and create database backups + for each data period. In most cases this is the **only function** that will need to be run. The other functions + listed below are for special circumstances when the user may want greater control over which steps are done and in + what order. + * The required arguments are: + * **apiLoader** : The full file path to the folder containing the EdFi.ApiLoader.Console executable (file path + acquired in Step 3 above). + * **working** : The full file path to a writable folder containing the working files for the Ed-Fi ApiLoader + e.g. "C:\Temp\API Client Working Folder". + * **xml** : The full file path to the folder containing the xml files created by the SDG (file path acquired in + Step 1 above). + * **xsd** : The full file path to the folder containing the Ed-Fi Xsd Schema files e.g. + "C:\dev\Ed-Fi-Standard\v3.1\Schemas\Bulk" (file path acquired in Step 2 above). + * The optional argument is: + * **updateConfig** - If set to $TRUE, the locust-config.json file will be updated automatically so that the + ChangeQuery test can be run without needing to update the config file. Its default value is $FALSE. + * The output from the ApiLoader is saved in individual "DataPeriod" text files in the current directory (e.g. + Administration). + * The database backup files are saved in the backup location specified by the locust-config.json file. Each + database backup file is numbered to match their corresponding Data Period. + +2. Sort-XmlFiles + * This function is used to sort, in place, the xml files created by the SDG into separate folders for each data + period. + * The required argument is: + * **xml** : The full file path to the folder containing the xml files created by the SDG (file path acquired in + Step 1 above). + +3. Run-ApiClientLoader + * This function is used to run the ApiLoader for a particular range of data periods and create backups for each data + period. It **does not** sort the xml files and it assumes that the xml files have been sorted previously. + * The required arguments are: + * **lastDataPeriod** : In the normal case of 7 data periods, this should be set to 7. If the user is wanting to + run the ApiLoader and generate backups for a particular range of data periods, this value should be set to the + max of the range. For instance if the range is data periods 3 through 5, this argument should be set to 5. + * **apiLoader** : The full file path to the folder containing the EdFi.ApiLoader.Console executable (file path + acquired in Step 3 above). + * **working** : The full file path to a writable folder containing the working files for the Ed-Fi ApiLoader + e.g. "C:\temp\API Client Working Folder". + * **xml** : The full file path to the folder containing the xml files created by the SDG (file path acquired in + Step 1 above). + * **xsd** : The full file path to the folder containing the Ed-Fi Xsd Schema files e.g. + "C:\dev\Ed-Fi-Standard\v3.1\Schemas\Bulk" (file path acquired in Step 2 above). + * The optional arguments are: + * **firstDataPeriod** : Its default value is 1. If the user wants to run the ApiLoader for a particular range of + data periods, this argument should be set to the min of the range. For instance if the range is data periods 3 + through 7, this argument should be set to 3. + * **updateConfig** - If set to $TRUE, the locust-config.json file will be updated automatically so that the + ChangeQuery test can be run without needing to update the config file. Its default value is $FALSE. + * The output from the ApiLoader is saved in individual "DataPeriod" text files in the current directory (e.g. + Administration). + * The database backup files are saved in the backup location specified by the locust-config.json file. Each + database backup file is numbered to match their corresponding Data Period. + +4. Create-Backup + * This function is used to create database backups. + * The required argument is: + * **dataPeriodNumber** - This integer value is used to name the database backup file correctly. It + assumes that the value provided is accurate. If a backup file was already created for that data period it will + be overwritten. + * The optional arguments are: + * **sqlBackupPath** - The full file path to the folder where the SQL backup file should be stored. If not set, + it defaults to the value provided in the locust-config.json file. + * **databaseName** - The name of the database that the backup is being generated for e.g. + "EdFi_Ods_Sandbox_minimalSandbox". If not set, it defaults to the database name provided in the ChangeQueryDataSet.json + file. + +5. ApiClientLoader + * This function will run the ApiLoader for only one data period. It *does not* create database backups. This + function also requires that the xml files have been sorted previously as occurs in the Sort-XmlFiles function above. + * The required arguments are: + * **dataPeriodNumber** - This integer value is used to grab the correct xml files to run the ApiLoader. + * **apiLoader** : The full file path to the folder containing the EdFi.ApiLoader.Console executable (file path + acquired in Step 3 above). + * **working** : The full file path to a writable folder containing the working files for the Ed-Fi ApiLoader + e.g. "C:\Temp\API Client Working Folder". + * **xml** : The full file path to the folder containing the xml files created by the SDG (file path acquired in + Step 1 above). + * **xsd** : The full file path to the folder containing the Ed-Fi Xsd Schema files e.g. + "C:\dev\Ed-Fi-Standard\v3.1\Schemas\Bulk" (file path acquired in Step 2 above). + * The output from the ApiLoader is saved in individual "DataPeriod" text files in the current directory (e.g. + Administration). + +## Using the Generated Data Sets + +If in Step 6, you ran GenerateAllSQLBackups or Run-ApiClientLoader and you set the optional argument UpdateConfig to +$TRUE, you can go ahead and run the ChangeQuery test without needing to do anything else. Otherwise, open the +locust-config.json file, update the change_query_backup_filenames array to list the names of the database backup files +that you just created and change the value of restore_database to "true". + + + + \ No newline at end of file diff --git a/docs/how-to-create-tests.md b/docs/how-to-create-tests.md index aec23acb..6cdab4d6 100644 --- a/docs/how-to-create-tests.md +++ b/docs/how-to-create-tests.md @@ -2,12 +2,13 @@ ## Resource Tests -When creating a test for a new resource, you will be creating 4 different classes: +When creating a test for a new resource, you will be creating 5 different classes: 1. Factory 2. Client 3. PipecleanTest 4. VolumeTest (if necessary) +5. ChangeQueryTest (if necessary) ### 1. Factory @@ -75,6 +76,8 @@ Perform the following steps to create a factory (let's call our example resource The client class contains an endpoint for the URL of the resource. This class creates and deletes a resource based on the data from the factory class, which you created earlier. Optionally, a client class will have a list of dependent resource clients that it needs in order to create the resource at hand. If so, then there will also be a custom `create_with_dependencies()` and `delete_with_dependencies()` method. Having a central place to perform these common actions will make writing pipeclean and volume tests easier. +#### Simple Clients (no dependencies) + Perform the following steps to create a simple (zero dependents) client: 1. Create the file for the resource client: @@ -95,6 +98,73 @@ Perform the following steps to create a simple (zero dependents) client: 4. For a simple resource, you would be finished. The EdFiAPIClient class that is inherited creates and deletes the resource behind the scenes. This class contains methods for GET, POST, PUT, and DELETE actions on the API. It also dynamically generates the factory class based on the file name and file path so be sure to stay consistent with the file-creation steps above. Remember to replace 'course' with the name of your new resource. +#### Complex Clients (1+ dependencies) + +Perform the following steps to create a complex (1+ dependents) client: + +1. Create the file for the resource client (CalendarDate will be our example here): + * ~Ed-Fi-Performance\edfi_performance\api\client\calendar_date.py + * Replace 'calendar_date' with the name of your new resource +2. Add the following import statements to the top of the file (Your import for the factory may look different): + + ```python + from edfi_performance.api.client import EdFiAPIClient + ``` + +3. Add the class along with the endpoint value for the specific resource and the dependencies it needs for creation + + ```python + class CalendarDateClient(EdFiAPIClient): + endpoint = 'calendarDates' + + dependencies = { + 'edfi_performance.api.client.calendar.CalendarClient': {}, + } + ``` + + * Notice that for the dependencies, we used a string representation of the client class that we want to depend on. You could also import the desired client class and use that instead of the string representation. There is also an empty curly brace. In that curly brace, you could optionally define the name of the Calendar Client instance, but by default, it will be `calendar_client`. Look at other client classes to see how this could vary. + +4. Next, we need to override the `create_with_dependencies()` method because we need to also create a dependency. Here's what that may look like: + + ```python + from edfi_performance.api.client import EdFiAPIClient + from edfi_performance.api.client.school import SchoolClient + from edfi_performance.factories.utils import RandomSuffixAttribute + + + class CalendarDateClient(EdFiAPIClient): + endpoint = 'calendarDates' + + dependencies = { + 'edfi_performance.api.client.calendar.CalendarClient': {}, + } + + def create_with_dependencies(self, **kwargs): + school_id = kwargs.pop('schoolId', SchoolClient.shared_elementary_school_id()) + custom_calendar_code = kwargs.pop('calendarCode', RandomSuffixAttribute("107SS111111")) + # Create a calendar + calendar_reference = self.calendar_client.create_with_dependencies( + schoolReference__schoolId=school_id, + calendarCode=custom_calendar_code) + + # Create first calendar date + return self.create_using_dependencies( + calendar_reference, + calendarReference__calendarCode=calendar_reference['attributes']['calendarCode'], + calendarReference__schoolId=school_id, + calendarReference__schoolYear=2014, + **kwargs) + ``` + + * Notice a few things: + * From the kwargs, we had to pull out 2 arguments that we passed in ('schoolId' and 'calendarCode'). The second argument in `kwargs.pop()` is the default if those values weren't passed in. Usually, it is volume test scenarios that are making use of kwargs. + * Some of our clients hold on to a shared resource so we don't have to create them every time. `SchoolClient.shared_elementary_school_id()` is the shared resource we use here that holds onto the schoolId for a school. We also have shared resources for student, staff, education organization, and others. Look at the codebase to see how they are used. + * When creating the dependency ('calendar'), we grab the instance of the dependency (`self.calendar_client`) and call its `create_with_dependencies()` method along with some arguments. + * When creating the resource, you may need to use the double-underscore factory boy style to set nested attributes to a value. Make sure to pass in the dependency reference (`calendar_reference`) as the first argument. If you have multiple dependencies, that argument will be a list instead. Look at SectionAttendanceTakenEventClient for an example of that. + * Make sure the order of creation is dependencies and then the resource, otherwise you won't be able to use the `create_using_dependencies()` method. If so, look at StudentClient to see how that would look. + +5. For a complex resource, you would be finished. The EdFiAPIClient class dynamically infers the factory class based on the file name and file path so be sure to stay consistent with the file-creation steps above. Remember to replace 'calendarDate' with the name of your new resource. + ### 3. Pipeclean Test The Pipeclean Test class methodically exercises all 5 endpoints for a given resource: GET, POST, GET (by id), PUT, and DELETE. This class contains the name and new value of the attribute that is going to be updated. Each Pipeclean Test class inherits from EdFiPipecleanTestBase, which is where the sequence of requests are made. Once this sequence of requests has reached the end, the locust client will move on to the next pipeclean test. @@ -118,7 +188,7 @@ Perform the following steps to create a simple pipeclean test: update_attribute_value = "Algebra II" ``` -4. For a simple pipeclean test, you would be finished. As explained earlier, the EdFiPipecleanTestBase performs the 5 API calls and moves on to the next resource test. Also, the client class is dynamically generated here so be sure to stay consistent with the file-creation steps above. Remember to replace 'course' with the name of your new resource. +4. For a simple pipeclean test, you would be finished. As explained earlier, the EdFiPipecleanTestBase performs the 5 API calls and moves on to the next resource test. Also, the client class is dynamically inferred here so be sure to stay consistent with the file-creation steps above. Remember to replace 'course' with the name of your new resource. ### 4. Volume Test @@ -206,7 +276,37 @@ Perform the following steps to create a simple volume test: * We now have two tasks with different weights. This is because we don't want bad requests to occur very often. In this case, there is a 1% chance of this bad request occurring * In the `deliberate_failure_scenario()`, a call to `run_unsuccessful_scenario()` is made. This method takes in succeed_on, a list containing the expected status code(s). It also takes in any keyword arguments that will make this test fail with a 403 response. -8. At this point, you have a complete volume test class. The EdFiVolumeTestBase allows the client class to be dynamically generated so be sure to stay consistent with the file-creation steps above. Remember to replace 'course' with the name of your new resource. +8. At this point, you have a complete volume test class. The EdFiVolumeTestBase allows the client class to be dynamically inferred so be sure to stay consistent with the file-creation steps above. Remember to replace 'course' with the name of your new resource. + + +### 5. Change Query Test + +The Change Query Test class exercises the ability to fetch changed resources by simulating a nightly sync process, repeatedly issuing paged GET requests for changed resources until all the changed resources have been fetched. + +Naturally, only a limited set of resources may be applicable for such a nightly sync, so there are relatively few such tests. + +Perform the following steps to create a change query test: + +1. Create the file for the resource change query test: + * ~Ed-Fi-X-Performance\edfi_performance\tasks\change_query\course.py + * Replace 'course' with the name of your new resource + +2. Add the following import statements to the top of the file: + + ```python + from edfi_performance.tasks.change_query import EdFiChangeQueryTestBase + ``` + +3. Add the change query test class, using `pass` to signify that there is no content in the class. + + ```python + class CourseChangeQueryTest(EdFiChangeQueryTestBase): + endpoint = 'courses' + ``` + + Replace 'Course' and 'courses' with the name of your new resource and its endpoint name. + +4. At this point, you have a complete change query test class. The entire behavior of the test is handled by `EdFiChangeQueryTestBase`. The `EdFiChangeQueryTestBase` allows the client class to be dynamically inferred so be sure to stay consistent with the file-creation steps above. Remember to replace `Course` with the name of your new resource. ## Composite Tests diff --git a/edfi_performance/api/basic_client/__init__.py b/edfi_performance/api/basic_client/__init__.py new file mode 100644 index 00000000..8d72522f --- /dev/null +++ b/edfi_performance/api/basic_client/__init__.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: Apache-2.0 +# Licensed to the Ed-Fi Alliance under one or more agreements. +# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +# See the LICENSE and NOTICES files in the project root for more information. + +import inspect +import logging +import urllib3 +import json +import traceback + +from locust.clients import HttpSession +from edfi_performance.config import get_config_value + +logger = logging.getLogger('locust.runners') +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +class EdFiBasicAPIClient(object): + + _http_client = None + _token = None + + def __init__(self, api_prefix='/data/v3/ed-fi'): + self.API_PREFIX = api_prefix + host = get_config_value('host') + self._http_client = HttpSession(host) + # Suppress exceptions thrown in the Test Lab environment + # when self-signed certificates are used. + self._http_client.verify = False + if self._token is None: + self._token = self.login() + + def login(self, succeed_on=None, name=None, **credentials_overrides): + if succeed_on is None: + succeed_on = [] + name = name or '/oauth/token' + payload = { + "client_id": get_config_value('client_id'), + "client_secret": get_config_value('client_secret'), + "grant_type": "client_credentials", + } + payload.update(credentials_overrides) + response = self._get_response( + 'post', + "/oauth/token", + payload, + succeed_on=succeed_on, + name=name) + self.log_response(response, ignore_error=response.status_code in succeed_on) + try: + token = json.loads(response.text)["access_token"] + return token + except (KeyError, ValueError): + # failed login + return None + + def get_headers(self): + if self._token is None: + raise ValueError("Need to log in before getting authorization headers!") + return { + "Authorization": "Bearer {}".format(self._token), + "Accept": "application/json", + "Content-Type": "application/json", + } + + def _get_response(self, method, *args, **kwargs): + method = getattr(self._http_client, method) + succeed_on = kwargs.pop('succeed_on', []) + with method(*args, catch_response=True, allow_redirects=False, **kwargs) as response: + if response.status_code in succeed_on: + # If told explicitly to succeed, mark success + response.success() + elif 300 <= response.status_code < 400: + # Mark 3xx Redirect responses as failure + response.failure("Status code {} is a failure".format(response.status_code)) + # All other status codes are treated normally + return response + + @staticmethod + def log_response(response, ignore_error=False, log_response_text=False): + if response.status_code >= 400 and not ignore_error: + frame = inspect.currentframe(1) + stack_trace = traceback.format_stack(frame) + logger.error(u''.join(stack_trace)) + + if log_response_text: + logger.debug(response.text) + + @staticmethod + def is_not_expected_result(response, expected_responses): + if response.status_code not in expected_responses: + message = 'Invalid response received' + try: + message = json.loads(response.text)['message'] + except Exception: + pass + print response.request.method + " " + str(response.status_code) + ' : ' + message + return True + return False + + def list_endpoint(self, endpoint, query=""): + return "{}/{}{}".format(self.API_PREFIX, endpoint, query) + + def get_list(self, endpoint, query=""): + response = self._get_response( + 'get', + self.list_endpoint(endpoint, query), + headers=self.get_headers(), + name=self.list_endpoint(endpoint)) + if self.is_not_expected_result(response, [200]): + return + self.log_response(response) + return json.loads(response.text) diff --git a/edfi_performance/api/client/__init__.py b/edfi_performance/api/client/__init__.py index f1101150..06140235 100644 --- a/edfi_performance/api/client/__init__.py +++ b/edfi_performance/api/client/__init__.py @@ -110,18 +110,14 @@ def __init__(self, host, token=None): self.generate_factory_class() @staticmethod - def log_response(caller_name, response, ignore_error=False): - log_message = "[{}] {}: {} {}".format( - caller_name, - response.request.method, - response.status_code, - response.text.replace('\r\n', '')) + def log_response(response, ignore_error=False, log_response_text=False): if response.status_code >= 400 and not ignore_error: frame = inspect.currentframe(1) stack_trace = traceback.format_stack(frame) - logger.error('\n'.join([log_message, ''.join(stack_trace)])) - else: - logger.debug(log_message) + logger.error(u''.join(stack_trace)) + + if log_response_text: + logger.debug(response.text) def list_endpoint(self, query=""): return "{}/{}{}".format(self.API_PREFIX, self.endpoint, query) @@ -161,7 +157,7 @@ def login(self, succeed_on=None, name=None, **credentials_overrides): payload, succeed_on=succeed_on, name=name) - self.log_response('login', response, ignore_error=response.status_code in succeed_on) + self.log_response(response, ignore_error=response.status_code in succeed_on) try: self.token = json.loads(response.text)["access_token"] return self.token @@ -187,7 +183,7 @@ def get_list(self, query=""): name=self.list_endpoint()) if self.is_not_expected_result(response, [200]): return - self.log_response('get_list', response) + self.log_response(response) return json.loads(response.text) def get_item(self, resource_id): @@ -198,7 +194,7 @@ def get_item(self, resource_id): name=self.detail_endpoint_nickname()) if self.is_not_expected_result(response, [200]): return - self.log_response('get_item', response) + self.log_response(response) return json.loads(response.text) def create(self, unique_id_field=None, name=None, **factory_kwargs): @@ -228,7 +224,7 @@ def create(self, unique_id_field=None, name=None, **factory_kwargs): if unique_id is not None: return None, None return None - self.log_response('create', response) + self.log_response(response) resource_id = response.headers['Location'].split('/')[-1].strip() if unique_id is not None: return resource_id, unique_id @@ -245,7 +241,7 @@ def update(self, resource_id, **update_kwargs): name=self.detail_endpoint_nickname()) if self.is_not_expected_result(response, [200, 204]): return - self.log_response('update', response) + self.log_response(response) new_id = response.headers['Location'].split('/')[-1].strip() assert new_id == resource_id return resource_id @@ -265,7 +261,7 @@ def delete(self, resource_id): name=self.detail_endpoint_nickname()) if self.is_not_expected_result(response, [204]): return - self.log_response('delete', response) + self.log_response(response) return response.status_code == 204 def create_with_dependencies(self, **kwargs): @@ -302,6 +298,30 @@ def create_with_dependencies(self, **kwargs): 'attributes': resource_attrs, } + def create_using_dependencies(self, dependency_reference=None, **kwargs): + resource_attrs = self.factory.build_dict(**kwargs) + resource_id = self.create(**resource_attrs) + + if isinstance(dependency_reference, list): + dependencies = {} + for obj in dependency_reference: + key, value = obj.items()[0] + dependencies[key] = value + + return { + 'resource_id': resource_id, + 'dependency_ids': dependencies, + 'attributes': resource_attrs, + } + + return { + 'resource_id': resource_id, + 'dependency_ids': { + 'dependency_reference': dependency_reference, + }, + 'attributes': resource_attrs, + } + def delete_with_dependencies(self, reference, **kwargs): """ Atomically delete an instance of this resource along with all @@ -315,6 +335,20 @@ def delete_with_dependencies(self, reference, **kwargs): :return: `None` """ self.delete(reference['resource_id']) + if len(self.dependencies) == 1: + self._get_dependency_client().delete_with_dependencies(reference['dependency_ids']['dependency_reference']) + elif len(self.dependencies) > 1: + for dependency in reference['dependency_ids']: + getattr(self, dependency).delete_with_dependencies(reference['dependency_ids'][dependency]) + + def _get_dependency_client(self): + subclient_class, client_name = self.dependencies.items()[0] + if isinstance(subclient_class, basestring): + subclient_class = _import_from_dotted_path(subclient_class) + subclient_name = client_name.get('client_name') + if subclient_name is None: + subclient_name = _title_case_to_snake_case(subclient_class.__name__) + return getattr(self, subclient_name) @classmethod def create_shared_resource(cls, value, **kwargs): diff --git a/edfi_performance/api/client/account.py b/edfi_performance/api/client/account.py index 184d660a..b45e5e01 100644 --- a/edfi_performance/api/client/account.py +++ b/edfi_performance/api/client/account.py @@ -22,7 +22,8 @@ def create_with_dependencies(self, **kwargs): account_code_attrs = account_code_reference['attributes'] edorg_id = account_code_attrs['educationOrganizationReference']['educationOrganizationId'] - account_attrs = self.factory.build_dict( + return self.create_using_dependencies( + account_code_reference, accountCodes=[{ 'accountCodeReference': dict( accountCodeNumber=account_code_attrs['accountCodeNumber'], @@ -32,21 +33,7 @@ def create_with_dependencies(self, **kwargs): ), }], educationOrganizationReference__educationOrganizationId=edorg_id, - fiscalYear=account_code_attrs['fiscalYear'], - ) - account_id = self.create(**account_attrs) - - return { - 'resource_id': account_id, - 'dependency_ids': { - 'account_code_id': account_code_reference['resource_id'], - }, - 'attributes': account_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.account_code_client.delete(reference['dependency_ids']['account_code_id']) + fiscalYear=account_code_attrs['fiscalYear']) class _AccountDependentMixin(object): @@ -61,23 +48,10 @@ def create_with_dependencies(self, **kwargs): account_reference = self.account_client.create_with_dependencies() account_identifier = account_reference['attributes']['accountIdentifier'] - resource_attrs = self.factory.build_dict( + return self.create_using_dependencies( + account_reference, accountReference__accountIdentifier=account_identifier, - **kwargs - ) - resource_id = self.create(**resource_attrs) - - return { - 'resource_id': resource_id, - 'dependency_ids': { - 'account_reference': account_reference, - }, - 'attributes': resource_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.account_client.delete_with_dependencies(reference['dependency_ids']['account_reference']) + **kwargs) class ActualClient(_AccountDependentMixin, EdFiAPIClient): diff --git a/edfi_performance/api/client/assessment.py b/edfi_performance/api/client/assessment.py index 4f4500f7..42a334d6 100644 --- a/edfi_performance/api/client/assessment.py +++ b/edfi_performance/api/client/assessment.py @@ -22,23 +22,11 @@ def create_with_dependencies(self, **kwargs): assessment_reference = self.assessment_client.create_with_dependencies() # Create assessment item - item_attrs = self.factory.build_dict( - assessmentReference__assessmentTitle=assessment_reference['attributes']['assessmentTitle'], + return self.create_using_dependencies( + assessment_reference, + assessmentReference__assessmentIdentifier=assessment_reference['attributes']['assessmentIdentifier'], **kwargs ) - item_id = self.create(**item_attrs) - - return { - 'resource_id': item_id, - 'dependency_ids': { - 'assessment_reference': assessment_reference, - }, - 'attributes': item_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.assessment_client.delete_with_dependencies(reference['dependency_ids']['assessment_reference']) class LearningObjectiveClient(EdFiAPIClient): @@ -61,23 +49,11 @@ def create_with_dependencies(self, **kwargs): assessment_reference = self.assessment_client.create_with_dependencies() # Create objective assessment - objective_attrs = self.factory.build_dict( - assessmentReference__assessmentTitle=assessment_reference['attributes']['assessmentTitle'], + return self.create_using_dependencies( + assessment_reference, + assessmentReference__assessmentIdentifier=assessment_reference['attributes']['assessmentIdentifier'], **kwargs ) - objective_id = self.create(**objective_attrs) - - return { - 'resource_id': objective_id, - 'dependency_ids': { - 'assessment_reference': assessment_reference, - }, - 'attributes': objective_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.assessment_client.delete_with_dependencies(reference['dependency_ids']['assessment_reference']) class StudentAssessmentClient(EdFiAPIClient): @@ -92,20 +68,8 @@ def create_with_dependencies(self, **kwargs): assessment_reference = self.assessment_client.create_with_dependencies() # Create student assessment - student_assessment_attrs = self.factory.build_dict( - assessmentReference__assessmentTitle=assessment_reference['attributes']['assessmentTitle'], + return self.create_using_dependencies( + assessment_reference, + assessmentReference__assessmentIdentifier=assessment_reference['attributes']['assessmentIdentifier'], **kwargs ) - student_assessment_id = self.create(**student_assessment_attrs) - - return { - 'resource_id': student_assessment_id, - 'dependency_ids': { - 'assessment_reference': assessment_reference, - }, - 'attributes': student_assessment_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.assessment_client.delete_with_dependencies(reference['dependency_ids']['assessment_reference']) diff --git a/edfi_performance/api/client/bell_schedule.py b/edfi_performance/api/client/bell_schedule.py index 1f9f7216..40caf24c 100644 --- a/edfi_performance/api/client/bell_schedule.py +++ b/edfi_performance/api/client/bell_schedule.py @@ -19,20 +19,8 @@ def create_with_dependencies(self, **kwargs): class_period_reference = self.class_period_client.create_with_dependencies() # Create bell schedule - schedule_attrs = self.factory.build_dict( + return self.create_using_dependencies( + class_period_reference, classPeriods__0__classPeriodReference__classPeriodName=class_period_reference['attributes']['classPeriodName'], **kwargs ) - schedule_id = self.create(**schedule_attrs) - - return { - 'resource_id': schedule_id, - 'dependency_ids': { - 'class_period_reference': class_period_reference, - }, - 'attributes': schedule_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.class_period_client.delete_with_dependencies(reference['dependency_ids']['class_period_reference']) diff --git a/edfi_performance/api/client/calendar_date.py b/edfi_performance/api/client/calendar_date.py index 19302672..69d65b2f 100644 --- a/edfi_performance/api/client/calendar_date.py +++ b/edfi_performance/api/client/calendar_date.py @@ -24,22 +24,9 @@ def create_with_dependencies(self, **kwargs): calendarCode=custom_calendar_code) # Create first calendar date - calendar_date_attrs = self.factory.build_dict( + return self.create_using_dependencies( + calendar_reference, calendarReference__calendarCode=calendar_reference['attributes']['calendarCode'], calendarReference__schoolId=school_id, calendarReference__schoolYear=2014, - **kwargs - ) - calendar_date_id = self.create(**calendar_date_attrs) - - return { - 'resource_id': calendar_date_id, - 'dependency_ids': { - 'calendar_reference': calendar_reference, - }, - 'attributes': calendar_date_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.calendar_client.delete_with_dependencies(reference['dependency_ids']['calendar_reference']) + **kwargs) diff --git a/edfi_performance/api/client/community.py b/edfi_performance/api/client/community.py index 427854ce..a63727b8 100644 --- a/edfi_performance/api/client/community.py +++ b/edfi_performance/api/client/community.py @@ -22,23 +22,11 @@ class CommunityProviderClient(EdFiAPIClient): def create_with_dependencies(self, **kwargs): org_reference = self.org_client.create_with_dependencies() - provider_attrs = self.factory.build_dict( + return self.create_using_dependencies( + org_reference, communityOrganizationReference__communityOrganizationId=org_reference['attributes']['communityOrganizationId'], **kwargs ) - provider_id = self.create(**provider_attrs) - - return { - 'resource_id': provider_id, - 'dependency_ids': { - 'org_reference': org_reference, - }, - 'attributes': provider_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.org_client.delete_with_dependencies(reference['dependency_ids']['org_reference']) class CommunityProviderLicenseClient(EdFiAPIClient): @@ -53,20 +41,8 @@ class CommunityProviderLicenseClient(EdFiAPIClient): def create_with_dependencies(self, **kwargs): provider_reference = self.provider_client.create_with_dependencies() - license_attrs = self.factory.build_dict( + return self.create_using_dependencies( + provider_reference, communityProviderReference__communityProviderId=provider_reference['attributes']['communityProviderId'], **kwargs ) - license_id = self.create(**license_attrs) - - return { - 'resource_id': license_id, - 'dependency_ids': { - 'provider_reference': provider_reference, - }, - 'attributes': license_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.provider_client.delete_with_dependencies(reference['dependency_ids']['provider_reference']) diff --git a/edfi_performance/api/client/composite.py b/edfi_performance/api/client/composite.py index f90498ed..6a1c9b07 100644 --- a/edfi_performance/api/client/composite.py +++ b/edfi_performance/api/client/composite.py @@ -21,7 +21,7 @@ def get_composite_list(self, resource_name, resource_id): name=self._composite_list_endpoint(resource_name, '{id}')) if self.is_not_expected_result(response, [200]): return - self.log_response('get_composite_list', response) + self.log_response(response) return json.loads(response.text) @classmethod @@ -42,5 +42,5 @@ def _get_all(self, resource): list_endpoint, headers=self.get_headers(), name=list_endpoint) - self.log_response('_get_all', response) + self.log_response(response) return json.loads(response.text) diff --git a/edfi_performance/api/client/course.py b/edfi_performance/api/client/course.py index 458b0b61..36442bc1 100644 --- a/edfi_performance/api/client/course.py +++ b/edfi_performance/api/client/course.py @@ -26,23 +26,11 @@ def create_with_dependencies(self, **kwargs): record_reference = self.record_client.create_with_dependencies(schoolId=school_id) # Create course transcript - transcript_attrs = self.factory.build_dict( + return self.create_using_dependencies( + record_reference, courseReference__educationOrganizationId=school_id, studentAcademicRecordReference__educationOrganizationId=school_id, studentAcademicRecordReference__studentUniqueId=record_reference['attributes']['studentReference'] ['studentUniqueId'], **kwargs ) - transcript_id = self.create(**transcript_attrs) - - return { - 'resource_id': transcript_id, - 'attributes': transcript_attrs, - 'dependency_ids': { - 'record_reference': record_reference - } - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.record_client.delete_with_dependencies(reference['dependency_ids']['record_reference']) diff --git a/edfi_performance/api/client/course_offering.py b/edfi_performance/api/client/course_offering.py index 4ea3e2dd..15a43f4f 100644 --- a/edfi_performance/api/client/course_offering.py +++ b/edfi_performance/api/client/course_offering.py @@ -19,22 +19,10 @@ def create_with_dependencies(self, **kwargs): session_reference = self.session_client.create_with_dependencies(schoolId=school_id) - course_offering_attrs = self.factory.build_dict( + return self.create_using_dependencies( + session_reference, sessionReference__schoolId=school_id, sessionReference__schoolYear=2014, sessionReference__sessionName=session_reference['attributes']['sessionName'], **kwargs ) - course_offering_id = self.create(**course_offering_attrs) - - return { - 'resource_id': course_offering_id, - 'dependency_ids': { - 'session_reference': session_reference, - }, - 'attributes': course_offering_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.session_client.delete_with_dependencies(reference['dependency_ids']['session_reference']) diff --git a/edfi_performance/api/client/discipline.py b/edfi_performance/api/client/discipline.py index 7050c903..84e32e81 100644 --- a/edfi_performance/api/client/discipline.py +++ b/edfi_performance/api/client/discipline.py @@ -24,24 +24,12 @@ def create_with_dependencies(self, **kwargs): staff_reference = self.staff_client.create_with_dependencies(schoolId=school_id) # Create discipline incident - incident_attrs = self.factory.build_dict( + return self.create_using_dependencies( + staff_reference, staffReference__staffUniqueId=staff_reference['attributes']['staffUniqueId'], schoolReference__schoolId=school_id, **kwargs ) - incident_id = self.create(**incident_attrs) - - return { - 'resource_id': incident_id, - 'dependency_ids': { - 'staff_reference': staff_reference - }, - 'attributes': incident_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.staff_client.delete_with_dependencies(reference['dependency_ids']['staff_reference']) class DisciplineActionClient(EdFiAPIClient): @@ -60,7 +48,8 @@ def create_with_dependencies(self, **kwargs): assoc_reference = self.assoc_client.create_with_dependencies(schoolId=school_id) # Create discipline action - action_attrs = self.factory.build_dict( + return self.create_using_dependencies( + assoc_reference, studentReference__studentUniqueId=assoc_reference['attributes']['studentReference']['studentUniqueId'], studentDisciplineIncidentAssociations__0__studentDisciplineIncidentAssociationReference__incidentIdentifier =assoc_reference['attributes']['disciplineIncidentReference']['incidentIdentifier'], @@ -69,16 +58,3 @@ def create_with_dependencies(self, **kwargs): studentDisciplineIncidentAssociations__0__studentDisciplineIncidentAssociationReference__schoolId=school_id, **kwargs ) - action_id = self.create(**action_attrs) - - return { - 'resource_id': action_id, - 'dependency_ids': { - 'assoc_reference': assoc_reference - }, - 'attributes': action_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.assoc_client.delete_with_dependencies(reference['dependency_ids']['assoc_reference']) diff --git a/edfi_performance/api/client/education.py b/edfi_performance/api/client/education.py index b41f0d58..764ae329 100644 --- a/edfi_performance/api/client/education.py +++ b/edfi_performance/api/client/education.py @@ -4,6 +4,7 @@ # See the LICENSE and NOTICES files in the project root for more information. from edfi_performance.api.client import EdFiAPIClient, get_config_value +from edfi_performance.api.client.school import SchoolClient class EducationContentClient(EdFiAPIClient): @@ -22,23 +23,11 @@ class EducationOrganizationInterventionPrescriptionAssociationClient(EdFiAPIClie def create_with_dependencies(self, **kwargs): rx_reference = self.prescription_client.create_with_dependencies() - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + rx_reference, interventionPrescriptionReference__interventionPrescriptionIdentificationCode= rx_reference['attributes']['interventionPrescriptionIdentificationCode'], ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'rx_reference': rx_reference, - }, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.prescription_client.delete_with_dependencies(reference['dependency_ids']['rx_reference']) class EducationOrganizationNetworkClient(EdFiAPIClient): @@ -57,24 +46,12 @@ class EducationOrganizationNetworkAssociationClient(EdFiAPIClient): def create_with_dependencies(self, **kwargs): network_reference = self.network_client.create_with_dependencies() - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + network_reference, educationOrganizationNetworkReference__educationOrganizationNetworkId= network_reference['attributes']['educationOrganizationNetworkId'], **kwargs ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'network_reference': network_reference, - }, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.network_client.delete_with_dependencies(reference['dependency_ids']['network_reference']) class EducationOrganizationPeerAssociationClient(EdFiAPIClient): @@ -85,29 +62,14 @@ class EducationOrganizationPeerAssociationClient(EdFiAPIClient): } def create_with_dependencies(self, **kwargs): - school_1_reference = self.school_client.create_with_dependencies() - school_2_reference = self.school_client.create_with_dependencies() + school_reference = self.school_client.create_with_dependencies() - assoc_attrs = self.factory.build_dict( - peerEducationOrganizationReference__educationOrganizationId=school_1_reference['attributes']['schoolId'], - educationOrganizationReference__educationOrganizationId=school_2_reference['attributes']['schoolId'], + return self.create_using_dependencies( + school_reference, + peerEducationOrganizationReference__educationOrganizationId=school_reference['attributes']['schoolId'], + educationOrganizationReference__educationOrganizationId=SchoolClient.shared_high_school_id(), **kwargs ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'school_1_reference': school_1_reference, - 'school_2_reference': school_2_reference, - }, - 'attributes': assoc_attrs - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.school_client.delete_with_dependencies(reference['dependency_ids']['school_1_reference']) - self.school_client.delete_with_dependencies(reference['dependency_ids']['school_2_reference']) class EducationServiceCenterClient(EdFiAPIClient): @@ -128,23 +90,11 @@ class LocalEducationAgencyClient(EdFiAPIClient): def create_with_dependencies(self, **kwargs): service_center_reference = self.service_center_client.create_with_dependencies() - agency_attrs = self.factory.build_dict( + return self.create_using_dependencies( + service_center_reference, educationServiceCenterReference__educationServiceCenterId= service_center_reference['attributes']['educationServiceCenterId'], ) - agency_id = self.create(**agency_attrs) - - return { - 'resource_id': agency_id, - 'dependency_ids': { - 'service_center_reference': service_center_reference, - }, - 'attributes': agency_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.service_center_client.delete_with_dependencies(reference['dependency_ids']['service_center_reference']) @classmethod def shared_education_organization_id(cls): @@ -164,27 +114,12 @@ class FeederSchoolAssociationClient(EdFiAPIClient): def create_with_dependencies(self, **kwargs): feeder_school_reference = self.school_client.create_with_dependencies() - school_reference = self.school_client.create_with_dependencies() - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + feeder_school_reference, feederSchoolReference__schoolId=feeder_school_reference['attributes']['schoolId'], - schoolReference__schoolId=school_reference['attributes']['schoolId'], + schoolReference__schoolId=SchoolClient.shared_elementary_school_id(), ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'feeder_school_reference': feeder_school_reference, - 'school_reference': school_reference - }, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.school_client.delete_with_dependencies(reference['dependency_ids']['feeder_school_reference']) - self.school_client.delete_with_dependencies(reference['dependency_ids']['school_reference']) class StateEducationAgencyClient(EdFiAPIClient): diff --git a/edfi_performance/api/client/grade.py b/edfi_performance/api/client/grade.py index 7d691315..ca27a4f4 100644 --- a/edfi_performance/api/client/grade.py +++ b/edfi_performance/api/client/grade.py @@ -23,10 +23,11 @@ def create_with_dependencies(self, **kwargs): assoc_reference = self.section_assoc_client.create_with_dependencies(schoolId=school_id, courseCode=course_code) grade_period = \ - assoc_reference['dependency_ids']['section_reference']['gradingPeriods'][0]['gradingPeriodReference'] + assoc_reference['dependency_ids']['section_client']['gradingPeriods'][0]['gradingPeriodReference'] section_reference = assoc_reference['attributes']['sectionReference'] - grade_attrs = self.factory.build_dict( + return self.create_using_dependencies( + assoc_reference, gradingPeriodReference__schoolId=school_id, gradingPeriodReference__periodSequence=grade_period['periodSequence'], gradingPeriodReference__gradingPeriodDescriptor=grade_period['gradingPeriodDescriptor'], @@ -41,15 +42,3 @@ def create_with_dependencies(self, **kwargs): studentSectionAssociationReference__sessionName=section_reference['sessionName'], **kwargs ) - grade_id = self.create(**grade_attrs) - return { - 'resource_id': grade_id, - 'dependency_ids': { - 'assoc_reference': assoc_reference - }, - 'attributes': grade_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.section_assoc_client.delete_with_dependencies(reference['dependency_ids']['assoc_reference']) diff --git a/edfi_performance/api/client/gradebook_entries.py b/edfi_performance/api/client/gradebook_entries.py index d345ced4..5cc765eb 100644 --- a/edfi_performance/api/client/gradebook_entries.py +++ b/edfi_performance/api/client/gradebook_entries.py @@ -18,24 +18,11 @@ def create_with_dependencies(self, **kwargs): section_reference = self.section_client.create_with_dependencies() section_attrs = section_reference['attributes'] - entry_attrs = self.factory.build_dict( + return self.create_using_dependencies( + section_reference, sectionReference__sectionIdentifier=section_attrs['sectionIdentifier'], sectionReference__localCourseCode=section_attrs['courseOfferingReference']['localCourseCode'], sectionReference__schoolId=section_attrs['courseOfferingReference']['schoolId'], sectionReference__schoolYear=section_attrs['courseOfferingReference']['schoolYear'], sectionReference__sessionName=section_attrs['courseOfferingReference']['sessionName'], ) - entry_id = self.create(**entry_attrs) - - return { - 'resource_id': entry_id, - 'dependency_ids': { - 'section_reference': section_reference, - }, - 'attributes': entry_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.section_client.delete_with_dependencies(reference['dependency_ids']['section_reference']) - diff --git a/edfi_performance/api/client/graduation_plan.py b/edfi_performance/api/client/graduation_plan.py index c451ebb8..dd89ccfc 100644 --- a/edfi_performance/api/client/graduation_plan.py +++ b/edfi_performance/api/client/graduation_plan.py @@ -18,20 +18,8 @@ class GraduationPlanClient(EdFiAPIClient): def create_with_dependencies(self, **kwargs): school_reference = self.school_client.create_with_dependencies() - graduation_plan_attrs = self.factory.build_dict( + return self.create_using_dependencies( + school_reference, educationOrganizationReference__educationOrganizationId=school_reference['attributes']['schoolId'], **kwargs ) - graduation_plan_id = self.create(**graduation_plan_attrs) - - return { - 'resource_id': graduation_plan_id, - 'dependency_ids': { - 'school_reference': school_reference, - }, - 'attributes': graduation_plan_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.school_client.delete_with_dependencies(reference['dependency_ids']['school_reference']) diff --git a/edfi_performance/api/client/intervention.py b/edfi_performance/api/client/intervention.py index a8c70e2f..59470edd 100644 --- a/edfi_performance/api/client/intervention.py +++ b/edfi_performance/api/client/intervention.py @@ -24,23 +24,11 @@ def create_with_dependencies(self, **kwargs): rx_reference = self.prescription_client.create_with_dependencies() # Create intervention - intervention_attrs = self.factory.build_dict( + return self.create_using_dependencies( + rx_reference, interventionPrescriptions__0__interventionPrescriptionReference__interventionPrescriptionIdentificationCode= rx_reference['attributes']['interventionPrescriptionIdentificationCode'], ) - intervention_id = self.create(**intervention_attrs) - - return { - 'resource_id': intervention_id, - 'dependency_ids': { - 'rx_reference': rx_reference, - }, - 'attributes': intervention_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.prescription_client.delete_with_dependencies(reference['dependency_ids']['rx_reference']) class InterventionStudyClient(EdFiAPIClient): @@ -57,20 +45,8 @@ def create_with_dependencies(self, **kwargs): rx_reference = self.prescription_client.create_with_dependencies() # Create intervention - study_attrs = self.factory.build_dict( + return self.create_using_dependencies( + rx_reference, interventionPrescriptionReference__interventionPrescriptionIdentificationCode= rx_reference['attributes']['interventionPrescriptionIdentificationCode'], ) - study_id = self.create(**study_attrs) - - return { - 'resource_id': study_id, - 'dependency_ids': { - 'rx_reference': rx_reference, - }, - 'attributes': study_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.prescription_client.delete_with_dependencies(reference['dependency_ids']['rx_reference']) diff --git a/edfi_performance/api/client/post_secondary.py b/edfi_performance/api/client/post_secondary.py index 426a58c7..3f025f9d 100644 --- a/edfi_performance/api/client/post_secondary.py +++ b/edfi_performance/api/client/post_secondary.py @@ -23,21 +23,9 @@ def create_with_dependencies(self, **kwargs): # Create new student for association institution_reference = self.institution_client.create_with_dependencies() - event_attrs = self.factory.build_dict( + return self.create_using_dependencies( + institution_reference, postSecondaryInstitutionReference__postSecondaryInstitutionId= institution_reference['attributes']['postSecondaryInstitutionId'], **kwargs ) - event_id = self.create(**event_attrs) - - return { - 'resource_id': event_id, - 'dependency_ids': { - 'institution_reference': institution_reference, - }, - 'attributes': event_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.institution_client.delete_with_dependencies(reference['dependency_ids']['institution_reference']) diff --git a/edfi_performance/api/client/report_card.py b/edfi_performance/api/client/report_card.py index 809f0d33..bba9c913 100644 --- a/edfi_performance/api/client/report_card.py +++ b/edfi_performance/api/client/report_card.py @@ -17,19 +17,7 @@ class ReportCardClient(EdFiAPIClient): def create_with_dependencies(self, **kwargs): period_reference = self.grading_period_client.create_with_dependencies() - card_attrs = self.factory.build_dict( + return self.create_using_dependencies( + period_reference, gradingPeriodReference__periodSequence=period_reference['attributes']['periodSequence'], ) - card_id = self.create(**card_attrs) - - return { - 'resource_id': card_id, - 'dependency_ids': { - 'period_reference': period_reference, - }, - 'attributes': card_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.grading_period_client.delete_with_dependencies(reference['dependency_ids']['period_reference']) diff --git a/edfi_performance/api/client/section.py b/edfi_performance/api/client/section.py index c5cba595..6efd8994 100644 --- a/edfi_performance/api/client/section.py +++ b/edfi_performance/api/client/section.py @@ -66,7 +66,7 @@ def create_with_dependencies(self, **kwargs): 'sectionIdentifier': section_attrs['sectionIdentifier'], 'sessionName': course_offering_attrs['sessionReference']['sessionName'], 'gradingPeriods': - course_offering_reference['dependency_ids']['session_reference']['attributes']['gradingPeriods'], + course_offering_reference['dependency_ids']['dependency_reference']['attributes']['gradingPeriods'], } def delete_with_dependencies(self, reference, **kwargs): @@ -97,7 +97,8 @@ def create_with_dependencies(self, **kwargs): ) section_attrs = section_reference['attributes'] - event_attrs = self.factory.build_dict( + return self.create_using_dependencies( + [{'calendar_date_client': calendar_date_reference}, {'section_client': section_reference}], calendarDateReference__schoolId=school_id, calendarDateReference__calendarCode=calendar_date_attrs['calendarReference']['calendarCode'], sectionReference__sectionIdentifier=section_attrs['sectionIdentifier'], @@ -106,18 +107,3 @@ def create_with_dependencies(self, **kwargs): sectionReference__schoolYear=section_attrs['courseOfferingReference']['schoolYear'], sectionReference__sessionName=section_attrs['courseOfferingReference']['sessionName'], ) - event_id = self.create(**event_attrs) - - return { - 'resource_id': event_id, - 'dependency_ids': { - 'calendar_date_reference': calendar_date_reference, - 'section_reference': section_reference, - }, - 'attributes': event_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.calendar_date_client.delete_with_dependencies(reference['dependency_ids']['calendar_date_reference']) - self.section_client.delete_with_dependencies(reference['dependency_ids']['section_reference']) diff --git a/edfi_performance/api/client/session.py b/edfi_performance/api/client/session.py index 212b1ee1..6ff572eb 100644 --- a/edfi_performance/api/client/session.py +++ b/edfi_performance/api/client/session.py @@ -32,21 +32,12 @@ def create_with_dependencies(self, **kwargs): ) # Create session referencing grading periods - session_attrs = self.factory.build_dict( + return self.create_using_dependencies( + [{'period_1_reference': period_1_reference}, {'period_2_reference': period_2_reference}], schoolReference__schoolId=school_id, gradingPeriods__0__gradingPeriodReference__periodSequence=period_1_reference['attributes']['periodSequence'], gradingPeriods__1__gradingPeriodReference__periodSequence=period_2_reference['attributes']['periodSequence'], **kwargs) - session_id = self.create(**session_attrs) - - return { - 'resource_id': session_id, - 'dependency_ids': { - 'period_1_reference': period_1_reference, - 'period_2_reference': period_2_reference, - }, - 'attributes': session_attrs, - } def delete_with_dependencies(self, reference, **kwargs): self.delete(reference['resource_id']) diff --git a/edfi_performance/api/client/staff.py b/edfi_performance/api/client/staff.py index ea605731..5803b12a 100644 --- a/edfi_performance/api/client/staff.py +++ b/edfi_performance/api/client/staff.py @@ -73,16 +73,7 @@ def create_with_dependencies(self, **kwargs): educationOrganizationReference__educationOrganizationId=edorg_id, ) assoc_overrides.update(kwargs) - assoc_attrs = self.factory.build_dict(**assoc_overrides) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) + return self.create_using_dependencies(**assoc_overrides) class StaffAbsenceEventClient(EdFiAPIClient): @@ -109,27 +100,13 @@ def create_with_dependencies(self, **kwargs): staff_reference = self.staff_client.create_with_dependencies(schoolId=school_id) # Create association between new staff and new cohort - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + [{'cohort_client': cohort_reference}, {'staff_client': staff_reference}], staffReference__staffUniqueId=staff_reference['attributes']['staffUniqueId'], cohortReference__cohortIdentifier=cohort_reference['attributes']['cohortIdentifier'], cohortReference__educationOrganizationId=school_id, **kwargs ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'cohort_reference': cohort_reference, - 'staff_reference': staff_reference - }, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.cohort_client.delete_with_dependencies(reference['dependency_ids']['cohort_reference']) - self.staff_client.delete_with_dependencies(reference['dependency_ids']['staff_reference']) class StaffEducationOrganizationContactAssociationClient(EdFiAPIClient): @@ -161,24 +138,12 @@ def create_with_dependencies(self, **kwargs): # Create staff record staff_reference = self.staff_client.create_with_dependencies(schoolId=school_id) - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + staff_reference, staffReference__staffUniqueId=staff_reference['attributes']['staffUniqueId'], schoolReference__schoolId=school_id, **kwargs ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'staff_reference': staff_reference, - }, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.staff_client.delete_with_dependencies(reference['dependency_ids']['staff_reference']) class StaffSectionAssociationClient(EdFiAPIClient): @@ -202,7 +167,8 @@ def create_with_dependencies(self, **kwargs): # Create association between staff and section sec_attrs = section_reference['attributes'] - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + [{'section_client': section_reference}, {'staff_client': staff_reference}], staffReference__staffUniqueId=staff_reference['attributes']['staffUniqueId'], sectionReference__sectionIdentifier=sec_attrs['sectionIdentifier'], sectionReference__localCourseCode=sec_attrs['courseOfferingReference']['localCourseCode'], @@ -210,18 +176,3 @@ def create_with_dependencies(self, **kwargs): sectionReference__schoolYear=sec_attrs['courseOfferingReference']['schoolYear'], sectionReference__sessionName=sec_attrs['courseOfferingReference']['sessionName'], ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'section_reference': section_reference, - 'staff_reference': staff_reference - }, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.staff_client.delete_with_dependencies(reference['dependency_ids']['staff_reference']) - self.section_client.delete_with_dependencies(reference['dependency_ids']['section_reference']) diff --git a/edfi_performance/api/client/student.py b/edfi_performance/api/client/student.py index a98bfdf2..53665822 100644 --- a/edfi_performance/api/client/student.py +++ b/edfi_performance/api/client/student.py @@ -26,24 +26,12 @@ def create_with_dependencies(self, **kwargs): parent_unique_id = kwargs.pop('parentUniqueId', ParentClient.shared_parent_id()) # Create parent - student association - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + student_reference, studentReference__studentUniqueId=student_reference['attributes']['studentUniqueId'], parentReference__parentUniqueId=parent_unique_id, **kwargs ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'student_reference': student_reference, - }, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.student_client.delete_with_dependencies(reference['dependency_ids']['student_reference']) class StudentClient(EdFiAPIClient): @@ -103,19 +91,10 @@ def create_with_dependencies(self, **kwargs): school_id = kwargs.pop('schoolId', SchoolClient.shared_elementary_school_id()) assoc_overrides = dict( studentReference__studentUniqueId=student_unique_id, - schoolReference__schoolId=school_id, + schoolReference__schoolId=school_id ) assoc_overrides.update(kwargs) - assoc_attrs = self.factory.build_dict(**assoc_overrides) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) + return self.create_using_dependencies(**assoc_overrides) class StudentEducationOrganizationAssociationClient(EdFiAPIClient): @@ -132,23 +111,11 @@ def create_with_dependencies(self, **kwargs): student_reference = self.student_client.create_with_dependencies(schoolId=school_id) # Create ed org - student association - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + student_reference, studentReference__studentUniqueId=student_reference['attributes']['studentUniqueId'], **kwargs ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'student_reference': student_reference - }, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.student_client.delete_with_dependencies(reference['dependency_ids']['student_reference']) class StudentCohortAssociationClient(EdFiAPIClient): @@ -171,27 +138,13 @@ def create_with_dependencies(self, **kwargs): ) # Create the cohort - student association - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + [{'cohort_client': cohort_reference}, {'student_client': student_reference}], studentReference__studentUniqueId=student_reference['attributes']['studentUniqueId'], cohortReference__cohortIdentifier=cohort_reference['attributes']['cohortIdentifier'], cohortReference__educationOrganizationId=school_id, **kwargs ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'student_reference': student_reference, - 'cohort_reference': cohort_reference - }, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.cohort_client.delete_with_dependencies(reference['dependency_ids']['cohort_reference']) - self.student_client.delete_with_dependencies(reference['dependency_ids']['student_reference']) class StudentTitleIPartAProgramAssociationClient(EdFiAPIClient): @@ -208,23 +161,11 @@ def create_with_dependencies(self, **kwargs): student_reference = self.student_client.create_with_dependencies(schoolId=school_id) # Create student program association - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + student_reference, studentReference__studentUniqueId=student_reference['attributes']['studentUniqueId'], **kwargs ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'student_reference': student_reference - }, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.student_client.delete_with_dependencies(reference['dependency_ids']['student_reference']) class StudentSpecialEducationProgramAssociationClient(EdFiAPIClient): @@ -241,23 +182,11 @@ def create_with_dependencies(self, **kwargs): student_reference = self.student_client.create_with_dependencies(schoolId=school_id) # Create student program association - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + student_reference, studentReference__studentUniqueId=student_reference['attributes']['studentUniqueId'], **kwargs ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'student_reference': student_reference - }, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.student_client.delete_with_dependencies(reference['dependency_ids']['student_reference']) class StudentProgramAssociationClient(EdFiAPIClient): @@ -274,23 +203,11 @@ def create_with_dependencies(self, **kwargs): student_reference = self.student_client.create_with_dependencies(schoolId=school_id) # Create student program association - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + student_reference, studentReference__studentUniqueId=student_reference['attributes']['studentUniqueId'], **kwargs ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'student_reference': student_reference - }, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.student_client.delete_with_dependencies(reference['dependency_ids']['student_reference']) class StudentDisciplineIncidentAssociationClient(EdFiAPIClient): @@ -313,27 +230,13 @@ def create_with_dependencies(self, **kwargs): incident_reference = self.incident_client.create_with_dependencies(schoolId=school_id) # Create student incident association - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + [{'incident_client': incident_reference}, {'student_client': student_reference}], studentReference__studentUniqueId=student_reference['attributes']['studentUniqueId'], disciplineIncidentReference__schoolId=school_id, disciplineIncidentReference__incidentIdentifier=incident_reference['attributes']['incidentIdentifier'], **kwargs ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'student_reference': student_reference, - 'incident_reference': incident_reference - }, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.incident_client.delete_with_dependencies(reference['dependency_ids']['incident_reference']) - self.student_client.delete_with_dependencies(reference['dependency_ids']['student_reference']) class StudentSectionAssociationClient(EdFiAPIClient): @@ -358,30 +261,16 @@ def create_with_dependencies(self, **kwargs): # Create association between newly created section and student section_attrs = section_reference['attributes'] - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + [{'section_client': section_reference}, {'student_client': student_reference}], studentReference__studentUniqueId=student_unique_id, sectionReference__sectionIdentifier=section_attrs['sectionIdentifier'], sectionReference__localCourseCode=section_attrs['courseOfferingReference']['localCourseCode'], sectionReference__schoolId=section_attrs['courseOfferingReference']['schoolId'], sectionReference__schoolYear=section_attrs['courseOfferingReference']['schoolYear'], sectionReference__sessionName=section_attrs['courseOfferingReference']['sessionName'], + **kwargs ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'attributes': assoc_attrs, - 'dependency_ids': { - 'student_reference': student_reference, - 'section_reference': section_reference, - } - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - dependencies = reference['dependency_ids'] - self.student_client.delete_with_dependencies(dependencies['student_reference']) - self.section_client.delete_with_dependencies(dependencies['section_reference']) class StudentSchoolAttendanceEventClient(EdFiAPIClient): @@ -402,28 +291,14 @@ def create_with_dependencies(self, **kwargs): session_reference = self.session_client.create_with_dependencies(schoolId=school_id) # Create student school attendance event - event_attrs = self.factory.build_dict( + return self.create_using_dependencies( + [{'session_client': session_reference}, {'student_client': student_reference}], studentReference__studentUniqueId=student_reference['attributes']['studentUniqueId'], sessionReference__sessionName=session_reference['attributes']['sessionName'], schoolReference__schoolId=school_id, sessionReference__schoolId=school_id, **kwargs ) - event_id = self.create(**event_attrs) - - return { - 'resource_id': event_id, - 'dependency_ids': { - 'student_reference': student_reference, - 'session_reference': session_reference - }, - 'attributes': event_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.session_client.delete_with_dependencies(reference['dependency_ids']['session_reference']) - self.student_client.delete_with_dependencies(reference['dependency_ids']['student_reference']) class StudentSectionAttendanceEventClient(EdFiAPIClient): @@ -449,7 +324,8 @@ def create_with_dependencies(self, **kwargs): # Create student section attendance event section_attrs = section_reference['attributes'] - event_attrs = self.factory.build_dict( + return self.create_using_dependencies( + [{'section_client': section_reference}, {'student_client': student_reference}], studentReference__studentUniqueId=student_reference['attributes']['studentUniqueId'], sectionReference__sectionIdentifier=section_attrs['sectionIdentifier'], sectionReference__localCourseCode=section_attrs['courseOfferingReference']['localCourseCode'], @@ -458,21 +334,6 @@ def create_with_dependencies(self, **kwargs): sectionReference__sessionName=section_attrs['courseOfferingReference']['sessionName'], **kwargs ) - event_id = self.create(**event_attrs) - - return { - 'resource_id': event_id, - 'dependency_ids': { - 'student_reference': student_reference, - 'section_reference': section_reference, - }, - 'attributes': event_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.section_client.delete_with_dependencies(reference['dependency_ids']['section_reference']) - self.student_client.delete_with_dependencies(reference['dependency_ids']['student_reference']) class StudentAcademicRecordClient(EdFiAPIClient): @@ -489,24 +350,12 @@ def create_with_dependencies(self, **kwargs): student_reference = self.student_client.create_with_dependencies(schoolId=school_id) # Create student academic record - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + student_reference, studentReference__studentUniqueId=student_reference['attributes']['studentUniqueId'], educationOrganizationReference__educationOrganizationId=school_id, **kwargs ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'student_reference': student_reference - }, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.student_client.delete_with_dependencies(reference['dependency_ids']['student_reference']) class StudentCompetencyObjectiveClient(EdFiAPIClient): @@ -525,25 +374,11 @@ def create_with_dependencies(self, **kwargs): objective_reference = self.competency_objective_client.create_with_dependencies() # Create student competency objective - student_objective_attrs = self.factory.build_dict( + return self.create_using_dependencies( + [{'grading_period_client': period_reference}, {'competency_objective_client': objective_reference}], gradingPeriodReference__periodSequence=period_reference['attributes']['periodSequence'], objectiveCompetencyObjectiveReference__objective=objective_reference['attributes']['objective'] ) - student_objective_id = self.create(**student_objective_attrs) - - return { - 'resource_id': student_objective_id, - 'dependency_ids': { - 'objective_reference': objective_reference, - 'period_reference': period_reference - }, - 'attributes': student_objective_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.competency_objective_client.delete_with_dependencies(reference['dependency_ids']['objective_reference']) - self.grading_period_client.delete_with_dependencies(reference['dependency_ids']['period_reference']) class StudentCTEProgramAssociationClient(EdFiAPIClient): @@ -596,26 +431,17 @@ def create_with_dependencies(self, **kwargs): **section_kwargs ) - student_entry_attrs = self.factory.build_dict( + return self.create_using_dependencies( + [{'assoc_client': student_section_reference}, {'entry_client': entry_id}], gradebookEntryReference=entry_attrs, studentSectionAssociationReference=assoc_attrs ) - student_entry_id = self.create(**student_entry_attrs) - - return { - 'resource_id': student_entry_id, - 'attributes': student_entry_attrs, - 'dependency_ids': { - 'entry_id': entry_id, - 'student_section_reference': student_section_reference - } - } def delete_with_dependencies(self, reference, **kwargs): self.delete(reference['resource_id']) dependencies = reference['dependency_ids'] - self.entry_client.delete(dependencies['entry_id']) - self.assoc_client.delete_with_dependencies(dependencies['student_section_reference']) + self.entry_client.delete(dependencies['entry_client']) + self.assoc_client.delete_with_dependencies(dependencies['assoc_client']) class StudentHomelessProgramAssociationClient(EdFiAPIClient): @@ -632,23 +458,11 @@ class StudentInterventionAssociationClient(EdFiAPIClient): def create_with_dependencies(self, **kwargs): intervention_reference = self.intervention_client.create_with_dependencies() - assoc_attrs = self.factory.build_dict( + return self.create_using_dependencies( + intervention_reference, interventionReference__interventionIdentificationCode=intervention_reference['attributes']['interventionIdentificationCode'], **kwargs ) - assoc_id = self.create(**assoc_attrs) - - return { - 'resource_id': assoc_id, - 'dependency_ids': { - 'intervention_reference': intervention_reference, - }, - 'attributes': assoc_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.intervention_client.delete_with_dependencies(reference['dependency_ids']['intervention_reference']) class StudentInterventionAttendanceEventClient(EdFiAPIClient): @@ -661,23 +475,11 @@ class StudentInterventionAttendanceEventClient(EdFiAPIClient): def create_with_dependencies(self, **kwargs): intervention_reference = self.intervention_client.create_with_dependencies() - event_attrs = self.factory.build_dict( + return self.create_using_dependencies( + intervention_reference, interventionReference__interventionIdentificationCode=intervention_reference['attributes']['interventionIdentificationCode'], **kwargs ) - event_id = self.create(**event_attrs) - - return { - 'resource_id': event_id, - 'dependency_ids': { - 'intervention_reference': intervention_reference, - }, - 'attributes': event_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.intervention_client.delete_with_dependencies(reference['dependency_ids']['intervention_reference']) class StudentLanguageInstructionProgramAssociationClient(EdFiAPIClient): @@ -699,26 +501,12 @@ def create_with_dependencies(self, **kwargs): period_reference = self.grading_period_client.create_with_dependencies() - student_objective_attrs = self.factory.build_dict( - learningObjectiveReference__objective=objective_reference['attributes']['objective'], + return self.create_using_dependencies( + [{'grading_period_client': period_reference}, {'objective_client': objective_reference}], + learningObjectiveReference__learningObjectiveId=objective_reference['attributes']['learningObjectiveId'], gradingPeriodReference__periodSequence=period_reference['attributes']['periodSequence'], **kwargs ) - student_objective_id = self.create(**student_objective_attrs) - - return { - 'resource_id': student_objective_id, - 'dependency_ids': { - 'objective_reference': objective_reference, - 'period_reference': period_reference - }, - 'attributes': student_objective_attrs, - } - - def delete_with_dependencies(self, reference, **kwargs): - self.delete(reference['resource_id']) - self.objective_client.delete_with_dependencies(reference['dependency_ids']['objective_reference']) - self.grading_period_client.delete_with_dependencies(reference['dependency_ids']['period_reference']) class StudentMigrantEducationProgramAssociationClient(EdFiAPIClient): diff --git a/edfi_performance/config/__init__.py b/edfi_performance/config/__init__.py index 983dbd38..5ff4904a 100644 --- a/edfi_performance/config/__init__.py +++ b/edfi_performance/config/__init__.py @@ -6,6 +6,7 @@ import json import os _config = None +_change_version = None def _get_config_file_path(): @@ -27,7 +28,38 @@ def _load_config(): raise ValueError("'{}' is a required key in {}".format(key, config_file)) +def _get_change_version_file_path(): + current_dir = os.path.dirname(os.path.abspath(__file__)) + current_dir = os.path.join(current_dir, '..') + current_dir = os.path.join(current_dir, '..') + change_version_file_path = os.path.join(current_dir, 'change_version_tracker.json') + if not os.path.isfile(change_version_file_path): + with open(change_version_file_path, "w") as change_version_file: + change_version_file.write('{\n "newest_change_version": 0\n}') + return change_version_file_path + + +def _load_change_version(): + global _change_version + + change_version_file = _get_change_version_file_path() + with open(change_version_file, 'r') as infile: + _change_version = json.loads(infile.read()) + + def get_config_value(key): - if _config is None: - _load_config() - return _config[key] + if key == 'newest_change_version': + if _change_version is None: + _load_change_version() + return _change_version[key] + else: + if _config is None: + _load_config() + return _config[key] + + +def set_change_version_value(value): + global _change_version + _change_version = value + with open(_get_change_version_file_path(), 'w') as change_version_file: + change_version_file.write(json.dumps({'newest_change_version': value}, indent=4, separators=(',', ': '))) diff --git a/edfi_performance/factories/resources/assessment.py b/edfi_performance/factories/resources/assessment.py index dd2c71ca..b7c2c557 100644 --- a/edfi_performance/factories/resources/assessment.py +++ b/edfi_performance/factories/resources/assessment.py @@ -12,13 +12,25 @@ class AssessmentFactory(APIFactory): - academicSubjectDescriptor = build_descriptor('AcademicSubject', 'English') - assessedGradeLevelDescriptor = build_descriptor('GradeLevel', 'Twelfth grade') + assessmentIdentifier = UniqueIdAttribute() + academicSubjects = factory.List([ + factory.Dict( + dict( + academicSubjectDescriptor=build_descriptor('AcademicSubject', 'English') + ) + ) + ]) + assessedGradeLevels = factory.List([ + factory.Dict( + dict( + gradeLevelDescriptor=build_descriptor('GradeLevel', 'Twelfth grade') + ) + ) + ]) assessmentTitle = RandomSuffixAttribute("AP - English") assessmentVersion = 2017 - categoryDescriptor = build_descriptor('Category', 'Advanced Placement') maxRawScore = 25 - namespace = 'uri://ed-fi.org/' + namespace = 'uri://ed-fi.org/Assessment/Assessment.xml' identificationCodes = factory.List([ factory.Dict( dict( @@ -61,10 +73,8 @@ class AssessmentFactory(APIFactory): class AssessmentItemFactory(APIFactory): assessmentReference = factory.Dict( dict( - academicSubjectDescriptor=build_descriptor('AcademicSubject', 'English'), - assessedGradeLevelDescriptor=build_descriptor('GradeLevel', 'Twelfth grade'), - assessmentTitle=None, # Must be entered by user - assessmentVersion=2017 + assessmentIdentifier=None, + namespace='uri://ed-fi.org/Assessment/Assessment.xml' ) ) identificationCode = UniqueIdAttribute() @@ -74,9 +84,22 @@ class AssessmentItemFactory(APIFactory): class LearningObjectiveFactory(APIFactory): - academicSubjectDescriptor = build_descriptor('AcademicSubject', 'Mathematics') + academicSubjects = factory.List([ + factory.Dict( + dict( + academicSubjectDescriptor=build_descriptor('AcademicSubject', 'Mathematics'), + namespace='uri://ed-fi.org/' + ) + ) + ]) objective = RandomSuffixAttribute("Number Operations and Concepts") - objectiveGradeLevelDescriptor = build_descriptor('GradeLevel', 'Sixth grade') + gradeLevels = factory.List([ + factory.Dict( + dict( + gradeLevelDescriptor=build_descriptor('GradeLevel', 'Sixth grade') + ) + ) + ]) description = ( "The student will demonstrate the ability to utilize numbers to perform" " operations with complex concepts." @@ -87,10 +110,14 @@ class LearningObjectiveFactory(APIFactory): class LearningStandardFactory(APIFactory): learningStandardId = UniqueIdAttribute() - academicSubjectDescriptor = build_descriptor('AcademicSubject', 'Mathematics') + academicSubjects = factory.List([ + factory.Dict( + dict(academicSubjectDescriptor=build_descriptor('AcademicSubject', 'Mathematics')) + ) + ]) courseTitle = "Advanced Math for students v4.4" description = "Unit 1 Advanced Math for students v4.4" - namespace = 'uri://ed-fi.org' + namespace = 'uri://ed-fi.org/LearningStandard/LearningStandard.xml' gradeLevels = factory.List([ factory.Dict( dict(gradeLevelDescriptor=build_descriptor('GradeLevel', 'Tenth grade')) @@ -101,10 +128,8 @@ class LearningStandardFactory(APIFactory): class ObjectiveAssessmentFactory(APIFactory): assessmentReference = factory.Dict( dict( - academicSubjectDescriptor=build_descriptor('AcademicSubject', 'English'), - assessedGradeLevelDescriptor=build_descriptor('GradeLevel', 'Twelfth grade'), - assessmentTitle=None, # Must be entered by user - assessmentVersion=2017 + assessmentIdentifier=None, + namespace='uri://ed-fi.org/Assessment/Assessment.xml' ) ) identificationCode = UniqueIdAttribute() @@ -112,13 +137,12 @@ class ObjectiveAssessmentFactory(APIFactory): class StudentAssessmentFactory(APIFactory): + studentAssessmentIdentifier = UniqueIdAttribute() studentReference = factory.Dict(dict(studentUniqueId=StudentClient.shared_student_id())) # Prepopulated student assessmentReference = factory.Dict( dict( - academicSubjectDescriptor=build_descriptor('AcademicSubject', 'English'), - assessedGradeLevelDescriptor=build_descriptor('GradeLevel', 'Twelfth grade'), - assessmentTitle=None, # Must be entered by user - assessmentVersion=2017 + assessmentIdentifier=None, + namespace='uri://ed-fi.org/Assessment/Assessment.xml' ) ) administrationDate = RandomDateAttribute() # Along with studentReference and assessmentReference, this is the PK diff --git a/edfi_performance/factories/resources/student.py b/edfi_performance/factories/resources/student.py index a7414f56..1d1ff97c 100644 --- a/edfi_performance/factories/resources/student.py +++ b/edfi_performance/factories/resources/student.py @@ -345,9 +345,8 @@ class StudentLearningObjectiveFactory(APIFactory): schoolYear=2014 )) learningObjectiveReference = factory.Dict(dict( - academicSubjectDescriptor=build_descriptor('AcademicSubject', 'Mathematics'), - objective=None, # Must be entered by user - objectiveGradeLevelDescriptor=build_descriptor('GradeLevel', 'Sixth grade') + learningObjectiveId=None, + namespace='uri://ed-fi.org' )) competencyLevelDescriptor = build_descriptor('CompetencyLevel', 'Proficient') diff --git a/edfi_performance/tasks/__init__.py b/edfi_performance/tasks/__init__.py index f9f32019..1b056c1d 100644 --- a/edfi_performance/tasks/__init__.py +++ b/edfi_performance/tasks/__init__.py @@ -121,6 +121,10 @@ def generate_client_class(self): elif 'volume' in self.__class__.__module__: class_name = self.__class__.__name__.replace('VolumeTest', 'Client') class_path = self.__class__.__module__.replace('tasks.volume', 'api.client') + '.' + class_name + elif 'change_query' in self.__class__.__module__: + class_name = self.__class__.__name__.replace('ChangeQueryTest', 'Client') + class_path = self.__class__.__module__.replace('tasks.change_query', 'api.client') + '.' + class_name + self.client_class = _import_from_dotted_path(class_path) @staticmethod diff --git a/edfi_performance/tasks/change_query/__init__.py b/edfi_performance/tasks/change_query/__init__.py new file mode 100644 index 00000000..15e9a040 --- /dev/null +++ b/edfi_performance/tasks/change_query/__init__.py @@ -0,0 +1,128 @@ +# SPDX-License-Identifier: Apache-2.0 +# Licensed to the Ed-Fi Alliance under one or more agreements. +# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +# See the LICENSE and NOTICES files in the project root for more information. + +import timeit +import traceback + +from greenlet import GreenletExit +from locust import task, TaskSequence, TaskSet, runners +from locust.exception import StopLocust, InterruptTaskSet +from edfi_performance.api.basic_client import EdFiBasicAPIClient +from edfi_performance.config import get_config_value, set_change_version_value + + +class EdFiChangeQueryTestBase(TaskSet): + """ + Base class for all "Change Query" tests for the Ed-Fi ODS API. Change Query + tests methodically page through a resource list's changes, simulating a nightly + sync client. + + The mere presence of a `EdFiChangeQueryTestBase` subclass with an endpoint name specified + is enough to test a resource. `ExampleChangeQueryTest` would perform a Change Query test + against resource "examples": + + Usage: + ``` + class ExampleChangeQueryTest(EdFiChangeQueryTestBase): + endpoint = 'examples' + + ``` + """ + _client = None + + def __init__(self, parent, *args, **kwargs): + super(EdFiChangeQueryTestBase, self).__init__(parent, *args, **kwargs) + self._client = EdFiBasicAPIClient() + + @task + def run_change_query_scenario(self): + try: + self._iterate_through_resource_table() + self._proceed_to_next_change_query_test() + except (StopLocust, GreenletExit, InterruptTaskSet, KeyboardInterrupt): + raise + except Exception: + traceback.print_exc() + self._proceed_to_next_change_query_test() + + def _iterate_through_resource_table(self): + num_of_results = 0 + offset = 0 + limit = 100 + time = 0 + min_change_version = get_config_value('newest_change_version') + if min_change_version != 0: + min_change_version += 1 + + while True: + if offset > 0 and offset % 10000 == 0: + print 'Offset has reached: {}'.format(offset) + query = "?offset={}&limit={}&minChangeVersion={}".format(offset, limit, min_change_version) + start = timeit.default_timer() + endpoint = self.endpoint + results = self._touch_get_list_endpoint(endpoint, query) + stop = timeit.default_timer() + time += (stop - start) + if results is None: + break + num_of_results += len(results) + if len(results) < limit: + break + offset += limit + + print '{} Sync: {} seconds'.format(endpoint, time) + print '{} results returned for {}'.format(num_of_results, endpoint) + + def _touch_get_list_endpoint(self, endpoint, query): + return self._client.get_list(endpoint, query) + + def _proceed_to_next_change_query_test(self): + self.interrupt() + + +class EdFiChangeQueryTaskSequence(TaskSequence): + """ + Base class for the sequence of tasks involved in a Change Query test. Logs in + to get a token to be shared among all child task sets. + + The change_query_tests.py locustfile will automatically detect and append each + child task set to be run to the `tasks` attribute. + """ + tasks = [] + + def __init__(self, *args, **kwargs): + super(EdFiChangeQueryTaskSequence, self).__init__(*args, **kwargs) + + +class EdFiChangeQueryTestTerminator(TaskSet): + """ + Append this to the end of a sequence of tasks to terminate the test run + after all other tasks finish. + + This is useful for Change Query tests since they want to hit each changed + resource once and then exit. + + Using this terminator with more than one Locust client will have + unpredictable behavior: the first client to hit this task will cause + the entire Locust run to quit. + """ + + _client = None + + def __init__(self, *args, **kwargs): + super(EdFiChangeQueryTestTerminator, self).__init__(*args, **kwargs) + self._client = EdFiBasicAPIClient('/data/v3') + + def _update_newest_change_version(self): + available_change_versions = self._client.get_list('availableChangeVersions') + if available_change_versions is not None: + newest_change_version = available_change_versions['NewestChangeVersion'] + set_change_version_value(newest_change_version) + print 'Current value of NewestChangeVersion: {}'.format(newest_change_version) + + @task + def finish_change_query_test_run(self): + self._update_newest_change_version() + runners.locust_runner.quit() diff --git a/edfi_performance/tasks/change_query/course.py b/edfi_performance/tasks/change_query/course.py new file mode 100644 index 00000000..6f4a4e8c --- /dev/null +++ b/edfi_performance/tasks/change_query/course.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: Apache-2.0 +# Licensed to the Ed-Fi Alliance under one or more agreements. +# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +# See the LICENSE and NOTICES files in the project root for more information. + +from edfi_performance.tasks.change_query import EdFiChangeQueryTestBase + + +class CourseChangeQueryTest(EdFiChangeQueryTestBase): + endpoint = 'courses' diff --git a/edfi_performance/tasks/change_query/course_offering.py b/edfi_performance/tasks/change_query/course_offering.py new file mode 100644 index 00000000..c770f60c --- /dev/null +++ b/edfi_performance/tasks/change_query/course_offering.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: Apache-2.0 +# Licensed to the Ed-Fi Alliance under one or more agreements. +# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +# See the LICENSE and NOTICES files in the project root for more information. + +from edfi_performance.tasks.change_query import EdFiChangeQueryTestBase + + +class CourseOfferingChangeQueryTest(EdFiChangeQueryTestBase): + endpoint = 'courseOfferings' diff --git a/edfi_performance/tasks/change_query/education.py b/edfi_performance/tasks/change_query/education.py new file mode 100644 index 00000000..08a80991 --- /dev/null +++ b/edfi_performance/tasks/change_query/education.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: Apache-2.0 +# Licensed to the Ed-Fi Alliance under one or more agreements. +# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +# See the LICENSE and NOTICES files in the project root for more information. + +from edfi_performance.tasks.change_query import EdFiChangeQueryTestBase + + +class LocalEducationAgencyChangeQueryTest(EdFiChangeQueryTestBase): + endpoint = 'localEducationAgencies' diff --git a/edfi_performance/tasks/change_query/school.py b/edfi_performance/tasks/change_query/school.py new file mode 100644 index 00000000..09a11ab3 --- /dev/null +++ b/edfi_performance/tasks/change_query/school.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: Apache-2.0 +# Licensed to the Ed-Fi Alliance under one or more agreements. +# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +# See the LICENSE and NOTICES files in the project root for more information. + +from edfi_performance.tasks.change_query import EdFiChangeQueryTestBase + + +class SchoolChangeQueryTest(EdFiChangeQueryTestBase): + endpoint = 'schools' diff --git a/edfi_performance/tasks/change_query/section.py b/edfi_performance/tasks/change_query/section.py new file mode 100644 index 00000000..27f7d68d --- /dev/null +++ b/edfi_performance/tasks/change_query/section.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: Apache-2.0 +# Licensed to the Ed-Fi Alliance under one or more agreements. +# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +# See the LICENSE and NOTICES files in the project root for more information. + +from edfi_performance.tasks.change_query import EdFiChangeQueryTestBase + + +class SectionChangeQueryTest(EdFiChangeQueryTestBase): + endpoint = 'sections' diff --git a/edfi_performance/tasks/change_query/session.py b/edfi_performance/tasks/change_query/session.py new file mode 100644 index 00000000..aec9d2fb --- /dev/null +++ b/edfi_performance/tasks/change_query/session.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: Apache-2.0 +# Licensed to the Ed-Fi Alliance under one or more agreements. +# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +# See the LICENSE and NOTICES files in the project root for more information. + +from edfi_performance.tasks.change_query import EdFiChangeQueryTestBase + + +class SessionChangeQueryTest(EdFiChangeQueryTestBase): + endpoint = 'sessions' diff --git a/edfi_performance/tasks/change_query/staff.py b/edfi_performance/tasks/change_query/staff.py new file mode 100644 index 00000000..d826a4f7 --- /dev/null +++ b/edfi_performance/tasks/change_query/staff.py @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: Apache-2.0 +# Licensed to the Ed-Fi Alliance under one or more agreements. +# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +# See the LICENSE and NOTICES files in the project root for more information. + +from edfi_performance.tasks.change_query import EdFiChangeQueryTestBase + + +class StaffChangeQueryTest(EdFiChangeQueryTestBase): + endpoint = 'staffs' + + +class StaffSectionAssociationChangeQueryTest(EdFiChangeQueryTestBase): + endpoint = 'staffSectionAssociations' diff --git a/edfi_performance/tasks/change_query/student.py b/edfi_performance/tasks/change_query/student.py new file mode 100644 index 00000000..2eac2680 --- /dev/null +++ b/edfi_performance/tasks/change_query/student.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +# Licensed to the Ed-Fi Alliance under one or more agreements. +# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +# See the LICENSE and NOTICES files in the project root for more information. + +from edfi_performance.tasks.change_query import EdFiChangeQueryTestBase + + +class StudentChangeQueryTest(EdFiChangeQueryTestBase): + endpoint = 'students' + + +class StudentSectionAssociationChangeQueryTest(EdFiChangeQueryTestBase): + endpoint = 'studentSectionAssociations' + + +class StudentSectionAttendanceEventChangeQueryTest(EdFiChangeQueryTestBase): + endpoint = 'studentSectionAttendanceEvents' + + +class StudentEducationOrganizationAssociationChangeQueryTest(EdFiChangeQueryTestBase): + endpoint = 'studentEducationOrganizationAssociations' diff --git a/edfi_performance/tasks/pipeclean/descriptors.py b/edfi_performance/tasks/pipeclean/descriptors.py index cfb26243..9849689e 100644 --- a/edfi_performance/tasks/pipeclean/descriptors.py +++ b/edfi_performance/tasks/pipeclean/descriptors.py @@ -104,6 +104,7 @@ 'licenseStatus', 'licenseType', 'limitedEnglishProficiency', + 'locale', 'localEducationAgencyCategory', 'magnetSpecialProgramEmphasisSchool', 'mediumOfInstruction', diff --git a/locust-config.json b/locust-config.json index b114866b..a5f7d671 100644 --- a/locust-config.json +++ b/locust-config.json @@ -11,5 +11,6 @@ "sql_data_path": "C:\\Program Files\\Microsoft SQL Server\\MSSQL14.MSSQLSERVER\\MSSQL\\DATA", "database_name": "EdFi_Ods_Sandbox_populatedSandbox", "backup_filename": "EdFi_Ods_Northridge.bak", + "change_query_backup_filenames": [ "EdFi_Ods_Northridge.bak" ], "restore_database": "false" } diff --git a/package.ps1 b/package.ps1 index 6e22f196..729effc6 100644 --- a/package.ps1 +++ b/package.ps1 @@ -55,6 +55,7 @@ main { --include=deploy.ps1 ` --include=volume_tests.py ` --include=pipeclean_tests.py ` + --include=change_query_tests.py ` --include=requirements.txt ` --include=edfi_performance/** ` --include=TestRunner.ps1 ` diff --git a/pipeclean_tests.py b/pipeclean_tests.py index 57f3e1f6..1a0f52b0 100644 --- a/pipeclean_tests.py +++ b/pipeclean_tests.py @@ -8,13 +8,13 @@ registered in the `tasks.pipeclean` package and combines them into a single locust which runs each scenario in order. -In order to restrict which scnearios are run, you can name *PipecleanTest +In order to restrict which scenarios are run, you can name *PipecleanTest classes on the command line in the locust invocation, and only those classes will be added to the run. E.g. -`locust -f pipeclean_tests.py -c 10 --no-web SchoolPipecleanTest StudentPipecleanTest` +`locust -f pipeclean_tests.py -c 1 --no-web SchoolPipecleanTest StudentPipecleanTest` """ import importlib import os