Skip to content

Commit

Permalink
CI: use refactored custom URL functions Bootloader.jl
Browse files Browse the repository at this point in the history
  • Loading branch information
@SimeonEhrig
SimeonEhrig committed Feb 18, 2025
1 parent c84193a commit 9294e3e
Show file tree
Hide file tree
Showing 6 changed files with 74 additions and 282 deletions.
10 changes: 9 additions & 1 deletion .ci/CI/src/Bootloader.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -501,8 +501,16 @@ function main()
end

if is_integ
custom_dependency_urls = CustomDependencyUrls()
append_custom_dependency_urls_from_git_message!(custom_dependency_urls)
append_custom_dependency_urls_from_env_var!(custom_dependency_urls)

add_integration_test_job_yaml!(
cpu_job_yaml, test_package, target_branch, tools_git_repo
cpu_job_yaml,
test_package,
target_branch,
custom_dependency_urls.integ,
tools_git_repo,
)
end

Expand Down
148 changes: 11 additions & 137 deletions .ci/CI/src/modules/IntegTest.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,85 +3,6 @@ using YAML: YAML
using Logging
using IntegrationTests

"""
Contains all git-related information about a package.
# Fields
- `url`: Git url of the original project.
- `modified_url`: Stores the Git url set by the environment variable.
- `env_var`: Name of the environment variable to set the modified_url.
"""
mutable struct PackageInfo
url::String
modified_url::String
env_var::String
PackageInfo(url, env_var) = new(url, "", env_var)
end

"""
extract_env_vars_from_git_message!(package_infos::AbstractDict{String, PackageInfo}, var_name = "CI_COMMIT_MESSAGE")
Parse the commit message, if set via variable (usual `CI_COMMIT_MESSAGE`), and set custom URLs.
"""
function extract_env_vars_from_git_message!(
package_infos::AbstractDict{String,PackageInfo}, var_name="CI_COMMIT_MESSAGE"
)
if haskey(ENV, var_name)
for line in split(ENV[var_name], "\n")
line = strip(line)
for pkg_info in values(package_infos)
if startswith(line, pkg_info.env_var * ": ")
ENV[pkg_info.env_var] = SubString(
line, length(pkg_info.env_var * ": ") + 1
)
end
end
end
end
end

"""
modify_package_url!(package_infos::AbstractDict{String, PackageInfo})
Iterate over all entries of package_info. If an environment variable exists with the same name as,
the `env_var` entry, set the value of the environment variable to `modified_url`.
"""
function modify_package_url!(package_infos::AbstractDict{String,PackageInfo})
for package_info in values(package_infos)
if haskey(ENV, package_info.env_var)
package_info.modified_url = ENV[package_info.env_var]
end
end
end

"""
modified_package_name(package_infos::AbstractDict{String, PackageInfo})
Read the name of the modified (project) package from the environment variable `CI_DEV_PKG_NAME`.
# Returns
- The name of the modified (project) package
"""
function modified_package_name(package_infos::AbstractDict{String,PackageInfo})
for env_var in ["CI_DEV_PKG_NAME", "CI_PROJECT_DIR"]
if !haskey(ENV, env_var)
error("Environment variable $env_var is not set.")
end
end

if !haskey(package_infos, ENV["CI_DEV_PKG_NAME"])
package_name = ENV["CI_DEV_PKG_NAME"]
error("Error unknown package name $package_name}")
else
return ENV["CI_DEV_PKG_NAME"]
end
end

function clean_pkg_name(pkg_name::AbstractString)
# remove color tags (?) from the package names
return replace(pkg_name, r"\{[^}]*\}" => "")
end

"""
generate_job_yaml!(
package_name::String,
Expand Down Expand Up @@ -114,17 +35,15 @@ function generate_job_yaml!(
test_package::TestPackage,
target_branch::AbstractString,
job_yaml::Dict,
package_infos::AbstractDict{String,PackageInfo},
custom_urls::Dict{String,String},
tools_git_repo::ToolsGitRepo,
stage::AbstractString="",
can_fail::Bool=false,
)
package_info = package_infos[package_name]
# if modified_url is empty, use original url
if package_info.modified_url == ""
url = package_info.url
if haskey(custom_urls, package_name)
url = custom_urls[package_name]
else
url = package_info.modified_url
url = "https://github.com/QEDjl-project/$(package_name).jl.git"
end

script = ["apt update", "apt install -y git", "cd /"]
Expand Down Expand Up @@ -201,38 +120,6 @@ function generate_dummy_job_yaml!(job_yaml::Dict)
)
end

"""
get_package_info()::Dict{String,PackageInfo}
Returns a list with QED project package information.
"""
function get_package_info()::Dict{String,PackageInfo}
return Dict(
"QuantumElectrodynamics" => PackageInfo(
"https://github.com/QEDjl-project/QuantumElectrodynamics.jl.git",
"CI_INTG_PKG_URL_QED",
),
"QEDfields" => PackageInfo(
"https://github.com/QEDjl-project/QEDfields.jl.git",
"CI_INTG_PKG_URL_QEDfields",
),
"QEDbase" => PackageInfo(
"https://github.com/QEDjl-project/QEDbase.jl.git", "CI_INTG_PKG_URL_QEDbase"
),
"QEDevents" => PackageInfo(
"https://github.com/QEDjl-project/QEDevents.jl.git",
"CI_INTG_PKG_URL_QEDevents",
),
"QEDprocesses" => PackageInfo(
"https://github.com/QEDjl-project/QEDprocesses.jl.git",
"CI_INTG_PKG_URL_QEDprocesses",
),
"QEDcore" => PackageInfo(
"https://github.com/QEDjl-project/QEDcore.jl.git", "CI_INTG_PKG_URL_QEDcore"
),
)
end

"""
add_integration_test_job_yaml!(
job_dict::Dict,
Expand All @@ -255,28 +142,21 @@ function add_integration_test_job_yaml!(
job_dict::Dict,
test_package::TestPackage,
target_branch::AbstractString,
custom_urls::Dict{String,String},
tools_git_repo::ToolsGitRepo,
)
_add_stage_once!(job_dict, "integ-test")

package_infos = get_package_info()
if target_branch != "main"
extract_env_vars_from_git_message!(package_infos)
if target_branch == "main"
empty!(custom_urls)
end
modify_package_url!(package_infos)

custom_urls = Dict{String,String}()
for (name, info) in package_infos
if info.modified_url != ""
custom_urls[name] = info.modified_url
end
end
qed_path = mktempdir(; cleanup=false)
compat_changes = Dict{String,String}()

pkg_tree = build_qed_dependency_graph!(qed_path, compat_changes, custom_urls)
depending_pkg = IntegrationTests.depending_projects(
test_package.name, collect(keys(package_infos)), pkg_tree
test_package.name, r"^QED*|^QuantumElectrodynamics$", pkg_tree
)

if isempty(depending_pkg)
Expand All @@ -299,20 +179,14 @@ function add_integration_test_job_yaml!(
# because of their current compat entries.
if target_branch == "main" && is_pull_request()
generate_job_yaml!(
p,
test_package,
"dev",
job_dict,
package_infos,
tools_git_repo,
"integ-test",
p, test_package, "dev", job_dict, custom_urls, tools_git_repo, "integ-test"
)
generate_job_yaml!(
p,
test_package,
"main",
job_dict,
package_infos,
custom_urls,
tools_git_repo,
"integ-test",
true,
Expand All @@ -326,7 +200,7 @@ function add_integration_test_job_yaml!(
# simplify the interface
"dev",
job_dict,
package_infos,
custom_urls,
tools_git_repo,
"integ-test",
)
Expand Down
46 changes: 10 additions & 36 deletions .ci/CI/src/modules/Utils.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,16 @@ struct ToolsGitRepo
branch::String
end

"""
struct CustomDependencyUrls
Stores custom repository URLs for QED packages which are dependency of the project to be tested.
# Members
- `unit::Dict{String,String}`: Custom dependencies of the unit tests
- `integ::Dict{String,String}`: Custom dependencies of the integration tests
"""
struct CustomDependencyUrls
unit::Dict{String,String}
integ::Dict{String,String}
Expand Down Expand Up @@ -445,42 +455,6 @@ function append_custom_dependency_urls_from_git_message!(
end
end

"""
extract_env_vars_from_git_message!(
env_prefix::AbstractString, var_name::AbstractString="CI_COMMIT_MESSAGE"
)
Parse the commit message, if set via variable (usual `CI_COMMIT_MESSAGE`) and set custom URLs.
A line is parsed if it has the shape of:
<env_prefix>_rest_of_the_env_name: <value>
Each parsed line is added to the environment variables:
ENV[<env_prefix>_rest_of_the_env_name] = <value>
# Args
- `env_prefix::AbstractString`: Parse all lines starting with the env_prefix
- `var_name::AbstractString``: Environemnt variable where git message is stored
(default: "CI_COMMIT_MESSAGE").
"""
function extract_env_vars_from_git_message!(
env_prefix::AbstractString, var_name::AbstractString="CI_COMMIT_MESSAGE"
)
if haskey(ENV, var_name)
@info "Found env variable $var_name"
for line in split(ENV[var_name], "\n")
line = strip(line)
if startswith(line, env_prefix)
(pkg_name, url) = split(line, ":"; limit=2)
@info "add " * pkg_name * "=" * strip(url)
ENV[pkg_name] = strip(url)
end
end
end
end

"""
_add_stage_once!(job_dict::Dict, stage_name::AbstractString)
Expand Down
68 changes: 36 additions & 32 deletions .ci/CI/test/Util/extract_custom_url_test.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,48 +156,52 @@ end
end

@testset "wrong custom integration test dependency URL" begin
custom_dependency_urls = CI.CustomDependencyUrls()
@test_throws ErrorException CI.append_custom_dependency_urls_from_git_message!(
custom_dependency_urls,
Dict{String,String}(
"CI_COMMIT_MESSAGE" => """Git headline
disable_info_logger_output() do
custom_dependency_urls = CI.CustomDependencyUrls()
@test_throws ErrorException CI.append_custom_dependency_urls_from_git_message!(
custom_dependency_urls,
Dict{String,String}(
"CI_COMMIT_MESSAGE" => """Git headline
This is a nice message.
And another line.
This is a nice message.
And another line.
CI_INTG_PKG_URL_QEDfields=https//github.com/integ/QEDfields
""",
),
)
CI_INTG_PKG_URL_QEDfields=https//github.com/integ/QEDfields
""",
),
)

@test_throws ErrorException CI.append_custom_dependency_urls_from_git_message!(
custom_dependency_urls,
Dict{String,String}(
"CI_COMMIT_MESSAGE" => """Git headline
@test_throws ErrorException CI.append_custom_dependency_urls_from_git_message!(
custom_dependency_urls,
Dict{String,String}(
"CI_COMMIT_MESSAGE" => """Git headline
This is a nice message.
And another line.
This is a nice message.
And another line.
CI_INTG_PKG_URL_QEDfields=https://github.com/integ/QEDfields
""",
),
)
CI_INTG_PKG_URL_QEDfields=https://github.com/integ/QEDfields
""",
),
)
end
end

@testset "wrong custom unit test dependency URL" begin
custom_dependency_urls = CI.CustomDependencyUrls()
@test_throws ErrorException CI.append_custom_dependency_urls_from_git_message!(
custom_dependency_urls,
Dict{String,String}(
"CI_COMMIT_MESSAGE" => """Git headline
disable_info_logger_output() do
custom_dependency_urls = CI.CustomDependencyUrls()
@test_throws ErrorException CI.append_custom_dependency_urls_from_git_message!(
custom_dependency_urls,
Dict{String,String}(
"CI_COMMIT_MESSAGE" => """Git headline
This is a nice message.
And another line.
This is a nice message.
And another line.
CI_UNIT_PKG_URL_QEDfields=https://github.com/integ/QEDfields
""",
),
)
CI_UNIT_PKG_URL_QEDfields=https://github.com/integ/QEDfields
""",
),
)
end
end
end

Expand Down
Loading

0 comments on commit 9294e3e

Please sign in to comment.