Release v2.0.0

.codeqlmanifest.json

@@ -1 +1 @@
-{ "provide": [ "codeql_modules/*/.codeqlmanifest.json", "cpp/.codeqlmanifest.json"] }
+{ "provide": [ "codeql_modules/*/.codeqlmanifest.json", "cpp/.codeqlmanifest.json", "c/.codeqlmanifest.json"] }

.github/actions/install-codeql/action.yml

@@ -90,7 +90,7 @@ runs:
 
  if [ "$CODEQL_STDLIB_VERSION" != "latest" ]
  then
- pushd codeql-stdlib
+ push codeql-stdlib
  echo "::debug::Switching to revision $CODEQL_STDLIB_VERSION"
  git checkout $CODEQL_STDLIB_VERSION
  popd

.github/workflows/code-scanning-pack-gen.yml

@@ -5,6 +5,7 @@ on:
  branches:
  - main
  - "rc/**"
+
  push:
  branches:
  - main
@@ -69,14 +70,17 @@ jobs:
  run: |
  PATH=$PATH:$CODEQL_HOME/codeql
  pip install -r scripts/requirements.txt
- find rule_packages/ -name '*.json' -exec basename {} .json \; | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 python3 scripts/generate_rules/generate_package_files.py -a
+ find rule_packages/cpp -name '*.json' -exec basename {} .json \; | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 python3 scripts/generate_rules/generate_package_files.py -a cpp
+ find rule_packages/c -name '*.json' -exec basename {} .json \; | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 python3 scripts/generate_rules/generate_package_files.py -a c
 
  echo "Generating help markdown file for cert"
  $CODEQL_LATEST_HOME/codeql/codeql generate query-help -vvv --format=markdown -o cpp/cert/src/ cpp/cert/src/rules
+ $CODEQL_LATEST_HOME/codeql/codeql generate query-help -vvv --format=markdown -o c/cert/src/ c/cert/src/rules
 
  codeql query compile --search-path cpp --threads 0 cpp
+ codeql query compile --search-path c --search-path cpp --threads 0 c
  cd ..
- zip -r codeql-coding-standards/code-scanning-cpp-query-pack-anon.zip codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/deviations codeql-coding-standards/scripts/reports
+ zip -r codeql-coding-standards/code-scanning-cpp-query-pack-anon.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/deviations codeql-coding-standards/scripts/reports
 
  - name: Upload GHAS Query Pack
  uses: actions/upload-artifact@v2
@@ -91,10 +95,10 @@ jobs:
  PATH=$PATH:$CODEQL_HOME/codeql
  mkdir -p lgtm-cpp-query-pack
  function copy_queries_for_pack {
- for q in $(codeql resolve queries cpp/$1/src/codeql-suites/$1-default.qls)
+ for q in $(codeql resolve queries $2/$1/src/codeql-suites/$1-default.qls)
  do
  copy_from_root_name="${q%.*}"
- copy_to_root_name=$(realpath --relative-to "./cpp/$1/src/" "$copy_from_root_name")
+ copy_to_root_name=$(realpath --relative-to "./$2/$1/src/" "$copy_from_root_name")
  query_dir=$(dirname "lgtm-cpp-query-pack/$copy_to_root_name")
  mkdir -p "$query_dir"
  # Copy each selected ql file, plus the related files (qhelp, qhelp implementation)
@@ -103,10 +107,18 @@ jobs:
  cp "$copy_from_root_name.qhelp" "lgtm-cpp-query-pack/$copy_to_root_name.qhelp"
  done
  }
- echo "Copying autosar-default queries"
- copy_queries_for_pack "autosar"
- echo "Copying cert-default queries"
- copy_queries_for_pack "cert"
+ 
+ echo "Copying autosar-default queries (CPP)"
+ copy_queries_for_pack "autosar" "cpp"
+ echo "Copying cert-default queries (CPP)"
+ copy_queries_for_pack "cert" "cpp"
+
+ echo "Copying misra-default queries (C)"
+ copy_queries_for_pack "misra" "c"
+ echo "Copying cert-default queries (C)"
+ copy_queries_for_pack "cert" "c"
+
+
  # Now copy all the .qll files
  lgtm_pack_dir=$(realpath lgtm-cpp-query-pack)
  for query_pack in autosar cert common
@@ -122,6 +134,22 @@ jobs:
  done
  popd
  done
+
+
+ for query_pack in misra cert common
+ do
+ echo "Copying $query_pack qll files"
+ pushd c/$query_pack/src
+ for query_library in $(find . -name \*.qll)
+ do
+ qll_dir=$(dirname "$lgtm_pack_dir/$query_library")
+ echo "Making $qll_dir"
+ mkdir -p "$qll_dir"
+ cp "$query_library" "$lgtm_pack_dir/$query_library"
+ done
+ popd
+ done
+
  cd lgtm-cpp-query-pack
  zip -9 -r ../lgtm-cpp-query-pack-anon.zip *
 
@@ -174,9 +202,19 @@ jobs:
  echo "Generating help markdown file for $s"
  $CODEQL_LATEST_HOME/codeql/codeql generate query-help -vvv --format=markdown -o cpp/$s/src/ cpp/$s/src/rules
  done
+
+ for s in "misra" "cert"
+ do
+ echo "Generating help markdown file for $s"
+ $CODEQL_LATEST_HOME/codeql/codeql generate query-help -vvv --format=markdown -o c/$s/src/ c/$s/src/rules
+ done
+
+
  codeql query compile --search-path cpp --threads 0 cpp
+ codeql query compile --search-path c --search-path cpp --threads 0 c
+
  cd ..
- zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/deviations codeql-coding-standards/scripts/reports
+ zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/deviations codeql-coding-standards/scripts/reports
 
  - name: Upload GHAS Query Pack
  uses: actions/upload-artifact@v2
@@ -191,18 +229,25 @@ jobs:
  PATH=$PATH:$CODEQL_HOME/codeql
  mkdir -p lgtm-cpp-query-pack
  function copy_queries_for_pack {
- for rule_dir in $(codeql resolve queries cpp/$1/src/codeql-suites/$1-default.qls | xargs -L1 dirname | uniq)
+ for rule_dir in $(codeql resolve queries $2/$1/src/codeql-suites/$1-default.qls | xargs -L1 dirname | uniq)
  do
- copy_to_root="lgtm-cpp-query-pack/$(realpath --relative-to "./cpp/$1/src/" "$rule_dir")"
+ copy_to_root="lgtm-cpp-query-pack/$(realpath --relative-to "./$2/$1/src/" "$rule_dir")"
  mkdir -p "$copy_to_root"
  # Copy each selected ql file, plus the related files (qhelp, qhelp implementation)
- find "$rule_dir" -name '*.ql' -o -name '*.qhelp' -o -name '*.cpp' -o -name '*.png' -exec cp -n {} "$copy_to_root" \;
+ find "$rule_dir" -name '*.ql' -o -name '*.qhelp' -o -name '*.c' -name '*.cpp' -o -name '*.png' -exec cp -n {} "$copy_to_root" \;
  done
  }
- echo "Copying autosar-default queries"
- copy_queries_for_pack "autosar"
- echo "Copying cert-default queries"
- copy_queries_for_pack "cert"
+ echo "Copying autosar-default queries (CPP)"
+ copy_queries_for_pack "autosar" "cpp"
+ echo "Copying cert-default queries (CPP)"
+ copy_queries_for_pack "cert" "cpp"
+
+ echo "Copying misra-default queries (C)"
+ copy_queries_for_pack "misra" "c"
+ echo "Copying cert-default queries (C)"
+ copy_queries_for_pack "cert" "c"
+
+
  # Now copy all the .qll files
  lgtm_pack_dir=$(realpath lgtm-cpp-query-pack)
  for query_pack in autosar cert common
@@ -218,6 +263,21 @@ jobs:
  done
  popd
  done
+
+ for query_pack in misra cert common
+ do
+ echo "Copying $query_pack qll files"
+ pushd c/$query_pack/src
+ for query_library in $(find . -name \*.qll)
+ do
+ qll_dir=$(dirname "$lgtm_pack_dir/$query_library")
+ echo "Making $qll_dir"
+ mkdir -p "$qll_dir"
+ cp "$query_library" "$lgtm_pack_dir/$query_library"
+ done
+ popd
+ done
+
  cd lgtm-cpp-query-pack
  zip -9 -r ../lgtm-cpp-query-pack.zip *
 

.github/workflows/codeql_unit_tests.yml

@@ -23,11 +23,11 @@ jobs:
  - name: Export unit test matrix
  id: export-unit-test-matrix
  run: |
+ echo "Merging Result:"
+ python scripts/create_language_matrix.py
  echo "::set-output name=matrix::$(
- jq --compact-output \
- '.supported_environment | map([.+{os: "ubuntu-20.04-xl", codeql_standard_library_ident : .codeql_standard_library | sub("\/"; "_")}]) | flatten | {include: .}' \
- supported_codeql_configs.json
- )"
+ python scripts/create_language_matrix.py | \
+ jq --compact-output 'map([.+{os: "ubuntu-20.04-xl", codeql_standard_library_ident : .codeql_standard_library | sub("\/"; "_")}]) | flatten | {include: .}')"
 
  run-test-suites:
  name: Run unit tests
@@ -37,7 +37,7 @@ jobs:
  strategy:
  fail-fast: false
  matrix: ${{ fromJSON(needs.prepare-unit-test-matrix.outputs.matrix) }}
-
+ 
  steps:
  - name: Checkout repository
  uses: actions/checkout@v2
@@ -99,8 +99,8 @@ jobs:
  codeql_home = os.environ['CODEQL_HOME']
  codeql_bin = os.path.join(codeql_home, 'codeql', 'codeql')
 
- cpp_root = Path(workspace, 'cpp')
- test_roots = list(map(str, cpp_root.glob('*/test')))
+ language_root = Path(workspace, '${{ matrix.language }}')
+ test_roots = list(map(str, language_root.glob('*/test')))
  for test_root in test_roots:
  print(f"Executing tests found (recursively) in the directory '{test_root}'")
  files_to_close = []
@@ -109,11 +109,11 @@ jobs:
  num_slices = 8
  procs = []
  for slice in range(1, num_slices+1):
- test_report_path = os.path.join(runner_temp, f"test_report_{runner_os}_{cli_version}_{stdlib_ref_ident}_slice_{slice}_of_{num_slices}.json")
+ test_report_path = os.path.join(runner_temp, "${{ matrix.language }}", f"test_report_{runner_os}_{cli_version}_{stdlib_ref_ident}_slice_{slice}_of_{num_slices}.json")
  os.makedirs(os.path.dirname(test_report_path), exist_ok=True)
  test_report_file = open(test_report_path, 'w')
  files_to_close.append(test_report_file)
- procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", f'--search-path={cpp_root}', *test_roots], stdout=test_report_file, stderr=subprocess.PIPE))
+ procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", f'--search-path={language_root}', *test_roots], stdout=test_report_file, stderr=subprocess.PIPE))
  for p in procs:
  p.wait()
  if p.returncode != 0:
@@ -135,7 +135,7 @@ jobs:
  with:
  name: test-results-${{ runner.os }}-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library_ident }}
  path: |
- ${{ runner.temp }}/test_report_${{ runner.os }}_${{ matrix.codeql_cli }}_${{ matrix.codeql_standard_library_ident }}_slice_*.json
+ ${{ runner.temp }}/${{ matrix.language }}/test_report_${{ runner.os }}_${{ matrix.codeql_cli }}_${{ matrix.codeql_standard_library_ident }}_slice_*.json
  if-no-files-found: error
 
  validate-test-results:

.github/workflows/create-draft-release.yml

@@ -5,14 +5,8 @@ on:
  inputs:
  release_version_tag:
  description: |
- The tag for the new draft release, e.g. v0.5.1 (exact string, including the v).
+ The tag for the new draft release, e.g. v0.5.1.
  required: true
-
- previous_release_version_tag:
- description: |
- The tag preceding this release, e.g. v0.5.0 (exact string, including the v).
- required: true
-
  codeql_analysis_threads:
  description: |
  Number of threads to evaluate queries
@@ -52,6 +46,6 @@ jobs:
 
  - name: Create draft release
  run: |
- scripts/release/create_draft_release.sh ${GITHUB_REF#refs/heads/} "$RELEASE_VERSION_TAG" "$PREVIOUS_RELEASE_VERSION_TAG"
+ scripts/release/create_draft_release.sh ${GITHUB_REF#refs/heads/} "$RELEASE_VERSION_TAG"
  env:
  GITHUB_TOKEN: ${{ github.token }}

.github/workflows/upgrade_codeql_dependencies.yml

@@ -42,7 +42,9 @@ jobs:
  unzip -q codeql-linux64.zip
 
  - name: Update CodeQL formatting based on new CLI version
- run: find cpp -name '*.ql' -or -name '*.qll' | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 codeql/codeql query format --in-place
+ run: |
+ find cpp -name '*.ql' -or -name '*.qll' | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 codeql/codeql query format --in-place
+ find c -name '*.ql' -or -name '*.qll' | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 codeql/codeql query format --in-place
 
  - name: Create Pull Request
  uses: peter-evans/create-pull-request@v3