diff --git a/config/dbscheme-fragments.json b/config/dbscheme-fragments.json new file mode 100644 index 000000000000..d3fae0127fa2 --- /dev/null +++ b/config/dbscheme-fragments.json @@ -0,0 +1,43 @@ +{ + "files": [ + "java/ql/lib/config/semmlecode.dbscheme", + "javascript/ql/lib/semmlecode.javascript.dbscheme", + "cpp/ql/lib/semmlecode.cpp.dbscheme", + "python/ql/lib/semmlecode.python.dbscheme", + "ruby/ql/lib/ruby.dbscheme", + "go/ql/lib/go.dbscheme", + "swift/prefix.dbscheme", + "swift/ql/lib/swift.dbscheme", + "csharp/ql/lib/semmlecode.csharp.dbscheme", + "ql/ql/src/ql.dbscheme" + ], + "fragments": [ + "/*- Files and folders -*/", + "/*- Compilations -*/", + "/*- Compilations (Java) -*/", + "/*- Diagnostic messages -*/", + "/*- Duplicate code -*/", + "/*- External data -*/", + "/*- External packages -*/", + "/*- External defects and metrics -*/", + "/*- Source location prefix -*/", + "/*- Lines of code -*/", + "/*- Snapshot date -*/", + "/*- Version control data -*/", + "/*- YAML -*/", + "/*- XML Files -*/", + "/*- SMAP -*/", + "/*- Java -*/", + "/*- configuration files with key value pairs -*/", + "/*- Kotlin -*/", + "/*- JavaScript-specific part -*/", + "/*- Ruby-specific part -*/", + "/*- ERB-specific part -*/", + "/*- GO-specific part -*/", + "/*- QL-specific part -*/", + "/*- Swift-specific part -*/", + "/*- C# dbscheme -*/", + "/*- C++ dbscheme -*/", + "/*- Python dbscheme -*/" + ] +} \ No newline at end of file diff --git a/config/sync-dbscheme-fragments.py b/config/sync-dbscheme-fragments.py new file mode 100755 index 000000000000..b6019544be81 --- /dev/null +++ b/config/sync-dbscheme-fragments.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import json +import os +import re +import itertools + +def make_groups(blocks): + groups = {} + for block in blocks: + groups.setdefault("".join(block["lines"]), []).append(block) + return list(groups.values()) + +def validate_fragments(fragments): + for header, blocks in fragments.items(): + groups = make_groups(blocks) + if len(groups) > 1: + print("Warning: '{}' fragments are different for {}".format(header, ["{}:{}:{}".format(group[0]["file"], group[0]["start"], group[0]["end"]) for group in groups])) + +def main(): + script_dir = os.path.dirname(os.path.realpath(__file__)) + + with open(os.path.join(script_dir, "dbscheme-fragments.json"), "r") as f: + config = json.load(f) + + fragment_headers = set(config["fragments"]) + fragments = {} + for file in config["files"]: + with open(os.path.join(os.path.dirname(script_dir), file), "r") as dbscheme: + header = None + line_number = 1 + block = { "file": file, "start": line_number, "end": None, "lines": [] } + def end_block(): + nonlocal header, block, line_number, fragments + block["end"] = line_number - 1 + if len(block["lines"]) > 0: + if header is None: + if re.match(r'(?m)^//.*$|/\*(\**[^\*])*\*+/', "".join(block["lines"])): + # Ignore comments at the beginning of the file + pass + else: + print("Warning: fragment without header: {}:{}:{}".format(block["file"], block["start"], block["end"])) + else: + fragments.setdefault(header, []).append(block) + for line in dbscheme: + m = re.match(r"^\/\*-.*-\*\/$", line) + if m: + end_block() + header = line.strip() + if header not in fragment_headers: + print("Warning: unknown fragment header: {}: {}:{}".format(header, file, line_number)) + block = { "file": file, "start": line_number, "end": None, "lines": [] } + block["lines"].append(line) + line_number += 1 + end_block() + validate_fragments(fragments) + +if __name__ == "__main__": + main() diff --git a/cpp/ql/lib/semmlecode.cpp.dbscheme b/cpp/ql/lib/semmlecode.cpp.dbscheme index 19887dbd3332..9a8aa05ae606 100644 --- a/cpp/ql/lib/semmlecode.cpp.dbscheme +++ b/cpp/ql/lib/semmlecode.cpp.dbscheme @@ -1,3 +1,4 @@ +/*- Compilations -*/ /** * An invocation of the compiler. Note that more than one file may be @@ -10,118 +11,119 @@ * directory from which the compiler was invoked. */ compilations( - /** - * An invocation of the compiler. Note that more than one file may - * be compiled per invocation. For example, this command compiles - * three source files: - * - * gcc -c f1.c f2.c f3.c - */ - unique int id : @compilation, - string cwd : string ref + /** + * An invocation of the compiler. Note that more than one file may + * be compiled per invocation. For example, this command compiles + * three source files: + * + * gcc -c f1.c f2.c f3.c + */ + unique int id : @compilation, + string cwd : string ref ); /** - * The arguments that were passed to the extractor for a compiler - * invocation. If `id` is for the compiler invocation - * - * gcc -c f1.c f2.c f3.c - * - * then typically there will be rows for - * - * num | arg - * --- | --- - * 0 | *path to extractor* - * 1 | `--mimic` - * 2 | `/usr/bin/gcc` - * 3 | `-c` - * 4 | f1.c - * 5 | f2.c - * 6 | f3.c - */ +* The arguments that were passed to the extractor for a compiler +* invocation. If `id` is for the compiler invocation +* +* gcc -c f1.c f2.c f3.c +* +* then typically there will be rows for +* +* num | arg +* --- | --- +* 0 | *path to extractor* +* 1 | `--mimic` +* 2 | `/usr/bin/gcc` +* 3 | `-c` +* 4 | f1.c +* 5 | f2.c +* 6 | f3.c +*/ #keyset[id, num] compilation_args( - int id : @compilation ref, - int num : int ref, - string arg : string ref + int id : @compilation ref, + int num : int ref, + string arg : string ref ); /** - * The source files that are compiled by a compiler invocation. - * If `id` is for the compiler invocation - * - * gcc -c f1.c f2.c f3.c - * - * then there will be rows for - * - * num | arg - * --- | --- - * 0 | f1.c - * 1 | f2.c - * 2 | f3.c - * - * Note that even if those files `#include` headers, those headers - * do not appear as rows. - */ +* The source files that are compiled by a compiler invocation. +* If `id` is for the compiler invocation +* +* gcc -c f1.c f2.c f3.c +* +* then there will be rows for +* +* num | arg +* --- | --- +* 0 | f1.c +* 1 | f2.c +* 2 | f3.c +* +* Note that even if those files `#include` headers, those headers +* do not appear as rows. +*/ #keyset[id, num] compilation_compiling_files( - int id : @compilation ref, - int num : int ref, - int file : @file ref + int id : @compilation ref, + int num : int ref, + int file : @file ref ); /** - * The time taken by the extractor for a compiler invocation. - * - * For each file `num`, there will be rows for - * - * kind | seconds - * ---- | --- - * 1 | CPU seconds used by the extractor frontend - * 2 | Elapsed seconds during the extractor frontend - * 3 | CPU seconds used by the extractor backend - * 4 | Elapsed seconds during the extractor backend - */ +* The time taken by the extractor for a compiler invocation. +* +* For each file `num`, there will be rows for +* +* kind | seconds +* ---- | --- +* 1 | CPU seconds used by the extractor frontend +* 2 | Elapsed seconds during the extractor frontend +* 3 | CPU seconds used by the extractor backend +* 4 | Elapsed seconds during the extractor backend +*/ #keyset[id, num, kind] compilation_time( - int id : @compilation ref, - int num : int ref, - /* kind: - 1 = frontend_cpu_seconds - 2 = frontend_elapsed_seconds - 3 = extractor_cpu_seconds - 4 = extractor_elapsed_seconds - */ - int kind : int ref, - float seconds : float ref + int id : @compilation ref, + int num : int ref, + /* kind: + 1 = frontend_cpu_seconds + 2 = frontend_elapsed_seconds + 3 = extractor_cpu_seconds + 4 = extractor_elapsed_seconds + */ + int kind : int ref, + float seconds : float ref ); /** - * An error or warning generated by the extractor. - * The diagnostic message `diagnostic` was generated during compiler - * invocation `compilation`, and is the `file_number_diagnostic_number`th - * message generated while extracting the `file_number`th file of that - * invocation. - */ +* An error or warning generated by the extractor. +* The diagnostic message `diagnostic` was generated during compiler +* invocation `compilation`, and is the `file_number_diagnostic_number`th +* message generated while extracting the `file_number`th file of that +* invocation. +*/ #keyset[compilation, file_number, file_number_diagnostic_number] diagnostic_for( - int diagnostic : @diagnostic ref, - int compilation : @compilation ref, - int file_number : int ref, - int file_number_diagnostic_number : int ref + int diagnostic : @diagnostic ref, + int compilation : @compilation ref, + int file_number : int ref, + int file_number_diagnostic_number : int ref ); /** - * If extraction was successful, then `cpu_seconds` and - * `elapsed_seconds` are the CPU time and elapsed time (respectively) - * that extraction took for compiler invocation `id`. - */ +* If extraction was successful, then `cpu_seconds` and +* `elapsed_seconds` are the CPU time and elapsed time (respectively) +* that extraction took for compiler invocation `id`. +*/ compilation_finished( - unique int id : @compilation ref, - float cpu_seconds : float ref, - float elapsed_seconds : float ref + unique int id : @compilation ref, + float cpu_seconds : float ref, + float elapsed_seconds : float ref ); +/*- External data -*/ /** * External data, loaded from CSV files during snapshot creation. See @@ -129,17 +131,21 @@ compilation_finished( * for more information. */ externalData( - int id : @externalDataElement, - string path : string ref, - int column: int ref, - string value : string ref + int id : @externalDataElement, + string path : string ref, + int column: int ref, + string value : string ref ); +/*- Source location prefix -*/ + /** * The source location of the snapshot. */ sourceLocationPrefix(string prefix : string ref); +/*- External packages -*/ + /** * Information about packages that provide code used during compilation. * The `id` is just a unique identifier. @@ -163,41 +169,35 @@ header_to_external_package( int package : @external_package ref ); -/* - * Version history - */ +/*- Version control data -*/ svnentries( - unique int id : @svnentry, - string revision : string ref, - string author : string ref, - date revisionDate : date ref, - int changeSize : int ref + unique int id : @svnentry, + string revision : string ref, + string author : string ref, + date revisionDate : date ref, + int changeSize : int ref ) svnaffectedfiles( - int id : @svnentry ref, - int file : @file ref, - string action : string ref + int id : @svnentry ref, + int file : @file ref, + string action : string ref ) svnentrymsg( - unique int id : @svnentry ref, - string message : string ref + unique int id : @svnentry ref, + string message : string ref ) svnchurn( - int commit : @svnentry ref, - int file : @file ref, - int addedLines : int ref, - int deletedLines : int ref + int commit : @svnentry ref, + int file : @file ref, + int addedLines : int ref, + int deletedLines : int ref ) -/* - * C++ dbscheme - */ - -@location = @location_stmt | @location_expr | @location_default ; +/*- Files and folders -*/ /** * The location of an element that is not an expression or a statement. @@ -207,15 +207,62 @@ svnchurn( * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). */ locations_default( - /** The location of an element that is not an expression or a statement. */ - unique int id: @location_default, - int container: @container ref, - int startLine: int ref, - int startColumn: int ref, - int endLine: int ref, - int endColumn: int ref + /** The location of an element that is not an expression or a statement. */ + unique int id: @location_default, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref ); +files( + unique int id: @file, + string name: string ref +); + +folders( + unique int id: @folder, + string name: string ref +); + +@container = @file | @folder + +containerparent( + int parent: @container ref, + unique int child: @container ref +); + +/*- Lines of code -*/ + +#keyset[element_id] +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); + +/*- Diagnostic messages -*/ + +diagnostics( + unique int id: @diagnostic, + int severity: int ref, + string error_tag: string ref, + string error_message: string ref, + string full_error_message: string ref, + int location: @location_default ref +); + +case @diagnostic.severity of + 10 = @diagnostic_debug +| 20 = @diagnostic_info +| 30 = @diagnostic_warning +| 40 = @diagnostic_error +; + +/*- C++ dbscheme -*/ + /** * The location of a statement. * The location spans column `startcolumn` of line `startline` to @@ -253,38 +300,7 @@ locations_expr( /** An element for which line-count information is available. */ @sourceline = @file | @function | @variable | @enumconstant | @xmllocatable; -numlines( - int element_id: @sourceline ref, - int num_lines: int ref, - int num_code: int ref, - int num_comment: int ref -); - -diagnostics( - unique int id: @diagnostic, - int severity: int ref, - string error_tag: string ref, - string error_message: string ref, - string full_error_message: string ref, - int location: @location_default ref -); - -files( - unique int id: @file, - string name: string ref -); - -folders( - unique int id: @folder, - string name: string ref -); - -@container = @folder | @file - -containerparent( - int parent: @container ref, - unique int child: @container ref -); +@location = @location_stmt | @location_expr | @location_default ; fileannotations( int id: @file ref, @@ -2137,76 +2153,74 @@ link_parent( int link_target : @link_target ref ); -/* XML Files */ +/*- XML Files -*/ -xmlEncoding(unique int id: @file ref, string encoding: string ref); +xmlEncoding( + unique int id: @file ref, + string encoding: string ref +); xmlDTDs( - unique int id: @xmldtd, - string root: string ref, - string publicId: string ref, - string systemId: string ref, - int fileid: @file ref + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref ); xmlElements( - unique int id: @xmlelement, - string name: string ref, - int parentid: @xmlparent ref, - int idx: int ref, - int fileid: @file ref + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref ); xmlAttrs( - unique int id: @xmlattribute, - int elementid: @xmlelement ref, - string name: string ref, - string value: string ref, - int idx: int ref, - int fileid: @file ref + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref ); xmlNs( - int id: @xmlnamespace, - string prefixName: string ref, - string URI: string ref, - int fileid: @file ref + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref ); xmlHasNs( - int elementId: @xmlnamespaceable ref, - int nsId: @xmlnamespace ref, - int fileid: @file ref + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref ); xmlComments( - unique int id: @xmlcomment, - string text: string ref, - int parentid: @xmlparent ref, - int fileid: @file ref + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref ); xmlChars( - unique int id: @xmlcharacters, - string text: string ref, - int parentid: @xmlparent ref, - int idx: int ref, - int isCDATA: int ref, - int fileid: @file ref + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref ); @xmlparent = @file | @xmlelement; @xmlnamespaceable = @xmlelement | @xmlattribute; xmllocations( - int xmlElement: @xmllocatable ref, - int location: @location_default ref + int xmlElement: @xmllocatable ref, + int location: @location_default ref ); -@xmllocatable = @xmlcharacters - | @xmlelement - | @xmlcomment - | @xmlattribute - | @xmldtd - | @file - | @xmlnamespace; +@xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace; + diff --git a/csharp/ql/lib/semmlecode.csharp.dbscheme b/csharp/ql/lib/semmlecode.csharp.dbscheme index cd877b8cc2fb..5981e383e872 100644 --- a/csharp/ql/lib/semmlecode.csharp.dbscheme +++ b/csharp/ql/lib/semmlecode.csharp.dbscheme @@ -9,65 +9,289 @@ * mechanism not work properly. */ +/*- Compilations -*/ + /** * An invocation of the compiler. Note that more than one file may be * compiled per invocation. For example, this command compiles three * source files: * - * csc f1.cs f2.cs f3.cs + * gcc -c f1.c f2.c f3.c * * The `id` simply identifies the invocation, while `cwd` is the working * directory from which the compiler was invoked. */ compilations( - unique int id : @compilation, - string cwd : string ref + /** + * An invocation of the compiler. Note that more than one file may + * be compiled per invocation. For example, this command compiles + * three source files: + * + * gcc -c f1.c f2.c f3.c + */ + unique int id : @compilation, + string cwd : string ref ); /** - * The arguments that were passed to the extractor for a compiler - * invocation. If `id` is for the compiler invocation - * - * csc f1.cs f2.cs f3.cs - * - * then typically there will be rows for - * - * num | arg - * --- | --- - * 0 | --compiler - * 1 | *path to compiler* - * 2 | f1.cs - * 3 | f2.cs - * 4 | f3.cs - */ +* The arguments that were passed to the extractor for a compiler +* invocation. If `id` is for the compiler invocation +* +* gcc -c f1.c f2.c f3.c +* +* then typically there will be rows for +* +* num | arg +* --- | --- +* 0 | *path to extractor* +* 1 | `--mimic` +* 2 | `/usr/bin/gcc` +* 3 | `-c` +* 4 | f1.c +* 5 | f2.c +* 6 | f3.c +*/ #keyset[id, num] compilation_args( - int id : @compilation ref, - int num : int ref, - string arg : string ref + int id : @compilation ref, + int num : int ref, + string arg : string ref ); /** - * The source files that are compiled by a compiler invocation. - * If `id` is for the compiler invocation - * - * csc f1.cs f2.cs f3.cs - * - * then there will be rows for - * - * num | arg - * --- | --- - * 0 | f1.cs - * 1 | f2.cs - * 2 | f3.cs - */ +* The source files that are compiled by a compiler invocation. +* If `id` is for the compiler invocation +* +* gcc -c f1.c f2.c f3.c +* +* then there will be rows for +* +* num | arg +* --- | --- +* 0 | f1.c +* 1 | f2.c +* 2 | f3.c +* +* Note that even if those files `#include` headers, those headers +* do not appear as rows. +*/ #keyset[id, num] compilation_compiling_files( - int id : @compilation ref, - int num : int ref, - int file : @file ref + int id : @compilation ref, + int num : int ref, + int file : @file ref +); + +/** +* The time taken by the extractor for a compiler invocation. +* +* For each file `num`, there will be rows for +* +* kind | seconds +* ---- | --- +* 1 | CPU seconds used by the extractor frontend +* 2 | Elapsed seconds during the extractor frontend +* 3 | CPU seconds used by the extractor backend +* 4 | Elapsed seconds during the extractor backend +*/ +#keyset[id, num, kind] +compilation_time( + int id : @compilation ref, + int num : int ref, + /* kind: + 1 = frontend_cpu_seconds + 2 = frontend_elapsed_seconds + 3 = extractor_cpu_seconds + 4 = extractor_elapsed_seconds + */ + int kind : int ref, + float seconds : float ref +); + +/** +* An error or warning generated by the extractor. +* The diagnostic message `diagnostic` was generated during compiler +* invocation `compilation`, and is the `file_number_diagnostic_number`th +* message generated while extracting the `file_number`th file of that +* invocation. +*/ +#keyset[compilation, file_number, file_number_diagnostic_number] +diagnostic_for( + int diagnostic : @diagnostic ref, + int compilation : @compilation ref, + int file_number : int ref, + int file_number_diagnostic_number : int ref +); + +/** +* If extraction was successful, then `cpu_seconds` and +* `elapsed_seconds` are the CPU time and elapsed time (respectively) +* that extraction took for compiler invocation `id`. +*/ +compilation_finished( + unique int id : @compilation ref, + float cpu_seconds : float ref, + float elapsed_seconds : float ref +); + +/*- Diagnostic messages -*/ + +diagnostics( + unique int id: @diagnostic, + int severity: int ref, + string error_tag: string ref, + string error_message: string ref, + string full_error_message: string ref, + int location: @location_default ref ); +case @diagnostic.severity of + 10 = @diagnostic_debug +| 20 = @diagnostic_info +| 30 = @diagnostic_warning +| 40 = @diagnostic_error +; + +/*- External data -*/ + +/** + * External data, loaded from CSV files during snapshot creation. See + * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data) + * for more information. + */ +externalData( + int id : @externalDataElement, + string path : string ref, + int column: int ref, + string value : string ref +); + +/*- Source location prefix -*/ + +/** + * The source location of the snapshot. + */ +sourceLocationPrefix(string prefix : string ref); + +/*- Files and folders -*/ + +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_default( + /** The location of an element that is not an expression or a statement. */ + unique int id: @location_default, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +files( + unique int id: @file, + string name: string ref +); + +folders( + unique int id: @folder, + string name: string ref +); + +@container = @file | @folder + +containerparent( + int parent: @container ref, + unique int child: @container ref +); + +/*- Lines of code -*/ + +#keyset[element_id] +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); + +/*- XML Files -*/ + +xmlEncoding( + unique int id: @file ref, + string encoding: string ref +); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace; + +/*- C# dbscheme -*/ + /** * The references used by a compiler invocation. * If `id` is for the compiler invocation @@ -89,96 +313,22 @@ compilation_referencing_files( int file : @file ref ); -/** - * The time taken by the extractor for a compiler invocation. - * - * For each file `num`, there will be rows for - * - * kind | seconds - * ---- | --- - * 1 | CPU seconds used by the extractor frontend - * 2 | Elapsed seconds during the extractor frontend - * 3 | CPU seconds used by the extractor backend - * 4 | Elapsed seconds during the extractor backend - */ -#keyset[id, num, kind] -compilation_time( - int id : @compilation ref, - int num : int ref, - /* kind: - 1 = frontend_cpu_seconds - 2 = frontend_elapsed_seconds - 3 = extractor_cpu_seconds - 4 = extractor_elapsed_seconds - */ - int kind : int ref, - float seconds : float ref -); - -/** - * An error or warning generated by the extractor. - * The diagnostic message `diagnostic` was generated during compiler - * invocation `compilation`, and is the `file_number_diagnostic_number`th - * message generated while extracting the `file_number`th file of that - * invocation. - */ -#keyset[compilation, file_number, file_number_diagnostic_number] -diagnostic_for( - unique int diagnostic : @diagnostic ref, - int compilation : @compilation ref, - int file_number : int ref, - int file_number_diagnostic_number : int ref -); - -diagnostics( - unique int id: @diagnostic, - int severity: int ref, - string error_tag: string ref, - string error_message: string ref, - string full_error_message: string ref, - int location: @location_default ref -); - extractor_messages( - unique int id: @extractor_message, - int severity: int ref, - string origin : string ref, - string text : string ref, - string entity : string ref, - int location: @location_default ref, - string stack_trace : string ref + unique int id: @extractor_message, + int severity: int ref, + string origin : string ref, + string text : string ref, + string entity : string ref, + int location: @location_default ref, + string stack_trace : string ref ); -/** - * If extraction was successful, then `cpu_seconds` and - * `elapsed_seconds` are the CPU time and elapsed time (respectively) - * that extraction took for compiler invocation `id`. - */ -compilation_finished( - unique int id : @compilation ref, - float cpu_seconds : float ref, - float elapsed_seconds : float ref -); compilation_assembly( - unique int id : @compilation ref, - int assembly: @assembly ref +unique int id : @compilation ref, +int assembly: @assembly ref ) -// Populated by the CSV extractor -externalData( - int id: @externalDataElement, - string path: string ref, - int column: int ref, - string value: string ref); - -sourceLocationPrefix( - string prefix: string ref); - -/* - * C# dbscheme - */ - /** ELEMENTS **/ @element = @declaration | @stmt | @expr | @modifier | @attribute | @namespace_declaration @@ -203,26 +353,12 @@ sourceLocationPrefix( @location = @location_default | @assembly; -locations_default( - unique int id: @location_default, - int file: @file ref, - int beginLine: int ref, - int beginColumn: int ref, - int endLine: int ref, - int endColumn: int ref); - locations_mapped( unique int id: @location_default ref, int mapped_to: @location_default ref); @sourceline = @file | @callable | @xmllocatable; -numlines( - int element_id: @sourceline ref, - int num_lines: int ref, - int num_code: int ref, - int num_comment: int ref); - assemblies( unique int id: @assembly, int file: @file ref, @@ -230,20 +366,6 @@ assemblies( string name: string ref, string version: string ref); -files( - unique int id: @file, - string name: string ref); - -folders( - unique int id: @folder, - string name: string ref); - -@container = @folder | @file ; - -containerparent( - int parent: @container ref, - unique int child: @container ref); - file_extraction_mode( unique int file: @file ref, int mode: int ref @@ -1297,68 +1419,6 @@ lambda_expr_return_type( @control_flow_element = @stmt | @expr; -/* XML Files */ - -xmlEncoding ( - unique int id: @file ref, - string encoding: string ref); - -xmlDTDs( - unique int id: @xmldtd, - string root: string ref, - string publicId: string ref, - string systemId: string ref, - int fileid: @file ref); - -xmlElements( - unique int id: @xmlelement, - string name: string ref, - int parentid: @xmlparent ref, - int idx: int ref, - int fileid: @file ref); - -xmlAttrs( - unique int id: @xmlattribute, - int elementid: @xmlelement ref, - string name: string ref, - string value: string ref, - int idx: int ref, - int fileid: @file ref); - -xmlNs( - int id: @xmlnamespace, - string prefixName: string ref, - string URI: string ref, - int fileid: @file ref); - -xmlHasNs( - int elementId: @xmlnamespaceable ref, - int nsId: @xmlnamespace ref, - int fileid: @file ref); - -xmlComments( - unique int id: @xmlcomment, - string text: string ref, - int parentid: @xmlparent ref, - int fileid: @file ref); - -xmlChars( - unique int id: @xmlcharacters, - string text: string ref, - int parentid: @xmlparent ref, - int idx: int ref, - int isCDATA: int ref, - int fileid: @file ref); - -@xmlparent = @file | @xmlelement; -@xmlnamespaceable = @xmlelement | @xmlattribute; - -xmllocations( - int xmlElement: @xmllocatable ref, - int location: @location_default ref); - -@xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace; - /* Comments */ commentline( diff --git a/go/ql/lib/go.dbscheme b/go/ql/lib/go.dbscheme index a58b81b1b4c4..b299d649c0c3 100644 --- a/go/ql/lib/go.dbscheme +++ b/go/ql/lib/go.dbscheme @@ -1,19 +1,20 @@ /** Auto-generated dbscheme; do not edit. */ - -/** Duplicate code **/ +/*- Duplicate code -*/ duplicateCode( unique int id : @duplication, - varchar(900) relativePath : string ref, - int equivClass : int ref); + string relativePath : string ref, + int equivClass : int ref +); similarCode( unique int id : @similarity, - varchar(900) relativePath : string ref, - int equivClass : int ref); + string relativePath : string ref, + int equivClass : int ref +); -@duplication_or_similarity = @duplication | @similarity; +@duplication_or_similarity = @duplication | @similarity tokens( int id : @duplication_or_similarity ref, @@ -21,25 +22,35 @@ tokens( int beginLine : int ref, int beginColumn : int ref, int endLine : int ref, - int endColumn : int ref); + int endColumn : int ref +); -/** External data **/ +/*- External data -*/ +/** + * External data, loaded from CSV files during snapshot creation. See + * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data) + * for more information. + */ externalData( int id : @externalDataElement, - varchar(900) path : string ref, + string path : string ref, int column: int ref, - varchar(900) value : string ref + string value : string ref ); -snapshotDate(unique date snapshotDate : date ref); +/*- Snapshot date -*/ -sourceLocationPrefix(varchar(900) prefix : string ref); +snapshotDate(unique date snapshotDate : date ref); +/*- Source location prefix -*/ -/* - * XML Files +/** + * The source location of the snapshot. */ +sourceLocationPrefix(string prefix : string ref); + +/*- XML Files -*/ xmlEncoding( unique int id: @file ref, @@ -110,34 +121,196 @@ xmllocations( @xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace; -compilations(unique int id: @compilation, string cwd: string ref); +/*- Compilations -*/ + +/** + * An invocation of the compiler. Note that more than one file may be + * compiled per invocation. For example, this command compiles three + * source files: + * + * gcc -c f1.c f2.c f3.c + * + * The `id` simply identifies the invocation, while `cwd` is the working + * directory from which the compiler was invoked. + */ +compilations( + /** + * An invocation of the compiler. Note that more than one file may + * be compiled per invocation. For example, this command compiles + * three source files: + * + * gcc -c f1.c f2.c f3.c + */ + unique int id : @compilation, + string cwd : string ref +); +/** +* The arguments that were passed to the extractor for a compiler +* invocation. If `id` is for the compiler invocation +* +* gcc -c f1.c f2.c f3.c +* +* then typically there will be rows for +* +* num | arg +* --- | --- +* 0 | *path to extractor* +* 1 | `--mimic` +* 2 | `/usr/bin/gcc` +* 3 | `-c` +* 4 | f1.c +* 5 | f2.c +* 6 | f3.c +*/ #keyset[id, num] -compilation_args(int id: @compilation ref, int num: int ref, string arg: string ref); +compilation_args( + int id : @compilation ref, + int num : int ref, + string arg : string ref +); +/** +* The source files that are compiled by a compiler invocation. +* If `id` is for the compiler invocation +* +* gcc -c f1.c f2.c f3.c +* +* then there will be rows for +* +* num | arg +* --- | --- +* 0 | f1.c +* 1 | f2.c +* 2 | f3.c +* +* Note that even if those files `#include` headers, those headers +* do not appear as rows. +*/ +#keyset[id, num] +compilation_compiling_files( + int id : @compilation ref, + int num : int ref, + int file : @file ref +); + +/** +* The time taken by the extractor for a compiler invocation. +* +* For each file `num`, there will be rows for +* +* kind | seconds +* ---- | --- +* 1 | CPU seconds used by the extractor frontend +* 2 | Elapsed seconds during the extractor frontend +* 3 | CPU seconds used by the extractor backend +* 4 | Elapsed seconds during the extractor backend +*/ #keyset[id, num, kind] -compilation_time(int id: @compilation ref, int num: int ref, int kind: int ref, float secs: float ref); +compilation_time( + int id : @compilation ref, + int num : int ref, + /* kind: + 1 = frontend_cpu_seconds + 2 = frontend_elapsed_seconds + 3 = extractor_cpu_seconds + 4 = extractor_elapsed_seconds + */ + int kind : int ref, + float seconds : float ref +); -diagnostic_for(unique int diagnostic: @diagnostic ref, int compilation: @compilation ref, int file_number: int ref, int file_number_diagnostic_number: int ref); +/** +* An error or warning generated by the extractor. +* The diagnostic message `diagnostic` was generated during compiler +* invocation `compilation`, and is the `file_number_diagnostic_number`th +* message generated while extracting the `file_number`th file of that +* invocation. +*/ +#keyset[compilation, file_number, file_number_diagnostic_number] +diagnostic_for( + int diagnostic : @diagnostic ref, + int compilation : @compilation ref, + int file_number : int ref, + int file_number_diagnostic_number : int ref +); -compilation_finished(unique int id: @compilation ref, float cpu_seconds: float ref, float elapsed_seconds: float ref); +/** +* If extraction was successful, then `cpu_seconds` and +* `elapsed_seconds` are the CPU time and elapsed time (respectively) +* that extraction took for compiler invocation `id`. +*/ +compilation_finished( + unique int id : @compilation ref, + float cpu_seconds : float ref, + float elapsed_seconds : float ref +); -#keyset[id, num] -compilation_compiling_files(int id: @compilation ref, int num: int ref, int file: @file ref); +/*- Diagnostic messages -*/ -diagnostics(unique int id: @diagnostic, int severity: int ref, string error_tag: string ref, string error_message: string ref, - string full_error_message: string ref, int location: @location ref); +diagnostics( + unique int id: @diagnostic, + int severity: int ref, + string error_tag: string ref, + string error_message: string ref, + string full_error_message: string ref, + int location: @location_default ref +); -locations_default(unique int id: @location_default, int file: @file ref, int beginLine: int ref, int beginColumn: int ref, - int endLine: int ref, int endColumn: int ref); +case @diagnostic.severity of + 10 = @diagnostic_debug +| 20 = @diagnostic_info +| 30 = @diagnostic_warning +| 40 = @diagnostic_error +; + +/*- Files and folders -*/ + +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_default( + /** The location of an element that is not an expression or a statement. */ + unique int id: @location_default, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); -numlines(int element_id: @sourceline ref, int num_lines: int ref, int num_code: int ref, int num_comment: int ref); +files( + unique int id: @file, + string name: string ref +); -files(unique int id: @file, string name: string ref); +folders( + unique int id: @folder, + string name: string ref +); + +@container = @file | @folder -folders(unique int id: @folder, string name: string ref); +containerparent( + int parent: @container ref, + unique int child: @container ref +); -containerparent(int parent: @container ref, unique int child: @container ref); +/*- Lines of code -*/ + +#keyset[element_id] +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); + +/*- GO-specific part -*/ has_location(unique int locatable: @locatable ref, int location: @location ref); @@ -231,8 +404,6 @@ variadic(int id: @signaturetype ref); typeparam(unique int tp: @typeparamtype ref, string name: string ref, int bound: @compositetype ref, int parent: @typeparamparentobject ref, int idx: int ref); -@container = @file | @folder; - @locatable = @xmllocatable | @node | @localscope; @node = @documentable | @exprparent | @modexprparent | @fieldparent | @stmtparent | @declparent | @typeparamdeclparent diff --git a/java/ql/lib/config/semmlecode.dbscheme b/java/ql/lib/config/semmlecode.dbscheme index 7cbc85b1f3ec..67f6c5c0644c 100644 --- a/java/ql/lib/config/semmlecode.dbscheme +++ b/java/ql/lib/config/semmlecode.dbscheme @@ -1,3 +1,4 @@ +/*- Compilations (Java) -*/ /** * An invocation of the compiler. Note that more than one file may be * compiled per invocation. For example, this command compiles three @@ -181,20 +182,32 @@ compilation_finished( int result : int ref ); +/*- Diagnostic messages -*/ + diagnostics( - unique int id: @diagnostic, - string generated_by: string ref, // TODO: Sync this with the other languages? - int severity: int ref, - string error_tag: string ref, - string error_message: string ref, - string full_error_message: string ref, - int location: @location_default ref + unique int id: @diagnostic, + string generated_by: string ref, // TODO: Sync this with the other languages? + int severity: int ref, + string error_tag: string ref, + string error_message: string ref, + string full_error_message: string ref, + int location: @location_default ref ); -/* - * External artifacts - */ +case @diagnostic.severity of + 10 = @diagnostic_debug +| 20 = @diagnostic_info +| 30 = @diagnostic_warning +| 40 = @diagnostic_error +; + +/*- External data -*/ +/** + * External data, loaded from CSV files during snapshot creation. See + * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data) + * for more information. + */ externalData( int id : @externalDataElement, string path : string ref, @@ -202,17 +215,18 @@ externalData( string value : string ref ); -snapshotDate( - unique date snapshotDate : date ref -); +/*- Snapshot date -*/ -sourceLocationPrefix( - string prefix : string ref -); +snapshotDate(unique date snapshotDate : date ref); -/* - * Duplicate code +/*- Source location prefix -*/ + +/** + * The source location of the snapshot. */ +sourceLocationPrefix(string prefix : string ref); + +/*- Duplicate code -*/ duplicateCode( unique int id : @duplication, @@ -237,6 +251,8 @@ tokens( int endColumn : int ref ); +/*- SMAP -*/ + /* * SMAP */ @@ -265,36 +281,25 @@ smap_lines( int outputLineIncrement: int ref ); -/* - * Locations and files - */ - -@location = @location_default ; +/*- Files and folders -*/ +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ locations_default( + /** The location of an element that is not an expression or a statement. */ unique int id: @location_default, - int file: @file ref, - int beginLine: int ref, - int beginColumn: int ref, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, int endLine: int ref, int endColumn: int ref ); -hasLocation( - int locatableid: @locatable ref, - int id: @location ref -); - -@sourceline = @locatable ; - -#keyset[element_id] -numlines( - int element_id: @sourceline ref, - int num_lines: int ref, - int num_code: int ref, - int num_comment: int ref -); - files( unique int id: @file, string name: string ref @@ -305,16 +310,33 @@ folders( string name: string ref ); -@container = @folder | @file +@container = @file | @folder containerparent( int parent: @container ref, unique int child: @container ref ); -/* - * Java - */ +/*- Lines of code -*/ + +#keyset[element_id] +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); + +/*- Java -*/ + +@location = @location_default ; + +hasLocation( + int locatableid: @locatable ref, + int id: @location ref +); + +@sourceline = @locatable ; cupackage( unique int id: @file ref, @@ -1060,9 +1082,7 @@ javadocText( @top = @element | @locatable | @folder; -/* - * XML Files - */ +/*- XML Files -*/ xmlEncoding( unique int id: @file ref, @@ -1133,9 +1153,7 @@ xmllocations( @xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace; -/* - * configuration files with key value pairs - */ +/*- configuration files with key value pairs -*/ configs( unique int id: @config @@ -1160,6 +1178,8 @@ configLocations( @configLocatable = @config | @configName | @configValue; +/*- Kotlin -*/ + ktComments( unique int id: @ktcomment, int kind: int ref, diff --git a/javascript/ql/lib/semmlecode.javascript.dbscheme b/javascript/ql/lib/semmlecode.javascript.dbscheme index 4d00210ca570..df77779bec97 100644 --- a/javascript/ql/lib/semmlecode.javascript.dbscheme +++ b/javascript/ql/lib/semmlecode.javascript.dbscheme @@ -1,51 +1,66 @@ /*** Standard fragments ***/ -/** Files and folders **/ +/*- Files and folders -*/ -@location = @location_default; - -locations_default(unique int id: @location_default, - int file: @file ref, - int beginLine: int ref, - int beginColumn: int ref, - int endLine: int ref, - int endColumn: int ref - ); - -@sourceline = @locatable; - -numlines(int element_id: @sourceline ref, - int num_lines: int ref, - int num_code: int ref, - int num_comment: int ref - ); +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_default( + /** The location of an element that is not an expression or a statement. */ + unique int id: @location_default, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); -files(unique int id: @file, - varchar(900) name: string ref); +files( + unique int id: @file, + string name: string ref +); -folders(unique int id: @folder, - varchar(900) name: string ref); +folders( + unique int id: @folder, + string name: string ref +); +@container = @file | @folder -@container = @folder | @file ; +containerparent( + int parent: @container ref, + unique int child: @container ref +); +/*- Lines of code -*/ -containerparent(int parent: @container ref, - unique int child: @container ref); +#keyset[element_id] +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); -/** Duplicate code **/ +/*- Duplicate code -*/ duplicateCode( unique int id : @duplication, - varchar(900) relativePath : string ref, - int equivClass : int ref); + string relativePath : string ref, + int equivClass : int ref +); similarCode( unique int id : @similarity, - varchar(900) relativePath : string ref, - int equivClass : int ref); + string relativePath : string ref, + int equivClass : int ref +); -@duplication_or_similarity = @duplication | @similarity; +@duplication_or_similarity = @duplication | @similarity tokens( int id : @duplication_or_similarity ref, @@ -53,51 +68,67 @@ tokens( int beginLine : int ref, int beginColumn : int ref, int endLine : int ref, - int endColumn : int ref); + int endColumn : int ref +); -/** External data **/ +/*- External data -*/ +/** + * External data, loaded from CSV files during snapshot creation. See + * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data) + * for more information. + */ externalData( int id : @externalDataElement, - varchar(900) path : string ref, + string path : string ref, int column: int ref, - varchar(900) value : string ref + string value : string ref ); +/*- Snapshot date -*/ + snapshotDate(unique date snapshotDate : date ref); -sourceLocationPrefix(varchar(900) prefix : string ref); +/*- Source location prefix -*/ -/** Version control data **/ +/** + * The source location of the snapshot. + */ +sourceLocationPrefix(string prefix : string ref); + +/*- Version control data -*/ svnentries( - int id : @svnentry, - varchar(500) revision : string ref, - varchar(500) author : string ref, + unique int id : @svnentry, + string revision : string ref, + string author : string ref, date revisionDate : date ref, int changeSize : int ref -); +) svnaffectedfiles( int id : @svnentry ref, int file : @file ref, - varchar(500) action : string ref -); + string action : string ref +) svnentrymsg( - int id : @svnentry ref, - varchar(500) message : string ref -); + unique int id : @svnentry ref, + string message : string ref +) svnchurn( int commit : @svnentry ref, int file : @file ref, int addedLines : int ref, int deletedLines : int ref -); +) +/*- JavaScript-specific part -*/ -/*** JavaScript-specific part ***/ +@location = @location_default + +@sourceline = @locatable; filetype( int file: @file ref, @@ -1046,7 +1077,43 @@ jsdoc_has_new_parameter (int fn: @jsdoc_function_type_expr ref); jsdoc_errors (unique int id: @jsdoc_error, int tag: @jsdoc_tag ref, varchar(900) message: string ref, varchar(900) tostring: string ref); -// YAML +@dataflownode = @expr | @function_decl_stmt | @class_decl_stmt | @namespace_declaration | @enum_declaration | @property; + +@optionalchainable = @call_expr | @propaccess; + +isOptionalChaining(int id: @optionalchainable ref); + +/** + * The time taken for the extraction of a file. + * This table contains non-deterministic content. + * + * The sum of the `time` column for each (`file`, `timerKind`) pair + * is the total time taken for extraction of `file`. The `extractionPhase` + * column provides a granular view of the extraction time of the file. + */ +extraction_time( + int file : @file ref, + // see `com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase`. + int extractionPhase: int ref, + // 0 for the elapsed CPU time in nanoseconds, 1 for the elapsed wallclock time in nanoseconds + int timerKind: int ref, + float time: float ref +) + +/** +* Non-timing related data for the extraction of a single file. +* This table contains non-deterministic content. +*/ +extraction_data( + int file : @file ref, + // the absolute path to the cache file + varchar(900) cacheFile: string ref, + boolean fromCache: boolean ref, + int length: int ref +) + +/*- YAML -*/ + #keyset[parent, idx] yaml (unique int id: @yaml_node, int kind: int ref, @@ -1084,24 +1151,24 @@ yaml_locations(unique int locatable: @yaml_locatable ref, @yaml_locatable = @yaml_node | @yaml_error; -/* XML Files */ +/*- XML Files -*/ xmlEncoding( unique int id: @file ref, - varchar(900) encoding: string ref + string encoding: string ref ); xmlDTDs( unique int id: @xmldtd, - varchar(900) root: string ref, - varchar(900) publicId: string ref, - varchar(900) systemId: string ref, + string root: string ref, + string publicId: string ref, + string systemId: string ref, int fileid: @file ref ); xmlElements( unique int id: @xmlelement, - varchar(900) name: string ref, + string name: string ref, int parentid: @xmlparent ref, int idx: int ref, int fileid: @file ref @@ -1110,16 +1177,16 @@ xmlElements( xmlAttrs( unique int id: @xmlattribute, int elementid: @xmlelement ref, - varchar(900) name: string ref, - varchar(3600) value: string ref, + string name: string ref, + string value: string ref, int idx: int ref, int fileid: @file ref ); xmlNs( int id: @xmlnamespace, - varchar(900) prefixName: string ref, - varchar(900) URI: string ref, + string prefixName: string ref, + string URI: string ref, int fileid: @file ref ); @@ -1131,14 +1198,14 @@ xmlHasNs( xmlComments( unique int id: @xmlcomment, - varchar(3600) text: string ref, + string text: string ref, int parentid: @xmlparent ref, int fileid: @file ref ); xmlChars( unique int id: @xmlcharacters, - varchar(3600) text: string ref, + string text: string ref, int parentid: @xmlparent ref, int idx: int ref, int isCDATA: int ref, @@ -1155,15 +1222,7 @@ xmllocations( @xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace; -@dataflownode = @expr | @function_decl_stmt | @class_decl_stmt | @namespace_declaration | @enum_declaration | @property; - -@optionalchainable = @call_expr | @propaccess; - -isOptionalChaining(int id: @optionalchainable ref); - -/* - * configuration files with key value pairs - */ +/*- configuration files with key value pairs -*/ configs( unique int id: @config @@ -1188,31 +1247,3 @@ configLocations( @configLocatable = @config | @configName | @configValue; -/** - * The time taken for the extraction of a file. - * This table contains non-deterministic content. - * - * The sum of the `time` column for each (`file`, `timerKind`) pair - * is the total time taken for extraction of `file`. The `extractionPhase` - * column provides a granular view of the extraction time of the file. - */ -extraction_time( - int file : @file ref, - // see `com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase`. - int extractionPhase: int ref, - // 0 for the elapsed CPU time in nanoseconds, 1 for the elapsed wallclock time in nanoseconds - int timerKind: int ref, - float time: float ref -) - -/** - * Non-timing related data for the extraction of a single file. - * This table contains non-deterministic content. - */ -extraction_data( - int file : @file ref, - // the absolute path to the cache file - varchar(900) cacheFile: string ref, - boolean fromCache: boolean ref, - int length: int ref -) diff --git a/python/ql/lib/semmlecode.python.dbscheme b/python/ql/lib/semmlecode.python.dbscheme index 47e552c4357a..8b0d2fe30b7c 100644 --- a/python/ql/lib/semmlecode.python.dbscheme +++ b/python/ql/lib/semmlecode.python.dbscheme @@ -16,9 +16,7 @@ * mechanism not work properly. */ - /* - * External artifacts - */ +/*- External defects and metrics -*/ externalDefects( unique int id : @externalDefect, @@ -35,31 +33,44 @@ externalMetrics( float value : float ref ); +/*- External data -*/ + +/** + * External data, loaded from CSV files during snapshot creation. See + * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data) + * for more information. + */ externalData( int id : @externalDataElement, - varchar(900) queryPath : string ref, + string path : string ref, int column: int ref, - varchar(900) data : string ref + string value : string ref ); -snapshotDate(unique date snapshotDate : date ref); +/*- Snapshot date -*/ -sourceLocationPrefix(varchar(900) prefix : string ref); +snapshotDate(unique date snapshotDate : date ref); +/*- Source location prefix -*/ -/* - * Duplicate code +/** + * The source location of the snapshot. */ +sourceLocationPrefix(string prefix : string ref); + +/*- Duplicate code -*/ duplicateCode( unique int id : @duplication, - varchar(900) relativePath : string ref, - int equivClass : int ref); + string relativePath : string ref, + int equivClass : int ref +); similarCode( unique int id : @similarity, - varchar(900) relativePath : string ref, - int equivClass : int ref); + string relativePath : string ref, + int equivClass : int ref +); @duplication_or_similarity = @duplication | @similarity @@ -69,31 +80,15 @@ tokens( int beginLine : int ref, int beginColumn : int ref, int endLine : int ref, - int endColumn : int ref); - -/* - * Line metrics - */ -py_codelines(int id : @py_scope ref, - int count : int ref); - -py_commentlines(int id : @py_scope ref, - int count : int ref); - -py_docstringlines(int id : @py_scope ref, - int count : int ref); + int endColumn : int ref +); -py_alllines(int id : @py_scope ref, - int count : int ref); - -/* - * Version history - */ +/*- Version control data -*/ svnentries( - int id : @svnentry, - varchar(500) revision : string ref, - varchar(500) author : string ref, + unique int id : @svnentry, + string revision : string ref, + string author : string ref, date revisionDate : date ref, int changeSize : int ref ) @@ -101,12 +96,12 @@ svnentries( svnaffectedfiles( int id : @svnentry ref, int file : @file ref, - varchar(500) action : string ref + string action : string ref ) svnentrymsg( - int id : @svnentry ref, - varchar(500) message : string ref + unique int id : @svnentry ref, + string message : string ref ) svnchurn( @@ -116,37 +111,72 @@ svnchurn( int deletedLines : int ref ) -/**************************** - Python dbscheme -****************************/ +/*- Lines of code -*/ -files(unique int id: @file, - varchar(900) name: string ref); +#keyset[element_id] +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); -folders(unique int id: @folder, - varchar(900) name: string ref); +/*- Files and folders -*/ -@container = @folder | @file; +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_default( + /** The location of an element that is not an expression or a statement. */ + unique int id: @location_default, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); -containerparent(int parent: @container ref, - unique int child: @container ref); +files( + unique int id: @file, + string name: string ref +); -@sourceline = @file | @py_Module | @xmllocatable; +folders( + unique int id: @folder, + string name: string ref +); -numlines(int element_id: @sourceline ref, - int num_lines: int ref, - int num_code: int ref, - int num_comment: int ref - ); +@container = @file | @folder -@location = @location_ast | @location_default ; +containerparent( + int parent: @container ref, + unique int child: @container ref +); -locations_default(unique int id: @location_default, - int file: @file ref, - int beginLine: int ref, - int beginColumn: int ref, - int endLine: int ref, - int endColumn: int ref); +/*- Python dbscheme -*/ + +/* + * Line metrics + */ +py_codelines(int id : @py_scope ref, + int count : int ref); + +py_commentlines(int id : @py_scope ref, + int count : int ref); + +py_docstringlines(int id : @py_scope ref, + int count : int ref); + +py_alllines(int id : @py_scope ref, + int count : int ref); + +@sourceline = @file | @py_Module | @xmllocatable; + +@location = @location_ast | @location_default ; locations_ast(unique int id: @location_ast, int module: @py_Module ref, @@ -1053,54 +1083,74 @@ py_decorated_object(int object : @py_object ref, @py_source_element = @py_ast_node | @container; -/* XML Files */ - -xmlEncoding (unique int id: @file ref, varchar(900) encoding: string ref); - -xmlDTDs (unique int id: @xmldtd, - varchar(900) root: string ref, - varchar(900) publicId: string ref, - varchar(900) systemId: string ref, - int fileid: @file ref); - -xmlElements (unique int id: @xmlelement, - varchar(900) name: string ref, - int parentid: @xmlparent ref, - int idx: int ref, - int fileid: @file ref); - -xmlAttrs (unique int id: @xmlattribute, - int elementid: @xmlelement ref, - varchar(900) name: string ref, - varchar(3600) value: string ref, - int idx: int ref, - int fileid: @file ref); - -xmlNs (int id: @xmlnamespace, - varchar(900) prefixName: string ref, - varchar(900) URI: string ref, - int fileid: @file ref); - -xmlHasNs (int elementId: @xmlnamespaceable ref, - int nsId: @xmlnamespace ref, - int fileid: @file ref); - -xmlComments (unique int id: @xmlcomment, - varchar(3600) text: string ref, - int parentid: @xmlparent ref, - int fileid: @file ref); - -xmlChars (unique int id: @xmlcharacters, - varchar(3600) text: string ref, - int parentid: @xmlparent ref, - int idx: int ref, - int isCDATA: int ref, - int fileid: @file ref); +/*- XML Files -*/ + +xmlEncoding( + unique int id: @file ref, + string encoding: string ref +); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); @xmlparent = @file | @xmlelement; @xmlnamespaceable = @xmlelement | @xmlattribute; -xmllocations(int xmlElement: @xmllocatable ref, - int location: @location_default ref); +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); @xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace; + diff --git a/ql/ql/src/ql.dbscheme b/ql/ql/src/ql.dbscheme index 2f4f6f7d26ff..47a06637005e 100644 --- a/ql/ql/src/ql.dbscheme +++ b/ql/ql/src/ql.dbscheme @@ -1,15 +1,23 @@ // CodeQL database schema for QL // Automatically generated from the tree-sitter grammar; do not edit -@location = @location_default - +/*- Files and folders -*/ + +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ locations_default( + /** The location of an element that is not an expression or a statement. */ unique int id: @location_default, - int file: @file ref, - int start_line: int ref, - int start_column: int ref, - int end_line: int ref, - int end_column: int ref + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref ); files( @@ -29,9 +37,14 @@ containerparent( unique int child: @container ref ); -sourceLocationPrefix( - string prefix: string ref -); +/*- Source location prefix -*/ + +/** + * The source location of the snapshot. + */ +sourceLocationPrefix(string prefix : string ref); + +/*- Diagnostic messages -*/ diagnostics( unique int id: @diagnostic, @@ -49,6 +62,9 @@ case @diagnostic.severity of | 40 = @diagnostic_error ; +/*- QL-specific part -*/ + +@location = @location_default @ql_add_expr_left_type = @ql_add_expr | @ql_aggregate | @ql_call_or_unqual_agg_expr | @ql_comp_term | @ql_conjunction | @ql_disjunction | @ql_expr_annotation | @ql_if_term | @ql_implication | @ql_in_expr | @ql_instance_of | @ql_literal | @ql_mul_expr | @ql_negation | @ql_par_expr | @ql_prefix_cast | @ql_qualified_expr | @ql_quantified | @ql_range | @ql_set_literal | @ql_special_call | @ql_super_ref | @ql_unary_expr | @ql_variable diff --git a/ruby/ql/lib/ruby.dbscheme b/ruby/ql/lib/ruby.dbscheme index ff289788b155..3588c314cc48 100644 --- a/ruby/ql/lib/ruby.dbscheme +++ b/ruby/ql/lib/ruby.dbscheme @@ -1,15 +1,23 @@ // CodeQL database schema for Ruby // Automatically generated from the tree-sitter grammar; do not edit -@location = @location_default - +/*- Files and folders -*/ + +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ locations_default( + /** The location of an element that is not an expression or a statement. */ unique int id: @location_default, - int file: @file ref, - int start_line: int ref, - int start_column: int ref, - int end_line: int ref, - int end_column: int ref + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref ); files( @@ -29,9 +37,14 @@ containerparent( unique int child: @container ref ); -sourceLocationPrefix( - string prefix: string ref -); +/*- Source location prefix -*/ + +/** + * The source location of the snapshot. + */ +sourceLocationPrefix(string prefix : string ref); + +/*- Diagnostic messages -*/ diagnostics( unique int id: @diagnostic, @@ -49,6 +62,9 @@ case @diagnostic.severity of | 40 = @diagnostic_error ; +/*- Ruby-specific part -*/ + +@location = @location_default @ruby_underscore_arg = @ruby_assignment | @ruby_binary | @ruby_conditional | @ruby_operator_assignment | @ruby_range | @ruby_unary | @ruby_underscore_primary @@ -1378,6 +1394,8 @@ ruby_ast_node_info( int loc: @location ref ); +/*- ERB-specific part -*/ + erb_comment_directive_child( unique int erb_comment_directive: @erb_comment_directive ref, unique int child: @erb_token_comment ref diff --git a/swift/prefix.dbscheme b/swift/prefix.dbscheme index b422464bb902..04c524176d90 100644 --- a/swift/prefix.dbscheme +++ b/swift/prefix.dbscheme @@ -1,6 +1,7 @@ +/*- Source location prefix -*/ + /** * The source location of the snapshot. */ -sourceLocationPrefix( - string prefix: string ref -); +sourceLocationPrefix(string prefix : string ref); + diff --git a/swift/ql/lib/swift.dbscheme b/swift/ql/lib/swift.dbscheme index e457bc35325b..d3b2b010500e 100644 --- a/swift/ql/lib/swift.dbscheme +++ b/swift/ql/lib/swift.dbscheme @@ -1,14 +1,11 @@ -// generated by codegen/codegen.py +/*- Source location prefix -*/ -// from prefix.dbscheme /** * The source location of the snapshot. */ -sourceLocationPrefix( - string prefix: string ref -); - +sourceLocationPrefix(string prefix : string ref); +/*- Swift-specific part -*/ // from schema.py @element =