Unified extractor: add AST schema, swift translation rules, and corpus framework

Add ast_types.yml defining the unified output AST schema with supertypes
(expr, stmt, condition, pattern) and named nodes (top_level, binary_expr,
name_expr, etc.).

Rewrite swift translation rules to map from tree-sitter Swift grammar to
the unified AST, using one-shot phase rules.

Update the generator to use the output AST schema for dbscheme/QL
generation, and normalize the extraction table prefix to 'unified'.

Improve the corpus test framework to include raw tree-sitter parse output,
type-error checking against the output schema, and better failure
reporting.

Regenerate Ast.qll, unified.dbscheme, and update BasicTest accordingly.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: asgerf

unified/extractor/ast_types.yml

@@ -0,0 +1,102 @@
+supertypes:
+  expr:
+    - binary_expr
+    - unary_expr
+    - name_expr
+    - lambda_expr
+    - unsupported_node
+  stmt:
+    - empty_stmt
+    - if_stmt
+    - variable_declaration_stmt
+    - guard_if_stmt
+    - unsupported_node
+  condition:
+    - expr_condition
+    - let_pattern_condition
+    - unsupported_node
+  pattern:
+    - var_pattern
+    - apply_pattern
+    - ignore_pattern
+    - unsupported_node
+named:
+  # Top-level is the root node, currently containing a list of expressions
+  top_level:
+    body*: expr
+
+  # Application of a binary operator, such as `a + b`
+  binary_expr:
+    left: expr
+    operator: operator
+    right: expr
+
+  # Application of a unary operator, such as `!x`
+  unary_expr:
+    operand: expr
+    operator: operator
+
+  # An identifier used in the context of an expression
+  name_expr:
+    identifier: identifier
+
+  lambda_expr:
+    parameter*: parameter
+    body: [expr, stmt]
+
+  # A parameter
+  parameter:
+    pattern: pattern
+
+  empty_stmt:
+
+  if_stmt:
+    condition: condition
+    then?: stmt
+    else?: stmt
+
+  variable_declaration_stmt:
+    variable_declarator+: variable_declarator
+
+  # A variable declaration, or assignment to a pattern.
+  # The initializer is optional (but typically only possible in combination with a simple variable pattern).
+  variable_declarator:
+    pattern: pattern
+    value?: expr
+
+  # Evaluate 'condition', and if false, execute 'else' which must break from the enclosing block scope (return, break, etc).
+  # Any variables bound by 'condition' will be in scope for the remainder of the enclosing block scope
+  # (which differs from how if_stmt works).
+  guard_if_stmt:
+    condition: condition
+    else: stmt
+
+  # Evaluates the given condition and interprets it as a boolean (by language conventions)
+  expr_condition:
+    expr: expr
+
+  # Evaluate 'expr' and match its result against 'pattern', and return true if it matches.
+  # Variables bound by the pattern will be in scope within the 'true' branch controlled by this condition.
+  let_pattern_condition:
+    pattern: pattern
+    value: expr
+
+  # A pattern matching anything, binding its value to the given variable
+  var_pattern:
+    identifier: identifier
+
+  # A pattern matching anything, binding no variables, usually using the syntax "_"
+  ignore_pattern:
+
+  # A pattern such as `Some(x)` where `Some` is the constructor and `x` is an argument
+  apply_pattern:
+    constructor: expr
+    argument*: expr
+
+  # An simple unqualified identifier token
+  identifier:
+
+  # A node that we don't yet translate
+  unsupported_node:
+
+  operator:

unified/extractor/src/extractor.rs

@@ -23,9 +23,17 @@ pub struct Options {
 pub fn run(options: Options) -> std::io::Result<()> {
     codeql_extractor::extractor::set_tracing_level("unified");
 
+    // The generated dbscheme/QL library uses the unified_* relation namespace.
+    // Keep per-language specs for parser/rules/file globs, but normalize the
+    // extraction table prefix so emitted TRAP relations match the dbscheme.
+    let mut languages = languages::all_language_specs();
+    for lang in &mut languages {
+        lang.prefix = "unified";
+    }
+
     let extractor = simple::Extractor {
         prefix: "unified".to_string(),
-        languages: languages::all_language_specs(),
+        languages,
         trap_dir: options.output_dir,
         trap_compression: trap::Compression::from_env("CODEQL_EXTRACTOR_UNIFIED_OPTION_TRAP_COMPRESSION"),
         source_archive_dir: options.source_archive_dir,

unified/extractor/src/generator.rs

@@ -3,6 +3,8 @@ use std::path::PathBuf;
 
 use codeql_extractor::generator::{generate, language::Language};
 
+use crate::languages;
+
 #[derive(Args)]
 pub struct Options {
     /// Path of the generated dbscheme file
@@ -17,10 +19,16 @@ pub struct Options {
 pub fn run(options: Options) -> std::io::Result<()> {
     codeql_extractor::extractor::set_tracing_level("unified");
 
+    // The QL-visible schema is the unified output AST, not the per-language
+    // input grammars. Pass it via `desugar.output_node_types_yaml` so the
+    // generator converts the YAML to JSON node-types.
+    let desugar = yeast::DesugaringConfig::new()
+        .with_output_node_types_yaml(languages::OUTPUT_AST_SCHEMA);
+
     let languages = vec![Language {
-        name: "Swift".to_owned(),
-        node_types: tree_sitter_swift::NODE_TYPES,
-        desugar: None,
+        name: "Unified".to_owned(),
+        node_types: "",   // unused: generator picks up output_node_types_yaml above
+        desugar: Some(desugar),
     }];
 
     generate(languages, options.dbscheme, options.library, "run unified/scripts/create-extractor-pack.sh")

unified/extractor/src/languages/mod.rs

@@ -3,6 +3,9 @@ use codeql_extractor::extractor::simple;
 #[path = "swift/swift.rs"]
 mod swift;
 
+/// Shared YEAST output AST schema for all languages.
+pub(crate) const OUTPUT_AST_SCHEMA: &str = include_str!("../../ast_types.yml");
+
 pub fn all_language_specs() -> Vec<simple::LanguageSpec> {
-    vec![swift::language_spec()]
+    vec![swift::language_spec(OUTPUT_AST_SCHEMA)]
 }

unified/extractor/src/languages/swift/swift.rs

@@ -1,18 +1,59 @@
 use codeql_extractor::extractor::simple;
 use yeast::{rule, DesugaringConfig, PhaseKind};
 
-fn desugaring_rules() -> Vec<yeast::Rule> {
+fn translation_rules() -> Vec<yeast::Rule> {
     vec![
         rule!(
-            (additive_expression)
+            (source_file (_)* @children)
             =>
-            (simple_identifier "blah")
+            (top_level
+                body: {..children}
+            )
+        ),
+        rule!(
+            (additive_expression
+                lhs: (_) @left
+                op: _ @operator
+                rhs: (_) @right)
+            =>
+            (binary_expr
+                left: {left}
+                operator: (operator #{operator})
+                right: {right})
+        ),
+        rule!(
+            (multiplicative_expression
+                lhs: (_) @left
+                op: _ @operator
+                rhs: (_) @right)
+            =>
+            (binary_expr
+                left: {left}
+                operator: (operator #{operator})
+                right: {right})
+        ),
+        rule!(
+            (simple_identifier)
+            =>
+            name_expr
+        ),
+        rule!(
+            (_)
+            =>
+            (unsupported_node)
+        ),
+        rule!(
+            _ @node
+            =>
+            {node}
         ),
     ]
 }
 
-pub fn language_spec() -> simple::LanguageSpec {
-    let desugar = DesugaringConfig::new().add_phase("desugar", PhaseKind::Repeating, desugaring_rules());
+pub fn language_spec(desugared_ast_schema: &'static str) -> simple::LanguageSpec {
+    let desugar = DesugaringConfig::new()
+        .add_phase("translate", PhaseKind::OneShot, translation_rules())
+        .with_output_node_types_yaml(desugared_ast_schema);
     simple::LanguageSpec {
         prefix: "swift",
         ts_language: tree_sitter_swift::LANGUAGE.into(),