source side validation

Author: matt-beanland

.formatter.exs

@@ -15,6 +15,7 @@ spark_locals_without_parens = [
   feature: 1,
   feature: 2,
   id: 1,
+  instance_ref: 2,
   is_enabled?: 1,
   major_version: 1,
   minor_version: 1,
@@ -23,14 +24,21 @@ spark_locals_without_parens = [
   parties: 3,
   party: 2,
   party: 3,
+  party_ref: 2,
+  party_ref: 3,
   patch_version: 1,
   place: 2,
   place: 3,
+  place_ref: 2,
+  place_ref: 3,
   places: 2,
   places: 3,
+  pool: 2,
   reference: 1,
   role: 2,
   role: 3,
+  source: 1,
+  target: 1,
   tmf_version: 1,
   type: 1,
   update: 1,

AGENTS.md

@@ -41,9 +41,14 @@ lib/diffo/provider/
     place_declaration.ex        # PlaceDeclaration struct
     party_role.ex               # PartyRole struct (Party/Place kinds)
     place_role.ex               # PlaceRole struct (Party/Place kinds)
-    persisters/                 # Spark transformers — bake DSL state into module
-    transformers/               # TransformBehaviour — action argument injection
-    verifiers/                  # Compile-time DSL correctness checks
+    relationship_step.ex        # RelationshipStep struct — pipeline step for relationships do
+    persisters/                 # Terminal bakers — run after all transformers; only read DSL state and bake module functions
+    transformers/
+      transform_relationships.ex  # TransformRelationships — resolves relationships pipeline, bakes permitted_source_roles/0 and permitted_target_roles/0
+    verifiers/
+      verify_relationships.ex     # Verifies relationship role declarations are atoms
+  changes/
+    validate_relationship_permitted.ex  # ValidateRelationshipPermitted — enforces relationships do policy on relate actions
   assigner/
     assigner.ex                 # Diffo.Provider.Assigner — assign/3 (pools do) and assign/4
     assignable_characteristic.ex # AssignableCharacteristic — pool bounds + algorithm
@@ -72,6 +77,7 @@ test/provider/extension/        # All provider extension tests
   instance_verifier_test.exs
   party_verifier_test.exs
   place_verifier_test.exs
+  relationship_dsl_test.exs     # Transformer baking, verifier errors, integration enforcement
   party_test.exs                # Integration: parties enforcement
   place_test.exs                # Integration: places enforcement
   specification_test.exs        # Integration: spec roundtrip
@@ -112,6 +118,11 @@ provider do
     pool :vlans, :vlan
   end
 
+  relationships do
+    source [:provides, :requires]   # pipeline — last step wins; omitting defaults to :none
+    target :all
+  end
+
   features do
     feature :advanced_routing, is_enabled?: false do
       characteristic :policy, MyApp.RoutingPolicy
@@ -179,6 +190,22 @@ whose names end in the same word get the same label, which causes read collision
 E.g. `Diffo.Test.Instance.CardInstance` → label `:CardInstance`,
 and `Diffo.Test.Characteristic.CardCharacteristic` → label `:CardCharacteristic` — no collision.
 
+## Spark transformer vs persister pipeline
+
+Spark runs two separate pipelines during compilation, in this order:
+
+1. **Transformers** (`transformers:` in the extension) — run in dependency order via `before?`/`after?`. Can read and modify DSL state. May also call `Transformer.persist/3` to bake results — a transformer that had to compute something to do its job should persist that result rather than delegating to a separate persister.
+2. **Persisters** (`persisters:` in the extension) — always run after ALL transformers from ALL extensions. `before?`/`after?` ordering works relative to other persisters only — ordering declarations targeting transformers are silently ignored.
+3. **Verifiers** — read-only, run last.
+
+**Rules:**
+- A module that injects into actions, modifies DSL state, or needs to order itself relative to Ash's own transformers belongs in `transformers:`.
+- A module that only reads final DSL state and bakes module functions belongs in `persisters:`.
+- A transformer that needs to expose baked state does not need a separate persister — call `Transformer.persist/3` inline and emit the module function via `Transformer.eval/3`.
+- Do not put a transformer in `persisters:` hoping `after?` declarations will order it relative to transformers — those declarations are silently ignored across pipeline boundaries.
+
+**Current state:** `TransformBehaviour` is misregistered under `persisters:` — a known issue tracked for refactoring. New transformers go under `transformers:`.
+
 ## Common agent mistakes
 
 - Using old `structure do` / top-level `instances do` — use `provider do` only.
@@ -188,12 +215,18 @@ and `Diffo.Test.Characteristic.CardCharacteristic` → label `:CardCharacteristi
 - Using the removed `AssignableValue` TypedStruct — it no longer exists; use `pools do`.
 - Calling `Assigner.assign/4` when a `pools do` declaration exists — prefer `Assigner.assign/3` which looks up the thing automatically.
 - Forgetting to call `Pool.update_pools/3` in `:define` actions when the resource has `pools do` — pool bounds (`first`, `last`, `algorithm`) are set here.
-- Using `characteristic :pool_name, Diffo.Provider.AssignedToRelationship` — `AssignedToRelationship` is not a characteristic; use `pools do / pool :name, :thing / end` instead.
-- Querying `Diffo.Provider.Relationship` for assignment records — assignment relationships are on `Diffo.Provider.AssignedToRelationship`; access them via `instance.assignments`.
+- Using `characteristic :pool_name, Diffo.Provider.AssignedToRelationship` — `AssignedToRelationship` no longer exists; use `pools do / pool :name, :thing / end` instead.
+- Querying `Diffo.Provider.Relationship` for assignment records — assignments are stored as `Diffo.Provider.DefinedSimpleRelationship`; access them via `instance.assignments`.
 - Filtering `instance.forward_relationships` for `type == :assignedTo` — those records no longer exist there; use `instance.assignments` directly.
 - Calling `build_before/1` or `build_after/2` in actions — these run automatically.
 - Declaring `:specified_by`, `:features`, `:characteristics` as action arguments.
+- Using module names (e.g. `MyApp.CardInstance`) as role values in `relationships do` — roles are atoms like `:provides`, not module references.
+- Forgetting that `relationships do` omitted means `:none` for both source and target — any update action with `argument :relationships, {:array, :struct}` will fail unless the resource declares permissions.
+- Thinking the Assigner requires `relationships do` permissions — it does not. The Assigner writes `DefinedSimpleRelationship` records directly via the Provider domain; `ValidateRelationshipPermitted` only runs on actions that carry `argument :relationships, {:array, :struct}`, which the Assigner's `assign_*` actions do not.
 - Editing `documentation/dsls/DSL-Diffo.Provider.Extension.md` — it is Spark-generated;
-  run `mix spark.cheat_sheets` to regenerate it.
+  run `mix spark.cheat_sheets` to regenerate it. Whenever you add, rename, or remove a DSL
+  entity or section, also check `.formatter.exs` — new entity names must be added to
+  `spark_locals_without_parens` (with each arity) so the Spark formatter omits parentheses.
+  Run `mix format` afterward to verify.
 - Editing content between `<!-- usage-rules-start -->` markers in `CLAUDE.md` — that is
   auto-generated by `mix usage_rules.sync`.

documentation/dsls/DSL-Diffo.Provider.Extension.md

@@ -111,6 +111,9 @@ Provider DSL — structure, roles, and behaviour for this resource kind
  * [instances](#provider-instances)
    * role
    * instance_ref
+ * [relationships](#provider-relationships)
+   * source
+   * target
  * [behaviour](#provider-behaviour)
    * actions
      * create
@@ -712,6 +715,75 @@ Declares a reference instance role — no direct edge created, reachable by grap
 Target: `Diffo.Provider.Extension.InstanceRole`
 
 
+### provider.relationships
+Relationship role permissions for this Instance — declares which aliases it may participate in as source or target. Omitting defaults to `:none` per direction.
+
+### Nested DSLs
+ * [source](#provider-relationships-source)
+ * [target](#provider-relationships-target)
+
+
+### Examples
+```
+relationships do
+  source [:provides, :requires]
+  target :all
+end
+
+```
+
+
+
+
+### provider.relationships.source
+```elixir
+source roles
+```
+
+
+Declares permitted source relationship roles — pipeline step, last declaration wins
+
+
+
+
+
+### Arguments
+
+| Name | Type | Default | Docs |
+|------|------|---------|------|
+| [`roles`](#provider-relationships-source-roles){: #provider-relationships-source-roles .spark-required} | `any` |  | `:all`, `:none`, or a list of role name atoms. |
+
+
+
+
+
+
+
+### provider.relationships.target
+```elixir
+target roles
+```
+
+
+Declares permitted target relationship roles — pipeline step, last declaration wins
+
+
+
+
+
+### Arguments
+
+| Name | Type | Default | Docs |
+|------|------|---------|------|
+| [`roles`](#provider-relationships-target-roles){: #provider-relationships-target-roles .spark-required} | `any` |  | `:all`, `:none`, or a list of role name atoms. |
+
+
+
+
+
+
+
+
 ### provider.behaviour
 Defines the behavioural wiring for the Instance — actions, and in future triggers
 

documentation/how_to/use_diffo_provider_extension.livemd

@@ -153,7 +153,7 @@ arguments automatically onto that action.
 
 Each characteristic is a dedicated Ash resource using the `Diffo.Provider.BaseCharacteristic` fragment. It carries direct typed attributes and a `:value` calculation that builds a companion `<Module>.Value` TypedStruct for ordered JSON encoding. The TypedStruct uses [AshJason.TypedStruct](https://hexdocs.pm/ash_jason/) to control field order in the JSON output.
 
-For partial resource allocation and assignment we've created `Diffo.Provider.Assigner`. The host resource declares a `pools do` section listing each assignable pool by name and the kind of thing being assigned. Pool bounds (first/last value, algorithm) are set via a `:define` action. Each assignment is stored as a `Diffo.Provider.AssignedToRelationship` node (Neo4j label `:AssignmentRelationship`) carrying `pool`, `thing`, and the `assigned` value. These are distinct from regular TMF `Diffo.Provider.Relationship` nodes and are accessible on an instance via `instance.assignments`.
+For partial resource allocation and assignment we've created `Diffo.Provider.Assigner`. The host resource declares a `pools do` section listing each assignable pool by name and the kind of thing being assigned. Pool bounds (first/last value, algorithm) are set via a `:define` action. Each assignment is stored as a `Diffo.Provider.DefinedSimpleRelationship` node carrying `type: :assignedTo` and a single `NameValuePrimitive` characteristic holding the thing name and assigned value. These are distinct from regular TMF `Diffo.Provider.Relationship` nodes and are accessible on an instance via `instance.assignments`.
 
 Let's imagine a Compute domain which operates GPU and NPU resources. We want to expose a Cluster composite resource which can be dynamically composed of a number of GPU and NPU cores.
 
@@ -278,6 +278,11 @@ defmodule Diffo.Compute.Cluster do
       place :data_centre, Diffo.Compute.DataCentre
     end
 
+    relationships do
+      source :all
+      target :all
+    end
+
     behaviour do
       actions do
         create :build
@@ -426,6 +431,11 @@ defmodule Diffo.Compute.GPU do
       pool :cores, :core
     end
 
+    relationships do
+      source :all
+      target :all
+    end
+
     behaviour do
       actions do
         create :build
@@ -783,16 +793,16 @@ gpu_1 = Compute.assign_gpu_core!(gpu_1, assignment)
 gpu_2 = Compute.assign_gpu_core!(gpu_2, assignment)
 ```
 
-Now our cluster should have a core from each GPU. Check in the Neo4j browser for `:AssignmentRelationship` nodes from `gpu_1` and `gpu_2` to `cluster_1`. There should be four — each carries `pool: :cores`, `thing: :core`, and the `assigned` integer value (e.g. 1, 2, 3 from gpu_1 and 1 from gpu_2).
+Now our cluster should have a core from each GPU. Check in the Neo4j browser for `:DefinedSimpleRelationship` nodes from `gpu_1` and `gpu_2` to `cluster_1`. There should be four — each has `type: :assignedTo` and a single characteristic carrying the thing name (`:core`) and the assigned integer value (e.g. 1, 2, 3 from gpu_1 and 1 from gpu_2).
 
 The GPU's `assignments` hold each assignment, showing the assigned core number in the JSON encoding as a `resourceRelationshipCharacteristic`:
 
 ```elixir
 Jason.encode!(gpu_1, pretty: true) |> IO.puts
 ```
 
-Make sure you have a look at it in the neo4j browser. There should be `:AssignmentRelationship` nodes from each GPU resource instance to the `cluster_1` resource instance, each carrying the assigned core number.
-There is no central assignment table — the `AssignedToRelationship` nodes ARE the assignments. They are separate from the regular `:Relationship` nodes used for TMF service/resource relationships, and are accessible in Elixir via `instance.assignments`.
+Make sure you have a look at it in the neo4j browser. There should be `:DefinedSimpleRelationship` nodes from each GPU resource instance to the `cluster_1` resource instance, each carrying the assigned core number.
+There is no central assignment table — the `DefinedSimpleRelationship` nodes ARE the assignments. They are separate from the regular `:Relationship` nodes used for TMF service/resource relationships, and are accessible in Elixir via `instance.assignments`.
 
 As an exercise, clone the GPU resource to create an NPU resource and assign some NPU cores from it to your cluster. Check that the assigned NPU cores are unique.
 
@@ -805,7 +815,7 @@ In this tutorial you've used Diffo's unified `provider do` extension to define a
 
 - A composite Cluster resource that receives GPU cores via `Diffo.Provider.Assigner`
 - A GPU resource using `pools do` to declare the `:cores` assignable pool — `pool :cores, :core` replaces the old `characteristic :cores, AssignableValue` pattern
-- Assignments stored on `Diffo.Provider.AssignedToRelationship` nodes (Neo4j label `:AssignmentRelationship`, distinct from TMF `:Relationship` nodes); accessible via `instance.assignments`
+- Assignments stored as `Diffo.Provider.DefinedSimpleRelationship` records with `type: :assignedTo` (distinct from TMF `Relationship` nodes); accessible via `instance.assignments`
 - `Tenant` and `Engineer` Party kinds declared with `provider do` that express which instances they operate and manage
 - A `DataCentre` Place kind that declares the instances located at it
 

lib/diffo/provider/assigner/assignable_characteristic.ex

@@ -20,6 +20,20 @@ defmodule Diffo.Provider.AssignableCharacteristic do
     plural_name :assignable_characteristics
   end
 
+  actions do
+    create :create do
+      accept [:name, :first, :last, :assignable_type, :algorithm, :thing]
+      argument :instance_id, :uuid
+      argument :feature_id, :uuid
+      change manage_relationship(:instance_id, :instance, type: :append)
+      change manage_relationship(:feature_id, :feature, type: :append)
+    end
+
+    update :update do
+      accept [:first, :last, :assignable_type, :algorithm]
+    end
+  end
+
   attributes do
     attribute :first, :integer do
       description "the first assignable value in the pool"
@@ -56,13 +70,13 @@ defmodule Diffo.Provider.AssignableCharacteristic do
   end
 
   calculations do
-    calculate :value, Diffo.Type.CharacteristicValue,
+    calculate :value,
+              Diffo.Type.CharacteristicValue,
               Diffo.Provider.Calculations.CharacteristicValue do
       public? true
     end
 
-    calculate :assigned_values, {:array, :integer},
-              Diffo.Provider.Calculations.AssignedValues do
+    calculate :assigned_values, {:array, :integer}, Diffo.Provider.Calculations.AssignedValues do
       public? true
       argument :thing, :atom, allow_nil?: false
     end
@@ -72,20 +86,6 @@ defmodule Diffo.Provider.AssignableCharacteristic do
     end
   end
 
-  actions do
-    create :create do
-      accept [:name, :first, :last, :assignable_type, :algorithm, :thing]
-      argument :instance_id, :uuid
-      argument :feature_id, :uuid
-      change manage_relationship(:instance_id, :instance, type: :append)
-      change manage_relationship(:feature_id, :feature, type: :append)
-    end
-
-    update :update do
-      accept [:first, :last, :assignable_type, :algorithm]
-    end
-  end
-
   preparations do
     prepare build(load: [:value, :free])
   end

lib/diffo/provider/assigner/assignable_characteristic/value.ex

@@ -6,19 +6,19 @@ defmodule Diffo.Provider.AssignableCharacteristic.Value do
   @moduledoc "JSON value struct for AssignableCharacteristic."
   use Ash.TypedStruct, extensions: [AshJason.TypedStruct]
 
+  jason do
+    pick [:first, :last, :assignable_type, :algorithm]
+    compact true
+    rename assignable_type: :type
+  end
+
   typed_struct do
     field :first, :integer, description: "the first assignable value in the pool"
     field :last, :integer, description: "the last assignable value in the pool"
     field :assignable_type, :string, description: "the type label of the assignable thing"
     field :algorithm, :atom, description: "the selection algorithm for auto-assign"
   end
 
-  jason do
-    pick [:first, :last, :assignable_type, :algorithm]
-    compact true
-    rename assignable_type: :type
-  end
-
   defimpl String.Chars do
     def to_string(struct), do: inspect(struct)
   end

lib/diffo/provider/assigner/assigner.ex

@@ -154,7 +154,7 @@ defmodule Diffo.Provider.Assigner do
        when is_struct(instance) and is_atom(pool) and is_atom(thing) and is_integer(value) do
     case pool_characteristic(instance.id, pool, thing) do
       {:ok, nil} -> false
-      {:ok, char} -> value in Enum.to_list(char.first..char.last) -- char.assigned_values
+      {:ok, char} -> value in (Enum.to_list(char.first..char.last) -- char.assigned_values)
       {:error, _} -> false
     end
   end

lib/diffo/provider/changes/validate_relationship_permitted.ex

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: 2025 diffo contributors <https://github.com/diffo-dev/diffo/graphs.contributors>
+#
+# SPDX-License-Identifier: MIT
+
+defmodule Diffo.Provider.Changes.ValidateRelationshipPermitted do
+  @moduledoc false
+  use Ash.Resource.Change
+
+  @impl true
+  def change(changeset, _opts, _context) do
+    case Ash.Changeset.get_argument(changeset, :relationships) do
+      nil -> changeset
+      [] -> changeset
+      rels -> validate_source_roles(changeset, rels)
+    end
+  end
+
+  defp validate_source_roles(changeset, rels) do
+    permitted = changeset.resource.permitted_source_roles()
+
+    Enum.reduce(rels, changeset, fn rel, cs ->
+      role = Map.get(rel, :alias) || Map.get(rel, "alias")
+
+      case check_permitted(role, permitted) do
+        :ok -> cs
+        {:error, msg} -> Ash.Changeset.add_error(cs, msg)
+      end
+    end)
+  end
+
+  defp check_permitted(_role, :all), do: :ok
+
+  defp check_permitted(_role, :none),
+    do: {:error, "relationships are not permitted as source on this resource"}
+
+  defp check_permitted(role, roles) when is_list(roles) do
+    if role in roles do
+      :ok
+    else
+      {:error, "relationship role #{inspect(role)} is not permitted as source on this resource"}
+    end
+  end
+end