From e599cb4f97d227902032dbce8d0a84c61dc98dfa Mon Sep 17 00:00:00 2001 From: Nicklas Lundin Date: Wed, 13 May 2026 15:10:57 +0200 Subject: [PATCH 1/2] fix(migrate-posthog): correct multivariant splits, setRule, and resolve verification - Use setRule for multi-value exact matches instead of or+eqRule - Add multivariant A/B split handling: one rule per condition with all variants and their percentages inside, not separate rules - Require positive AND negative resolve tests before reporting success - Require full context attributes in resolve calls - Bump version to 0.2.2 Co-Authored-By: Claude Opus 4.6 (1M context) --- .claude-plugin/marketplace.json | 4 +-- .claude-plugin/plugin.json | 2 +- skills/migrate-posthog/SKILL.md | 44 ++++++++++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 3ce90cc..ab7370a 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,13 +1,13 @@ { "name": "confidence", - "version": "0.2.1", + "version": "0.2.2", "owner": { "name": "Spotify Confidence" }, "plugins": [ { "name": "confidence", - "version": "0.2.1", + "version": "0.2.2", "description": "Access Confidence feature flags, experiments, and migration tools directly from Claude Code.", "source": { "source": "url", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 20457c6..53a4662 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "confidence", "description": "Access Confidence feature flags, experiments, and migration tools directly from Claude Code.", - "version": "0.2.1", + "version": "0.2.2", "author": { "name": "Spotify Confidence", "url": "https://confidence.spotify.com" diff --git a/skills/migrate-posthog/SKILL.md b/skills/migrate-posthog/SKILL.md index 1968c94..5c21d6e 100644 --- a/skills/migrate-posthog/SKILL.md +++ b/skills/migrate-posthog/SKILL.md @@ -671,8 +671,8 @@ Use this when generating `addTargetingRule` payloads in the plan file. |---------|-------------------| | `exact: "X"` | `eqRule` with `stringValue` | | `is_not: "X"` | NOT expression wrapping `eqRule` | -| `exact: ["A","B"]` | `or` expression with one `eqRule` per value | -| `is_not: ["A","B"]` | NOT wrapping `or` of `eqRule` per value | +| `exact: ["A","B"]` | `setRule` with `values` array | +| `is_not: ["A","B"]` | NOT wrapping `setRule` with `values` array | | `gte: N` | `rangeRule` with `startInclusive` | | `gt: N` | `rangeRule` with `startExclusive` | | `lt: N` | `rangeRule` with `endExclusive` | @@ -689,6 +689,30 @@ Use `and` expression with `operands` array. **Multiple groups (OR):** PostHog groups are ORed. Use `or` expression. +### Multivariant A/B Split Handling + +**CRITICAL:** A single Confidence targeting rule CAN assign multiple +variants at different split percentages. Use ONE rule per targeting +condition, listing all variants and their shares in that rule. + +**How to map PostHog splits to Confidence rules:** + +For a 2-variant flag (e.g. control 50% / treatment 50%): +- Add ONE rule with two variant assignments: + control at 50%, treatment at 50%. + +For a 3+ variant flag (e.g. control 34% / A 33% / B 33%): +- Add ONE rule with three variant assignments: + control at 34%, A at 33%, B at 33%. + +**Do NOT create separate rules per variant.** One targeting rule = +one set of targeting conditions, with the variant split defined +inside that rule. The `rolloutPercentage` on the rule controls +what fraction of users who match the targeting conditions enter the +rule at all (use 100% unless you want a partial rollout on top of +the targeting). The variant percentages within the rule control the +split among those who enter. + --- ## Plan Flag: Template @@ -805,12 +829,15 @@ During execution, each flag will be created one by one, interactively. **Description:** **Rules:** **Rollout:** +**Variants:** **PostHog bucketing:** <"distinct_id (per user)" or "group type (per company/group)"> **Confidence entity:** +**Confidence rollout:** **Action:** [ ] Migrate [ ] Skip **MCP Commands:** - + + --- @@ -908,11 +935,22 @@ STEP 3: addTargetingRule STEP 4: resolveFlag (verification) → Only NOW resolve to verify the flag works + → MUST test BOTH positive AND negative cases: + a. Resolve with a context that SHOULD match the targeting rule + → Verify the expected variant is returned + b. Resolve with a context that SHOULD NOT match + → Verify no variant / default is returned + → For attribute-based targeting (country, plan, etc.), the resolve + call MUST include those attributes in the evaluation context. + Without them, the targeting conditions cannot be evaluated and + may appear to match when they wouldn't in production. → If resolve fails with "No active flags found": something went wrong in steps 1-2 — diagnose, don't skip → If all rules show "Rule is inactive" / no match: targeting rules were likely added while flag was archived. Re-add the targeting rule now that the flag is active. + → Do NOT report a flag as successfully migrated until both + positive and negative resolve tests pass. ``` **Why this matters:** Confidence flags can be in states that From edf616e72f2e5516b0b7d4227b76876125399286 Mon Sep 17 00:00:00 2001 From: Nicklas Lundin Date: Wed, 13 May 2026 15:44:34 +0200 Subject: [PATCH 2/2] chore: remove manual version bump, release-please owns this now Co-Authored-By: Claude Opus 4.6 (1M context) --- .claude-plugin/marketplace.json | 4 ++-- .claude-plugin/plugin.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index ab7370a..3ce90cc 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,13 +1,13 @@ { "name": "confidence", - "version": "0.2.2", + "version": "0.2.1", "owner": { "name": "Spotify Confidence" }, "plugins": [ { "name": "confidence", - "version": "0.2.2", + "version": "0.2.1", "description": "Access Confidence feature flags, experiments, and migration tools directly from Claude Code.", "source": { "source": "url", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 53a4662..20457c6 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "confidence", "description": "Access Confidence feature flags, experiments, and migration tools directly from Claude Code.", - "version": "0.2.2", + "version": "0.2.1", "author": { "name": "Spotify Confidence", "url": "https://confidence.spotify.com"