Holdouts
Create holdout
Create a new holdout in the specified project.
Authorization
ApiKey read, writeAuthorization<token>
In: header
Scope: read, write
Path Parameters
projectKey*string
The project key
Format
stringenvironmentKey*string
The environment key
Format
stringRequest Body
application/json
name?string
A human-friendly name for the holdout
key?string
A key that identifies the holdout
description?string
Description of the holdout
randomizationunit?string
The chosen randomization unit for the holdout base experiment
attributes?array<string>
The attributes that the holdout iteration's results can be sliced by
holdoutamount?string
Audience allocation for the holdout
primarymetrickey?string
The key of the primary metric for this holdout
metrics?array<>
Details on the metrics for this experiment
prerequisiteflagkey?string
The key of the flag that the holdout is dependent on
maintainerId?string
Maintainer id
Response Body
application/json
application/json
curl -X POST "https://example.com/api/v2/projects/string/environments/string/holdouts" \ -H "Content-Type: application/json" \ -d '{}'{ "_id": "string", "status": "created", "description": "string", "holdoutAmount": "string", "createdAt": 0, "updatedAt": 0, "baseExperiment": { "_id": "12ab3c45de678910fgh12345", "key": "experiment-key-123abc", "name": "Example experiment", "description": "An example experiment, used in testing", "_maintainerId": "12ab3c45de678910fgh12345", "_creationDate": "1654104600000", "environmentKey": "string", "methodology": "bayesian", "dataSource": "snowflake", "archivedDate": "1654104600000", "tags": [ "experiment", "feature" ], "_links": { "parent": { "href": "/api/v2/projects/my-project/environments/my-environment", "type": "application/json" }, "self": { "href": "/api/v2/projects/my-project/environments/my-environment/experiments/my-experiment", "type": "application/json" } }, "holdoutId": "f3b74309-d581-44e1-8a2b-bb2933b4fe40", "currentIteration": { "_id": "12ab3c45de678910fgh12345", "hypothesis": "The new button placement will increase conversion", "status": "running", "createdAt": "1654104600000", "startedAt": "1655314200000", "endedAt": "1656610200000", "winningTreatmentId": "122c9f3e-da26-4321-ba68-e0fc02eced58", "winningReason": "We ran this iteration for two weeks and the winning variation was clear", "canReshuffleTraffic": true, "flags": { "property1": { "targetingRule": "fallthrough", "targetingRuleDescription": "Customers who live in Canada", "targetingRuleClauses": [ null ], "flagConfigVersion": 12, "notInExperimentVariationId": "e432f62b-55f6-49dd-a02f-eb24acf39d05", "_links": { "self": { "href": "/api/v2/flags/my-project/my-flag", "type": "application/json" } } }, "property2": { "targetingRule": "fallthrough", "targetingRuleDescription": "Customers who live in Canada", "targetingRuleClauses": [ null ], "flagConfigVersion": 12, "notInExperimentVariationId": "e432f62b-55f6-49dd-a02f-eb24acf39d05", "_links": { "self": { "href": "/api/v2/flags/my-project/my-flag", "type": "application/json" } } } }, "reallocationFrequencyMillis": 3600000, "version": 0, "primaryMetric": { "key": "metric-key-123abc", "_versionId": "string", "name": "My metric", "kind": "custom", "isNumeric": true, "eventKey": "event-key-123abc", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } }, "isGroup": true, "metrics": [ { "key": "metric-key-123abc", "_versionId": "version-id-123abc", "name": "Example metric", "kind": "custom", "isNumeric": true, "unitAggregationType": "sum", "analysisType": "mean", "eventKey": "event-key-123abc", "analysisUnit": "string", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } }, "nameInGroup": "Step 1", "randomizationUnits": [ "user" ], "analysisUnits": [ "user" ] } ], "analysisType": "mean", "analysisUnit": "user" }, "primarySingleMetric": { "key": "metric-key-123abc", "_versionId": "version-id-123abc", "name": "Example metric", "kind": "custom", "isNumeric": true, "unitAggregationType": "sum", "analysisType": "mean", "eventKey": "event-key-123abc", "analysisUnit": "string", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } } }, "primaryFunnel": { "key": "metric-group-key-123abc", "name": "My metric group", "kind": "funnel", "_links": { "parent": { "href": "/api/v2/projects/my-project", "type": "application/json" }, "self": { "href": "/api/v2/projects/my-project/metric-groups/my-metric-group", "type": "application/json" } }, "metrics": [ { "key": "metric-key-123abc", "_versionId": "version-id-123abc", "name": "Example metric", "kind": "custom", "isNumeric": true, "unitAggregationType": "sum", "analysisType": "mean", "eventKey": "event-key-123abc", "analysisUnit": "string", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } }, "nameInGroup": "Step 1", "randomizationUnits": [ "user" ], "analysisUnits": [ "user" ] } ] }, "randomizationUnit": "user", "attributes": [ "string" ], "treatments": [ { "_id": "122c9f3e-da26-4321-ba68-e0fc02eced58", "name": "Treatment 1", "allocationPercent": "10", "baseline": true, "parameters": [ { "variationId": "string", "flagKey": "string" } ] } ], "secondaryMetrics": [ { "key": "metric-key-123abc", "_versionId": "version-id-123abc", "name": "Example metric", "kind": "custom", "isNumeric": true, "unitAggregationType": "sum", "analysisType": "mean", "eventKey": "event-key-123abc", "analysisUnit": "string", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } } } ], "metrics": [ { "key": "metric-key-123abc", "_versionId": "string", "name": "My metric", "kind": "custom", "isNumeric": true, "eventKey": "event-key-123abc", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } }, "isGroup": true, "metrics": [ { "key": "metric-key-123abc", "_versionId": "version-id-123abc", "name": "Example metric", "kind": "custom", "isNumeric": true, "unitAggregationType": "sum", "analysisType": "mean", "eventKey": "event-key-123abc", "analysisUnit": "string", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } }, "nameInGroup": "Step 1", "randomizationUnits": [ "user" ], "analysisUnits": [ "user" ] } ], "analysisType": "mean", "analysisUnit": "user" } ], "layerSnapshot": { "key": "checkout-flow", "name": "Checkout Flow", "reservationPercent": 10, "otherReservationPercent": 70 }, "covariateInfo": { "id": "74a49a2b-4834-4246-917e-5d85231d8c2a", "fileName": "covariate.csv", "createdAt": "1654104600000" } }, "type": "experiment", "_access": { "denied": [ { "action": "string", "reason": { "resources": [ "proj/*:env/*;qa_*:/flag/*" ], "notResources": [ "string" ], "actions": [ "*" ], "notActions": [ "string" ], "effect": "allow", "role_name": "string" } } ], "allowed": [ { "action": "string", "reason": { "resources": [ "proj/*:env/*;qa_*:/flag/*" ], "notResources": [ "string" ], "actions": [ "*" ], "notActions": [ "string" ], "effect": "allow", "role_name": "string" } } ] }, "draftIteration": { "_id": "12ab3c45de678910fgh12345", "hypothesis": "The new button placement will increase conversion", "status": "running", "createdAt": "1654104600000", "startedAt": "1655314200000", "endedAt": "1656610200000", "winningTreatmentId": "122c9f3e-da26-4321-ba68-e0fc02eced58", "winningReason": "We ran this iteration for two weeks and the winning variation was clear", "canReshuffleTraffic": true, "flags": { "property1": { "targetingRule": "fallthrough", "targetingRuleDescription": "Customers who live in Canada", "targetingRuleClauses": [ null ], "flagConfigVersion": 12, "notInExperimentVariationId": "e432f62b-55f6-49dd-a02f-eb24acf39d05", "_links": { "self": { "href": "/api/v2/flags/my-project/my-flag", "type": "application/json" } } }, "property2": { "targetingRule": "fallthrough", "targetingRuleDescription": "Customers who live in Canada", "targetingRuleClauses": [ null ], "flagConfigVersion": 12, "notInExperimentVariationId": "e432f62b-55f6-49dd-a02f-eb24acf39d05", "_links": { "self": { "href": "/api/v2/flags/my-project/my-flag", "type": "application/json" } } } }, "reallocationFrequencyMillis": 3600000, "version": 0, "primaryMetric": { "key": "metric-key-123abc", "_versionId": "string", "name": "My metric", "kind": "custom", "isNumeric": true, "eventKey": "event-key-123abc", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } }, "isGroup": true, "metrics": [ { "key": "metric-key-123abc", "_versionId": "version-id-123abc", "name": "Example metric", "kind": "custom", "isNumeric": true, "unitAggregationType": "sum", "analysisType": "mean", "eventKey": "event-key-123abc", "analysisUnit": "string", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } }, "nameInGroup": "Step 1", "randomizationUnits": [ "user" ], "analysisUnits": [ "user" ] } ], "analysisType": "mean", "analysisUnit": "user" }, "primarySingleMetric": { "key": "metric-key-123abc", "_versionId": "version-id-123abc", "name": "Example metric", "kind": "custom", "isNumeric": true, "unitAggregationType": "sum", "analysisType": "mean", "eventKey": "event-key-123abc", "analysisUnit": "string", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } } }, "primaryFunnel": { "key": "metric-group-key-123abc", "name": "My metric group", "kind": "funnel", "_links": { "parent": { "href": "/api/v2/projects/my-project", "type": "application/json" }, "self": { "href": "/api/v2/projects/my-project/metric-groups/my-metric-group", "type": "application/json" } }, "metrics": [ { "key": "metric-key-123abc", "_versionId": "version-id-123abc", "name": "Example metric", "kind": "custom", "isNumeric": true, "unitAggregationType": "sum", "analysisType": "mean", "eventKey": "event-key-123abc", "analysisUnit": "string", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } }, "nameInGroup": "Step 1", "randomizationUnits": [ "user" ], "analysisUnits": [ "user" ] } ] }, "randomizationUnit": "user", "attributes": [ "string" ], "treatments": [ { "_id": "122c9f3e-da26-4321-ba68-e0fc02eced58", "name": "Treatment 1", "allocationPercent": "10", "baseline": true, "parameters": [ { "variationId": "string", "flagKey": "string" } ] } ], "secondaryMetrics": [ { "key": "metric-key-123abc", "_versionId": "version-id-123abc", "name": "Example metric", "kind": "custom", "isNumeric": true, "unitAggregationType": "sum", "analysisType": "mean", "eventKey": "event-key-123abc", "analysisUnit": "string", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } } } ], "metrics": [ { "key": "metric-key-123abc", "_versionId": "string", "name": "My metric", "kind": "custom", "isNumeric": true, "eventKey": "event-key-123abc", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } }, "isGroup": true, "metrics": [ { "key": "metric-key-123abc", "_versionId": "version-id-123abc", "name": "Example metric", "kind": "custom", "isNumeric": true, "unitAggregationType": "sum", "analysisType": "mean", "eventKey": "event-key-123abc", "analysisUnit": "string", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } }, "nameInGroup": "Step 1", "randomizationUnits": [ "user" ], "analysisUnits": [ "user" ] } ], "analysisType": "mean", "analysisUnit": "user" } ], "layerSnapshot": { "key": "checkout-flow", "name": "Checkout Flow", "reservationPercent": 10, "otherReservationPercent": 70 }, "covariateInfo": { "id": "74a49a2b-4834-4246-917e-5d85231d8c2a", "fileName": "covariate.csv", "createdAt": "1654104600000" } }, "previousIterations": [ { "_id": "12ab3c45de678910fgh12345", "hypothesis": "The new button placement will increase conversion", "status": "running", "createdAt": "1654104600000", "startedAt": "1655314200000", "endedAt": "1656610200000", "winningTreatmentId": "122c9f3e-da26-4321-ba68-e0fc02eced58", "winningReason": "We ran this iteration for two weeks and the winning variation was clear", "canReshuffleTraffic": true, "flags": { "property1": { "targetingRule": "fallthrough", "targetingRuleDescription": "Customers who live in Canada", "targetingRuleClauses": [ null ], "flagConfigVersion": 12, "notInExperimentVariationId": "e432f62b-55f6-49dd-a02f-eb24acf39d05", "_links": { "self": { "href": "/api/v2/flags/my-project/my-flag", "type": "application/json" } } }, "property2": { "targetingRule": "fallthrough", "targetingRuleDescription": "Customers who live in Canada", "targetingRuleClauses": [ null ], "flagConfigVersion": 12, "notInExperimentVariationId": "e432f62b-55f6-49dd-a02f-eb24acf39d05", "_links": { "self": { "href": "/api/v2/flags/my-project/my-flag", "type": "application/json" } } } }, "reallocationFrequencyMillis": 3600000, "version": 0, "primaryMetric": { "key": "metric-key-123abc", "_versionId": "string", "name": "My metric", "kind": "custom", "isNumeric": true, "eventKey": "event-key-123abc", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } }, "isGroup": true, "metrics": [ { "key": "metric-key-123abc", "_versionId": "version-id-123abc", "name": "Example metric", "kind": "custom", "isNumeric": true, "unitAggregationType": "sum", "analysisType": "mean", "eventKey": "event-key-123abc", "analysisUnit": "string", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } }, "nameInGroup": "Step 1", "randomizationUnits": [ "user" ], "analysisUnits": [ "user" ] } ], "analysisType": "mean", "analysisUnit": "user" }, "primarySingleMetric": { "key": "metric-key-123abc", "_versionId": "version-id-123abc", "name": "Example metric", "kind": "custom", "isNumeric": true, "unitAggregationType": "sum", "analysisType": "mean", "eventKey": "event-key-123abc", "analysisUnit": "string", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } } }, "primaryFunnel": { "key": "metric-group-key-123abc", "name": "My metric group", "kind": "funnel", "_links": { "parent": { "href": "/api/v2/projects/my-project", "type": "application/json" }, "self": { "href": "/api/v2/projects/my-project/metric-groups/my-metric-group", "type": "application/json" } }, "metrics": [ { "key": "metric-key-123abc", "_versionId": "version-id-123abc", "name": "Example metric", "kind": "custom", "isNumeric": true, "unitAggregationType": "sum", "analysisType": "mean", "eventKey": "event-key-123abc", "analysisUnit": "string", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } }, "nameInGroup": "Step 1", "randomizationUnits": [ "user" ], "analysisUnits": [ "user" ] } ] }, "randomizationUnit": "user", "attributes": [ "string" ], "treatments": [ { "_id": "122c9f3e-da26-4321-ba68-e0fc02eced58", "name": "Treatment 1", "allocationPercent": "10", "baseline": true, "parameters": [ { "variationId": "string", "flagKey": "string" } ] } ], "secondaryMetrics": [ { "key": "metric-key-123abc", "_versionId": "version-id-123abc", "name": "Example metric", "kind": "custom", "isNumeric": true, "unitAggregationType": "sum", "analysisType": "mean", "eventKey": "event-key-123abc", "analysisUnit": "string", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } } } ], "metrics": [ { "key": "metric-key-123abc", "_versionId": "string", "name": "My metric", "kind": "custom", "isNumeric": true, "eventKey": "event-key-123abc", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } }, "isGroup": true, "metrics": [ { "key": "metric-key-123abc", "_versionId": "version-id-123abc", "name": "Example metric", "kind": "custom", "isNumeric": true, "unitAggregationType": "sum", "analysisType": "mean", "eventKey": "event-key-123abc", "analysisUnit": "string", "_links": { "self": { "href": "/api/v2/metrics/my-project/my-metric", "type": "application/json" } }, "nameInGroup": "Step 1", "randomizationUnits": [ "user" ], "analysisUnits": [ "user" ] } ], "analysisType": "mean", "analysisUnit": "user" } ], "layerSnapshot": { "key": "checkout-flow", "name": "Checkout Flow", "reservationPercent": 10, "otherReservationPercent": 70 }, "covariateInfo": { "id": "74a49a2b-4834-4246-917e-5d85231d8c2a", "fileName": "covariate.csv", "createdAt": "1654104600000" } } ], "analysisConfig": { "bayesianThreshold": "string", "significanceThreshold": "string", "testDirection": "string", "multipleComparisonCorrectionMethod": "bonferroni", "multipleComparisonCorrectionScope": "variations", "sequentialTestingEnabled": true }, "mutableFieldsByStatus": { "not_started": { "property1": [ "string" ], "property2": [ "string" ] }, "running": { "property1": [ "string" ], "property2": [ "string" ] }, "stopped": { "property1": [ "string" ], "property2": [ "string" ] } } }, "experiments": [ { "key": "string", "name": "string", "environment": "string" } ]}