Last active
October 9, 2025 07:44
-
-
Save pokutuna/e0cb2a67e85426d73e3b967fda08563c to your computer and use it in GitHub Desktop.
promptfoo config-schema.json with assertionTemplates support
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "$ref": "#/definitions/PromptfooConfigSchema", | |
| "definitions": { | |
| "PromptfooConfigSchema": { | |
| "type": "object", | |
| "properties": { | |
| "tags": { | |
| "type": "object", | |
| "additionalProperties": { | |
| "type": "string" | |
| } | |
| }, | |
| "description": { | |
| "type": "string" | |
| }, | |
| "providers": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "allOf": [ | |
| { | |
| "type": "object", | |
| "properties": { | |
| "label": { | |
| "type": "string" | |
| } | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "array", | |
| "items": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "type": "object", | |
| "additionalProperties": { | |
| "type": "object", | |
| "properties": { | |
| "id": {}, | |
| "label": {}, | |
| "config": {}, | |
| "prompts": { | |
| "type": "array", | |
| "items": { | |
| "type": "string" | |
| } | |
| }, | |
| "transform": { | |
| "type": "string" | |
| }, | |
| "delay": { | |
| "type": "number" | |
| }, | |
| "env": { | |
| "type": "object", | |
| "properties": { | |
| "AI21_API_BASE_URL": { | |
| "type": "string" | |
| }, | |
| "AI21_API_KEY": { | |
| "type": "string" | |
| }, | |
| "AIML_API_KEY": { | |
| "type": "string" | |
| }, | |
| "ANTHROPIC_API_KEY": { | |
| "type": "string" | |
| }, | |
| "ANTHROPIC_BASE_URL": { | |
| "type": "string" | |
| }, | |
| "AWS_BEDROCK_REGION": { | |
| "type": "string" | |
| }, | |
| "AZURE_API_BASE_URL": { | |
| "type": "string" | |
| }, | |
| "AZURE_API_HOST": { | |
| "type": "string" | |
| }, | |
| "AZURE_API_KEY": { | |
| "type": "string" | |
| }, | |
| "AZURE_AUTHORITY_HOST": { | |
| "type": "string" | |
| }, | |
| "AZURE_CLIENT_ID": { | |
| "type": "string" | |
| }, | |
| "AZURE_CLIENT_SECRET": { | |
| "type": "string" | |
| }, | |
| "AZURE_DEPLOYMENT_NAME": { | |
| "type": "string" | |
| }, | |
| "AZURE_EMBEDDING_DEPLOYMENT_NAME": { | |
| "type": "string" | |
| }, | |
| "AZURE_OPENAI_API_BASE_URL": { | |
| "type": "string" | |
| }, | |
| "AZURE_OPENAI_API_HOST": { | |
| "type": "string" | |
| }, | |
| "AZURE_OPENAI_API_KEY": { | |
| "type": "string" | |
| }, | |
| "AZURE_OPENAI_BASE_URL": { | |
| "type": "string" | |
| }, | |
| "AZURE_OPENAI_DEPLOYMENT_NAME": { | |
| "type": "string" | |
| }, | |
| "AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME": { | |
| "type": "string" | |
| }, | |
| "AZURE_TENANT_ID": { | |
| "type": "string" | |
| }, | |
| "AZURE_TOKEN_SCOPE": { | |
| "type": "string" | |
| }, | |
| "CLOUDFLARE_ACCOUNT_ID": { | |
| "type": "string" | |
| }, | |
| "CLOUDFLARE_API_KEY": { | |
| "type": "string" | |
| }, | |
| "COMETAPI_KEY": { | |
| "type": "string" | |
| }, | |
| "COHERE_API_KEY": { | |
| "type": "string" | |
| }, | |
| "COHERE_CLIENT_NAME": { | |
| "type": "string" | |
| }, | |
| "DATABRICKS_TOKEN": { | |
| "type": "string" | |
| }, | |
| "DATABRICKS_WORKSPACE_URL": { | |
| "type": "string" | |
| }, | |
| "DOCKER_MODEL_RUNNER_BASE_URL": { | |
| "type": "string" | |
| }, | |
| "DOCKER_MODEL_RUNNER_API_KEY": { | |
| "type": "string" | |
| }, | |
| "FAL_KEY": { | |
| "type": "string" | |
| }, | |
| "GITHUB_TOKEN": { | |
| "type": "string" | |
| }, | |
| "GOOGLE_API_HOST": { | |
| "type": "string" | |
| }, | |
| "GOOGLE_API_BASE_URL": { | |
| "type": "string" | |
| }, | |
| "GOOGLE_API_KEY": { | |
| "type": "string" | |
| }, | |
| "GOOGLE_PROJECT_ID": { | |
| "type": "string" | |
| }, | |
| "GOOGLE_LOCATION": { | |
| "type": "string" | |
| }, | |
| "GOOGLE_GENERATIVE_AI_API_KEY": { | |
| "type": "string" | |
| }, | |
| "GEMINI_API_KEY": { | |
| "type": "string" | |
| }, | |
| "GROQ_API_KEY": { | |
| "type": "string" | |
| }, | |
| "HELICONE_API_KEY": { | |
| "type": "string" | |
| }, | |
| "HF_API_TOKEN": { | |
| "type": "string" | |
| }, | |
| "HF_TOKEN": { | |
| "type": "string" | |
| }, | |
| "HYPERBOLIC_API_KEY": { | |
| "type": "string" | |
| }, | |
| "HUGGING_FACE_HUB_TOKEN": { | |
| "type": "string" | |
| }, | |
| "JFROG_API_KEY": { | |
| "type": "string" | |
| }, | |
| "LANGFUSE_HOST": { | |
| "type": "string" | |
| }, | |
| "LANGFUSE_PUBLIC_KEY": { | |
| "type": "string" | |
| }, | |
| "LANGFUSE_SECRET_KEY": { | |
| "type": "string" | |
| }, | |
| "LITELLM_API_BASE": { | |
| "type": "string" | |
| }, | |
| "LLAMA_BASE_URL": { | |
| "type": "string" | |
| }, | |
| "LOCALAI_BASE_URL": { | |
| "type": "string" | |
| }, | |
| "MISTRAL_API_BASE_URL": { | |
| "type": "string" | |
| }, | |
| "MISTRAL_API_HOST": { | |
| "type": "string" | |
| }, | |
| "MISTRAL_API_KEY": { | |
| "type": "string" | |
| }, | |
| "NSCALE_SERVICE_TOKEN": { | |
| "type": "string" | |
| }, | |
| "NSCALE_API_KEY": { | |
| "type": "string" | |
| }, | |
| "OLLAMA_API_KEY": { | |
| "type": "string" | |
| }, | |
| "OLLAMA_BASE_URL": { | |
| "type": "string" | |
| }, | |
| "OPENAI_API_BASE_URL": { | |
| "type": "string" | |
| }, | |
| "OPENAI_API_HOST": { | |
| "type": "string" | |
| }, | |
| "OPENAI_API_KEY": { | |
| "type": "string" | |
| }, | |
| "OPENAI_BASE_URL": { | |
| "type": "string" | |
| }, | |
| "OPENAI_ORGANIZATION": { | |
| "type": "string" | |
| }, | |
| "PALM_API_HOST": { | |
| "type": "string" | |
| }, | |
| "PALM_API_KEY": { | |
| "type": "string" | |
| }, | |
| "PORTKEY_API_KEY": { | |
| "type": "string" | |
| }, | |
| "PROMPTFOO_CA_CERT_PATH": { | |
| "type": "string" | |
| }, | |
| "PROMPTFOO_PFX_CERT_PATH": { | |
| "type": "string" | |
| }, | |
| "PROMPTFOO_PFX_PASSWORD": { | |
| "type": "string" | |
| }, | |
| "PROMPTFOO_JKS_CERT_PATH": { | |
| "type": "string" | |
| }, | |
| "PROMPTFOO_JKS_PASSWORD": { | |
| "type": "string" | |
| }, | |
| "PROMPTFOO_JKS_ALIAS": { | |
| "type": "string" | |
| }, | |
| "PROMPTFOO_INSECURE_SSL": { | |
| "type": "string" | |
| }, | |
| "REPLICATE_API_KEY": { | |
| "type": "string" | |
| }, | |
| "REPLICATE_API_TOKEN": { | |
| "type": "string" | |
| }, | |
| "VERTEX_API_HOST": { | |
| "type": "string" | |
| }, | |
| "VERTEX_API_KEY": { | |
| "type": "string" | |
| }, | |
| "VERTEX_API_VERSION": { | |
| "type": "string" | |
| }, | |
| "VERTEX_PROJECT_ID": { | |
| "type": "string" | |
| }, | |
| "VERTEX_PUBLISHER": { | |
| "type": "string" | |
| }, | |
| "VERTEX_REGION": { | |
| "type": "string" | |
| }, | |
| "VOYAGE_API_BASE_URL": { | |
| "type": "string" | |
| }, | |
| "VOYAGE_API_KEY": { | |
| "type": "string" | |
| }, | |
| "WATSONX_AI_APIKEY": { | |
| "type": "string" | |
| }, | |
| "WATSONX_AI_AUTH_TYPE": { | |
| "type": "string" | |
| }, | |
| "WATSONX_AI_BEARER_TOKEN": { | |
| "type": "string" | |
| }, | |
| "WATSONX_AI_PROJECT_ID": { | |
| "type": "string" | |
| }, | |
| "AZURE_CONTENT_SAFETY_ENDPOINT": { | |
| "type": "string" | |
| }, | |
| "AZURE_CONTENT_SAFETY_API_KEY": { | |
| "type": "string" | |
| }, | |
| "AZURE_CONTENT_SAFETY_API_VERSION": { | |
| "type": "string" | |
| }, | |
| "AWS_REGION": { | |
| "type": "string" | |
| }, | |
| "AWS_DEFAULT_REGION": { | |
| "type": "string" | |
| }, | |
| "AWS_SAGEMAKER_MAX_TOKENS": { | |
| "type": "string" | |
| }, | |
| "AWS_SAGEMAKER_TEMPERATURE": { | |
| "type": "string" | |
| }, | |
| "AWS_SAGEMAKER_TOP_P": { | |
| "type": "string" | |
| }, | |
| "AWS_SAGEMAKER_MAX_RETRIES": { | |
| "type": "string" | |
| }, | |
| "PROMPTFOO_EVAL_TIMEOUT_MS": { | |
| "type": "string" | |
| } | |
| }, | |
| "additionalProperties": false | |
| } | |
| }, | |
| "additionalProperties": false | |
| } | |
| }, | |
| { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/providers/anyOf/2/items/anyOf/1/additionalProperties" | |
| }, | |
| { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/providers/anyOf/1" | |
| } | |
| ] | |
| } | |
| } | |
| ] | |
| }, | |
| "prompts": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "type": "array", | |
| "items": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "id": { | |
| "type": "string" | |
| }, | |
| "label": { | |
| "type": "string" | |
| }, | |
| "raw": { | |
| "type": "string" | |
| } | |
| }, | |
| "required": ["id"], | |
| "additionalProperties": false | |
| }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "id": { | |
| "type": "string" | |
| }, | |
| "raw": { | |
| "type": "string" | |
| }, | |
| "display": { | |
| "type": "string" | |
| }, | |
| "label": { | |
| "type": "string" | |
| }, | |
| "config": {} | |
| }, | |
| "required": ["raw", "label"], | |
| "additionalProperties": false | |
| } | |
| ] | |
| } | |
| }, | |
| { | |
| "type": "object", | |
| "additionalProperties": { | |
| "type": "string" | |
| } | |
| } | |
| ] | |
| }, | |
| "tests": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "type": "array", | |
| "items": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "description": { | |
| "type": "string" | |
| }, | |
| "vars": { | |
| "type": "object", | |
| "additionalProperties": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "boolean" | |
| }, | |
| { | |
| "type": "array", | |
| "items": { | |
| "type": ["string", "number", "boolean"] | |
| } | |
| }, | |
| { | |
| "type": "object", | |
| "additionalProperties": {} | |
| }, | |
| { | |
| "type": "array" | |
| } | |
| ] | |
| } | |
| }, | |
| "provider": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/providers/anyOf/2/items/anyOf/1/additionalProperties" | |
| }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "callApi": {}, | |
| "label": {}, | |
| "transform": { | |
| "type": "string" | |
| }, | |
| "delay": { | |
| "type": "number" | |
| }, | |
| "config": {} | |
| }, | |
| "additionalProperties": false | |
| } | |
| ] | |
| }, | |
| "providerOutput": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "type": "object", | |
| "properties": {}, | |
| "additionalProperties": false | |
| } | |
| ] | |
| }, | |
| "assert": { | |
| "type": "array", | |
| "items": { | |
| "anyOf": [ | |
| { | |
| "type": "object", | |
| "properties": { | |
| "type": { | |
| "type": "string", | |
| "const": "assert-set" | |
| }, | |
| "assert": { | |
| "type": "array", | |
| "items": { | |
| "type": "object", | |
| "properties": { | |
| "type": { | |
| "anyOf": [ | |
| { | |
| "type": "string", | |
| "enum": [ | |
| "answer-relevance", | |
| "bleu", | |
| "classifier", | |
| "contains", | |
| "contains-all", | |
| "contains-any", | |
| "contains-html", | |
| "contains-json", | |
| "contains-sql", | |
| "contains-xml", | |
| "context-faithfulness", | |
| "context-recall", | |
| "context-relevance", | |
| "conversation-relevance", | |
| "cost", | |
| "equals", | |
| "factuality", | |
| "finish-reason", | |
| "g-eval", | |
| "gleu", | |
| "guardrails", | |
| "icontains", | |
| "icontains-all", | |
| "icontains-any", | |
| "is-html", | |
| "is-json", | |
| "is-refusal", | |
| "is-sql", | |
| "is-valid-function-call", | |
| "is-valid-openai-function-call", | |
| "is-valid-openai-tools-call", | |
| "is-xml", | |
| "javascript", | |
| "latency", | |
| "levenshtein", | |
| "llm-rubric", | |
| "pi", | |
| "meteor", | |
| "model-graded-closedqa", | |
| "model-graded-factuality", | |
| "moderation", | |
| "perplexity", | |
| "perplexity-score", | |
| "python", | |
| "regex", | |
| "rouge-n", | |
| "similar", | |
| "starts-with", | |
| "trace-error-spans", | |
| "trace-span-count", | |
| "trace-span-duration", | |
| "webhook" | |
| ] | |
| }, | |
| { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/assert/items/anyOf/0/properties/assert/items/properties/type/anyOf/0" | |
| }, | |
| { | |
| "type": "string", | |
| "enum": ["select-best", "human", "max-score"] | |
| }, | |
| {} | |
| ] | |
| }, | |
| "value": {}, | |
| "config": { | |
| "type": "object", | |
| "additionalProperties": {} | |
| }, | |
| "threshold": { | |
| "type": "number" | |
| }, | |
| "weight": { | |
| "type": "number" | |
| }, | |
| "provider": {}, | |
| "rubricPrompt": {}, | |
| "metric": { | |
| "type": "string" | |
| }, | |
| "transform": { | |
| "type": "string" | |
| }, | |
| "contextTransform": { | |
| "type": "string" | |
| } | |
| }, | |
| "additionalProperties": false | |
| } | |
| }, | |
| "weight": { | |
| "type": "number" | |
| }, | |
| "metric": { | |
| "type": "string" | |
| }, | |
| "threshold": { | |
| "type": "number" | |
| }, | |
| "config": { | |
| "type": "object", | |
| "additionalProperties": {} | |
| } | |
| }, | |
| "required": ["type", "assert"], | |
| "additionalProperties": false | |
| }, | |
| { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/assert/items/anyOf/0/properties/assert/items" | |
| }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "$ref": { | |
| "type": "string" | |
| } | |
| }, | |
| "required": ["$ref"], | |
| "additionalProperties": false | |
| } | |
| ] | |
| } | |
| }, | |
| "assertScoringFunction": { | |
| "anyOf": [ | |
| { | |
| "type": "string", | |
| "pattern": "^file:\\/\\/.*\\.(js|cjs|mjs|ts|cts|mts|py)(?::[\\w.]+)?$" | |
| }, | |
| {} | |
| ] | |
| }, | |
| "options": { | |
| "allOf": [ | |
| { | |
| "type": "object", | |
| "properties": { | |
| "prefix": { | |
| "type": "string" | |
| }, | |
| "suffix": { | |
| "type": "string" | |
| } | |
| } | |
| }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "postprocess": { | |
| "type": "string" | |
| }, | |
| "transform": { | |
| "type": "string" | |
| }, | |
| "transformVars": { | |
| "type": "string" | |
| }, | |
| "storeOutputAs": { | |
| "type": "string" | |
| } | |
| } | |
| }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "rubricPrompt": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "type": "array", | |
| "items": { | |
| "type": "string" | |
| } | |
| }, | |
| { | |
| "type": "array", | |
| "items": { | |
| "type": "object", | |
| "properties": { | |
| "role": { | |
| "type": "string" | |
| }, | |
| "content": { | |
| "type": "string" | |
| } | |
| }, | |
| "required": ["role", "content"], | |
| "additionalProperties": false | |
| } | |
| } | |
| ] | |
| }, | |
| "provider": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| {}, | |
| { | |
| "anyOf": [ | |
| { | |
| "not": {} | |
| }, | |
| { | |
| "type": "object", | |
| "additionalProperties": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| {} | |
| ] | |
| } | |
| } | |
| ] | |
| } | |
| ] | |
| }, | |
| "factuality": { | |
| "type": "object", | |
| "properties": { | |
| "subset": { | |
| "type": "number" | |
| }, | |
| "superset": { | |
| "type": "number" | |
| }, | |
| "agree": { | |
| "type": "number" | |
| }, | |
| "disagree": { | |
| "type": "number" | |
| }, | |
| "differButFactual": { | |
| "type": "number" | |
| } | |
| }, | |
| "additionalProperties": false | |
| } | |
| } | |
| }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "disableVarExpansion": { | |
| "type": "boolean" | |
| }, | |
| "disableConversationVar": { | |
| "type": "boolean" | |
| }, | |
| "runSerially": { | |
| "type": "boolean" | |
| } | |
| } | |
| } | |
| ] | |
| }, | |
| "threshold": { | |
| "type": "number" | |
| }, | |
| "metadata": { | |
| "allOf": [ | |
| { | |
| "type": "object", | |
| "additionalProperties": {} | |
| }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "pluginConfig": {}, | |
| "strategyConfig": {} | |
| } | |
| } | |
| ] | |
| } | |
| }, | |
| "additionalProperties": false | |
| }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "path": { | |
| "type": "string" | |
| }, | |
| "config": { | |
| "type": "object", | |
| "additionalProperties": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "boolean" | |
| }, | |
| { | |
| "type": "array", | |
| "items": { | |
| "type": ["string", "number", "boolean"] | |
| } | |
| }, | |
| { | |
| "type": "object", | |
| "additionalProperties": {} | |
| }, | |
| {} | |
| ] | |
| } | |
| } | |
| }, | |
| "required": ["path"], | |
| "additionalProperties": false | |
| } | |
| ] | |
| } | |
| }, | |
| { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/2" | |
| } | |
| ] | |
| }, | |
| "scenarios": { | |
| "type": "array", | |
| "items": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "description": { | |
| "type": "string" | |
| }, | |
| "config": { | |
| "type": "array", | |
| "items": { | |
| "type": "object", | |
| "properties": { | |
| "description": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/description" | |
| }, | |
| "vars": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/vars" | |
| }, | |
| "provider": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/provider" | |
| }, | |
| "providerOutput": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/providerOutput" | |
| }, | |
| "assert": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/assert" | |
| }, | |
| "assertScoringFunction": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/assertScoringFunction" | |
| }, | |
| "options": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/options" | |
| }, | |
| "threshold": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/threshold" | |
| }, | |
| "metadata": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/metadata" | |
| } | |
| }, | |
| "additionalProperties": false | |
| } | |
| }, | |
| "tests": { | |
| "type": "array", | |
| "items": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1" | |
| } | |
| } | |
| }, | |
| "required": ["config", "tests"], | |
| "additionalProperties": false | |
| } | |
| ] | |
| } | |
| }, | |
| "defaultTest": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "vars": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/vars" | |
| }, | |
| "provider": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/provider" | |
| }, | |
| "providerOutput": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/providerOutput" | |
| }, | |
| "assert": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/assert" | |
| }, | |
| "assertScoringFunction": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/assertScoringFunction" | |
| }, | |
| "options": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/options" | |
| }, | |
| "threshold": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/threshold" | |
| }, | |
| "metadata": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/metadata" | |
| } | |
| }, | |
| "additionalProperties": false | |
| } | |
| ] | |
| }, | |
| "outputPath": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "type": "array", | |
| "items": { | |
| "type": "string" | |
| } | |
| } | |
| ] | |
| }, | |
| "sharing": { | |
| "anyOf": [ | |
| { | |
| "type": "boolean" | |
| }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "apiBaseUrl": { | |
| "type": "string" | |
| }, | |
| "appBaseUrl": { | |
| "type": "string" | |
| } | |
| }, | |
| "additionalProperties": false | |
| } | |
| ] | |
| }, | |
| "nunjucksFilters": { | |
| "type": "object", | |
| "additionalProperties": { | |
| "type": "string" | |
| } | |
| }, | |
| "env": { | |
| "anyOf": [ | |
| { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/providers/anyOf/2/items/anyOf/1/additionalProperties/properties/env" | |
| }, | |
| { | |
| "type": "object", | |
| "additionalProperties": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "boolean" | |
| } | |
| ] | |
| } | |
| } | |
| ] | |
| }, | |
| "derivedMetrics": { | |
| "type": "array", | |
| "items": { | |
| "type": "object", | |
| "properties": { | |
| "name": { | |
| "type": "string" | |
| }, | |
| "value": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| } | |
| ] | |
| } | |
| }, | |
| "required": ["name", "value"], | |
| "additionalProperties": false | |
| } | |
| }, | |
| "extensions": { | |
| "anyOf": [ | |
| { | |
| "type": "array", | |
| "items": { | |
| "type": "string" | |
| } | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "metadata": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/metadata/allOf/0" | |
| }, | |
| "redteam": { | |
| "type": "object", | |
| "properties": { | |
| "injectVar": { | |
| "type": "string", | |
| "description": "Variable to inject. Can be a string or array of strings. If string, it's transformed to an array. Inferred from the prompts by default." | |
| }, | |
| "purpose": { | |
| "type": "string", | |
| "description": "Purpose override string - describes the prompt templates" | |
| }, | |
| "testGenerationInstructions": { | |
| "type": "string", | |
| "description": "Additional instructions for test generation applied to each plugin" | |
| }, | |
| "provider": { | |
| "anyOf": [ | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/providers/anyOf/2/items/anyOf/1/additionalProperties" | |
| }, | |
| { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/provider/anyOf/2" | |
| } | |
| ], | |
| "description": "Provider used for generating adversarial inputs" | |
| }, | |
| "numTests": { | |
| "type": "integer", | |
| "exclusiveMinimum": 0, | |
| "description": "Number of tests to generate" | |
| }, | |
| "language": { | |
| "type": "string", | |
| "description": "Language of tests ot generate for this plugin" | |
| }, | |
| "entities": { | |
| "type": "array", | |
| "items": { | |
| "type": "string" | |
| }, | |
| "description": "Names of people, brands, or organizations related to your LLM application" | |
| }, | |
| "plugins": { | |
| "type": "array", | |
| "items": { | |
| "anyOf": [ | |
| { | |
| "anyOf": [ | |
| { | |
| "type": "string", | |
| "enum": [ | |
| "aegis", | |
| "agentic:memory-poisoning", | |
| "ascii-smuggling", | |
| "beavertails", | |
| "bfla", | |
| "bias", | |
| "bias:age", | |
| "bias:disability", | |
| "bias:gender", | |
| "bias:race", | |
| "bola", | |
| "cca", | |
| "competitors", | |
| "contracts", | |
| "cross-session-leak", | |
| "cyberseceval", | |
| "debug-access", | |
| "default", | |
| "divergent-repetition", | |
| "donotanswer", | |
| "eu:ai-act", | |
| "eu:ai-act:annex3:biometric-id", | |
| "eu:ai-act:annex3:critical-infrastructure", | |
| "eu:ai-act:annex3:education", | |
| "eu:ai-act:annex3:employment", | |
| "eu:ai-act:annex3:essential-services", | |
| "eu:ai-act:annex3:justice-democracy", | |
| "eu:ai-act:annex3:law-enforcement", | |
| "eu:ai-act:annex3:migration-border", | |
| "eu:ai-act:art5:biometric-categorisation", | |
| "eu:ai-act:art5:exploitation-of-vulnerabilities", | |
| "eu:ai-act:art5:remote-biometric-id-live", | |
| "eu:ai-act:art5:remote-biometric-id-post", | |
| "eu:ai-act:art5:social-scoring", | |
| "eu:ai-act:art5:subliminal-manipulation", | |
| "excessive-agency", | |
| "financial:calculation-error", | |
| "financial:compliance-violation", | |
| "financial:confidential-disclosure", | |
| "financial:counterfactual", | |
| "financial:data-leakage", | |
| "financial:defamation", | |
| "financial:hallucination", | |
| "financial:impartiality", | |
| "financial:misconduct", | |
| "financial:sycophancy", | |
| "foundation", | |
| "guardrails-eval", | |
| "hallucination", | |
| "harmbench", | |
| "harmful", | |
| "harmful:chemical-biological-weapons", | |
| "harmful:child-exploitation", | |
| "harmful:copyright-violations", | |
| "harmful:cybercrime", | |
| "harmful:cybercrime:malicious-code", | |
| "harmful:graphic-content", | |
| "harmful:harassment-bullying", | |
| "harmful:hate", | |
| "harmful:illegal-activities", | |
| "harmful:illegal-drugs", | |
| "harmful:illegal-drugs:meth", | |
| "harmful:indiscriminate-weapons", | |
| "harmful:insults", | |
| "harmful:intellectual-property", | |
| "harmful:misinformation-disinformation", | |
| "harmful:non-violent-crime", | |
| "harmful:privacy", | |
| "harmful:profanity", | |
| "harmful:radicalization", | |
| "harmful:self-harm", | |
| "harmful:sex-crime", | |
| "harmful:sexual-content", | |
| "harmful:specialized-advice", | |
| "harmful:unsafe-practices", | |
| "harmful:violent-crime", | |
| "harmful:weapons:ied", | |
| "hijacking", | |
| "illegal-activity", | |
| "imitation", | |
| "indirect-prompt-injection", | |
| "intent", | |
| "iso:42001", | |
| "iso:42001:accountability", | |
| "iso:42001:fairness", | |
| "iso:42001:privacy", | |
| "iso:42001:robustness", | |
| "iso:42001:safety", | |
| "iso:42001:security", | |
| "iso:42001:transparency", | |
| "mcp", | |
| "medical", | |
| "medical:anchoring-bias", | |
| "medical:hallucination", | |
| "medical:incorrect-knowledge", | |
| "medical:off-label-use", | |
| "medical:prioritization-error", | |
| "medical:sycophancy", | |
| "misinformation", | |
| "mitre:atlas", | |
| "mitre:atlas:exfiltration", | |
| "mitre:atlas:impact", | |
| "mitre:atlas:initial-access", | |
| "mitre:atlas:ml-attack-staging", | |
| "mitre:atlas:reconnaissance", | |
| "mitre:atlas:resource-development", | |
| "nist:ai", | |
| "nist:ai:measure", | |
| "nist:ai:measure:1.1", | |
| "nist:ai:measure:1.2", | |
| "nist:ai:measure:2.1", | |
| "nist:ai:measure:2.10", | |
| "nist:ai:measure:2.11", | |
| "nist:ai:measure:2.12", | |
| "nist:ai:measure:2.13", | |
| "nist:ai:measure:2.2", | |
| "nist:ai:measure:2.3", | |
| "nist:ai:measure:2.4", | |
| "nist:ai:measure:2.5", | |
| "nist:ai:measure:2.6", | |
| "nist:ai:measure:2.7", | |
| "nist:ai:measure:2.8", | |
| "nist:ai:measure:2.9", | |
| "nist:ai:measure:3.1", | |
| "nist:ai:measure:3.2", | |
| "nist:ai:measure:3.3", | |
| "nist:ai:measure:4.1", | |
| "nist:ai:measure:4.2", | |
| "nist:ai:measure:4.3", | |
| "off-topic", | |
| "overreliance", | |
| "owasp:agentic:t01", | |
| "owasp:api", | |
| "owasp:api:01", | |
| "owasp:api:02", | |
| "owasp:api:03", | |
| "owasp:api:04", | |
| "owasp:api:05", | |
| "owasp:api:06", | |
| "owasp:api:07", | |
| "owasp:api:08", | |
| "owasp:api:09", | |
| "owasp:api:10", | |
| "owasp:llm", | |
| "owasp:llm:01", | |
| "owasp:llm:02", | |
| "owasp:llm:03", | |
| "owasp:llm:04", | |
| "owasp:llm:05", | |
| "owasp:llm:06", | |
| "owasp:llm:07", | |
| "owasp:llm:08", | |
| "owasp:llm:09", | |
| "owasp:llm:10", | |
| "owasp:llm:redteam:implementation", | |
| "owasp:llm:redteam:model", | |
| "owasp:llm:redteam:runtime", | |
| "owasp:llm:redteam:system", | |
| "personal-safety", | |
| "pii", | |
| "pii:api-db", | |
| "pii:direct", | |
| "pii:session", | |
| "pii:social", | |
| "pliny", | |
| "policy", | |
| "politics", | |
| "prompt-extraction", | |
| "rag-document-exfiltration", | |
| "rag-poisoning", | |
| "rbac", | |
| "reasoning-dos", | |
| "religion", | |
| "shell-injection", | |
| "special-token-injection", | |
| "sql-injection", | |
| "ssrf", | |
| "system-prompt-override", | |
| "tool-discovery", | |
| "tool-discovery:multi-turn", | |
| "toxic-chat", | |
| "toxicity", | |
| "unsafebench", | |
| "unverifiable-claims", | |
| "vlguard", | |
| "xstest" | |
| ] | |
| }, | |
| { | |
| "type": "string" | |
| } | |
| ], | |
| "description": "Name of the plugin or path to custom plugin" | |
| }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "id": { | |
| "anyOf": [ | |
| { | |
| "type": "string", | |
| "enum": [ | |
| "aegis", | |
| "agentic:memory-poisoning", | |
| "ascii-smuggling", | |
| "beavertails", | |
| "bfla", | |
| "bias", | |
| "bias:age", | |
| "bias:disability", | |
| "bias:gender", | |
| "bias:race", | |
| "bola", | |
| "cca", | |
| "competitors", | |
| "contracts", | |
| "cross-session-leak", | |
| "cyberseceval", | |
| "debug-access", | |
| "default", | |
| "divergent-repetition", | |
| "donotanswer", | |
| "eu:ai-act", | |
| "eu:ai-act:annex3:biometric-id", | |
| "eu:ai-act:annex3:critical-infrastructure", | |
| "eu:ai-act:annex3:education", | |
| "eu:ai-act:annex3:employment", | |
| "eu:ai-act:annex3:essential-services", | |
| "eu:ai-act:annex3:justice-democracy", | |
| "eu:ai-act:annex3:law-enforcement", | |
| "eu:ai-act:annex3:migration-border", | |
| "eu:ai-act:art5:biometric-categorisation", | |
| "eu:ai-act:art5:exploitation-of-vulnerabilities", | |
| "eu:ai-act:art5:remote-biometric-id-live", | |
| "eu:ai-act:art5:remote-biometric-id-post", | |
| "eu:ai-act:art5:social-scoring", | |
| "eu:ai-act:art5:subliminal-manipulation", | |
| "excessive-agency", | |
| "financial:calculation-error", | |
| "financial:compliance-violation", | |
| "financial:confidential-disclosure", | |
| "financial:counterfactual", | |
| "financial:data-leakage", | |
| "financial:defamation", | |
| "financial:hallucination", | |
| "financial:impartiality", | |
| "financial:misconduct", | |
| "financial:sycophancy", | |
| "foundation", | |
| "guardrails-eval", | |
| "hallucination", | |
| "harmbench", | |
| "harmful", | |
| "harmful:chemical-biological-weapons", | |
| "harmful:child-exploitation", | |
| "harmful:copyright-violations", | |
| "harmful:cybercrime", | |
| "harmful:cybercrime:malicious-code", | |
| "harmful:graphic-content", | |
| "harmful:harassment-bullying", | |
| "harmful:hate", | |
| "harmful:illegal-activities", | |
| "harmful:illegal-drugs", | |
| "harmful:illegal-drugs:meth", | |
| "harmful:indiscriminate-weapons", | |
| "harmful:insults", | |
| "harmful:intellectual-property", | |
| "harmful:misinformation-disinformation", | |
| "harmful:non-violent-crime", | |
| "harmful:privacy", | |
| "harmful:profanity", | |
| "harmful:radicalization", | |
| "harmful:self-harm", | |
| "harmful:sex-crime", | |
| "harmful:sexual-content", | |
| "harmful:specialized-advice", | |
| "harmful:unsafe-practices", | |
| "harmful:violent-crime", | |
| "harmful:weapons:ied", | |
| "hijacking", | |
| "illegal-activity", | |
| "imitation", | |
| "indirect-prompt-injection", | |
| "intent", | |
| "iso:42001", | |
| "iso:42001:accountability", | |
| "iso:42001:fairness", | |
| "iso:42001:privacy", | |
| "iso:42001:robustness", | |
| "iso:42001:safety", | |
| "iso:42001:security", | |
| "iso:42001:transparency", | |
| "mcp", | |
| "medical", | |
| "medical:anchoring-bias", | |
| "medical:hallucination", | |
| "medical:incorrect-knowledge", | |
| "medical:off-label-use", | |
| "medical:prioritization-error", | |
| "medical:sycophancy", | |
| "misinformation", | |
| "mitre:atlas", | |
| "mitre:atlas:exfiltration", | |
| "mitre:atlas:impact", | |
| "mitre:atlas:initial-access", | |
| "mitre:atlas:ml-attack-staging", | |
| "mitre:atlas:reconnaissance", | |
| "mitre:atlas:resource-development", | |
| "nist:ai", | |
| "nist:ai:measure", | |
| "nist:ai:measure:1.1", | |
| "nist:ai:measure:1.2", | |
| "nist:ai:measure:2.1", | |
| "nist:ai:measure:2.10", | |
| "nist:ai:measure:2.11", | |
| "nist:ai:measure:2.12", | |
| "nist:ai:measure:2.13", | |
| "nist:ai:measure:2.2", | |
| "nist:ai:measure:2.3", | |
| "nist:ai:measure:2.4", | |
| "nist:ai:measure:2.5", | |
| "nist:ai:measure:2.6", | |
| "nist:ai:measure:2.7", | |
| "nist:ai:measure:2.8", | |
| "nist:ai:measure:2.9", | |
| "nist:ai:measure:3.1", | |
| "nist:ai:measure:3.2", | |
| "nist:ai:measure:3.3", | |
| "nist:ai:measure:4.1", | |
| "nist:ai:measure:4.2", | |
| "nist:ai:measure:4.3", | |
| "off-topic", | |
| "overreliance", | |
| "owasp:agentic:t01", | |
| "owasp:api", | |
| "owasp:api:01", | |
| "owasp:api:02", | |
| "owasp:api:03", | |
| "owasp:api:04", | |
| "owasp:api:05", | |
| "owasp:api:06", | |
| "owasp:api:07", | |
| "owasp:api:08", | |
| "owasp:api:09", | |
| "owasp:api:10", | |
| "owasp:llm", | |
| "owasp:llm:01", | |
| "owasp:llm:02", | |
| "owasp:llm:03", | |
| "owasp:llm:04", | |
| "owasp:llm:05", | |
| "owasp:llm:06", | |
| "owasp:llm:07", | |
| "owasp:llm:08", | |
| "owasp:llm:09", | |
| "owasp:llm:10", | |
| "owasp:llm:redteam:implementation", | |
| "owasp:llm:redteam:model", | |
| "owasp:llm:redteam:runtime", | |
| "owasp:llm:redteam:system", | |
| "personal-safety", | |
| "pii", | |
| "pii:api-db", | |
| "pii:direct", | |
| "pii:session", | |
| "pii:social", | |
| "pliny", | |
| "policy", | |
| "politics", | |
| "prompt-extraction", | |
| "rag-document-exfiltration", | |
| "rag-poisoning", | |
| "rbac", | |
| "reasoning-dos", | |
| "religion", | |
| "shell-injection", | |
| "special-token-injection", | |
| "sql-injection", | |
| "ssrf", | |
| "system-prompt-override", | |
| "tool-discovery", | |
| "tool-discovery:multi-turn", | |
| "toxic-chat", | |
| "toxicity", | |
| "unsafebench", | |
| "unverifiable-claims", | |
| "vlguard", | |
| "xstest" | |
| ] | |
| }, | |
| { | |
| "type": "string" | |
| } | |
| ], | |
| "description": "Name of the plugin" | |
| }, | |
| "numTests": { | |
| "type": "integer", | |
| "exclusiveMinimum": 0, | |
| "default": 5, | |
| "description": "Number of tests to generate for this plugin" | |
| }, | |
| "config": { | |
| "type": "object", | |
| "additionalProperties": {}, | |
| "description": "Plugin-specific configuration" | |
| }, | |
| "severity": { | |
| "type": "string", | |
| "enum": ["critical", "high", "medium", "low"], | |
| "description": "Severity level for this plugin" | |
| } | |
| }, | |
| "required": ["id"], | |
| "additionalProperties": false | |
| } | |
| ] | |
| }, | |
| "description": "Plugins to use for redteam generation", | |
| "default": ["default"] | |
| }, | |
| "strategies": { | |
| "type": "array", | |
| "items": { | |
| "anyOf": [ | |
| { | |
| "anyOf": [ | |
| { | |
| "type": "string", | |
| "enum": [ | |
| "audio", | |
| "base64", | |
| "basic", | |
| "best-of-n", | |
| "camelcase", | |
| "citation", | |
| "crescendo", | |
| "custom", | |
| "default", | |
| "emoji", | |
| "gcg", | |
| "goat", | |
| "hex", | |
| "homoglyph", | |
| "image", | |
| "jailbreak", | |
| "jailbreak:composite", | |
| "jailbreak:likert", | |
| "jailbreak:tree", | |
| "layer", | |
| "leetspeak", | |
| "math-prompt", | |
| "mischievous-user", | |
| "morse", | |
| "multilingual", | |
| "other-encodings", | |
| "piglatin", | |
| "prompt-injection", | |
| "retry", | |
| "rot13", | |
| "video" | |
| ] | |
| }, | |
| { | |
| "type": "string" | |
| }, | |
| { | |
| "type": "string" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "id": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/redteam/properties/strategies/items/anyOf/0" | |
| }, | |
| "config": { | |
| "type": "object", | |
| "additionalProperties": {}, | |
| "description": "Strategy-specific configuration" | |
| } | |
| }, | |
| "required": ["id"], | |
| "additionalProperties": false | |
| } | |
| ] | |
| }, | |
| "description": "Strategies to use for redteam generation.\n\nDefaults to basic, jailbreak, jailbreak:composite\nSupports audio, base64, basic, best-of-n, camelcase, citation, crescendo, custom, default, emoji, gcg, goat, hex, homoglyph, image, jailbreak, jailbreak:composite, jailbreak:likert, jailbreak:tree, layer, leetspeak, math-prompt, mischievous-user, morse, multilingual, other-encodings, piglatin, prompt-injection, retry, rot13, video", | |
| "default": ["default"] | |
| }, | |
| "maxConcurrency": { | |
| "type": "integer", | |
| "exclusiveMinimum": 0, | |
| "description": "Maximum number of concurrent API calls" | |
| }, | |
| "delay": { | |
| "type": "integer", | |
| "minimum": 0, | |
| "description": "Delay in milliseconds between plugin API calls" | |
| }, | |
| "excludeTargetOutputFromAgenticAttackGeneration": { | |
| "type": "boolean", | |
| "description": "Whether to exclude target output from the agentific attack generation process" | |
| } | |
| }, | |
| "additionalProperties": false | |
| }, | |
| "writeLatestResults": { | |
| "type": "boolean" | |
| }, | |
| "tracing": { | |
| "type": "object", | |
| "properties": { | |
| "enabled": { | |
| "type": "boolean", | |
| "default": false | |
| }, | |
| "otlp": { | |
| "type": "object", | |
| "properties": { | |
| "http": { | |
| "type": "object", | |
| "properties": { | |
| "enabled": { | |
| "type": "boolean", | |
| "default": true | |
| }, | |
| "port": { | |
| "type": "number", | |
| "default": 4318 | |
| }, | |
| "host": { | |
| "type": "string", | |
| "default": "0.0.0.0" | |
| }, | |
| "acceptFormats": { | |
| "type": "array", | |
| "items": { | |
| "type": "string", | |
| "enum": ["protobuf", "json"] | |
| }, | |
| "default": ["json"] | |
| } | |
| }, | |
| "additionalProperties": false | |
| }, | |
| "grpc": { | |
| "type": "object", | |
| "properties": { | |
| "enabled": { | |
| "type": "boolean", | |
| "default": false | |
| }, | |
| "port": { | |
| "type": "number", | |
| "default": 4317 | |
| } | |
| }, | |
| "additionalProperties": false | |
| } | |
| }, | |
| "additionalProperties": false | |
| }, | |
| "storage": { | |
| "type": "object", | |
| "properties": { | |
| "type": { | |
| "type": "string", | |
| "enum": ["sqlite"], | |
| "default": "sqlite" | |
| }, | |
| "retentionDays": { | |
| "type": "number", | |
| "default": 30 | |
| } | |
| }, | |
| "additionalProperties": false | |
| }, | |
| "forwarding": { | |
| "type": "object", | |
| "properties": { | |
| "enabled": { | |
| "type": "boolean", | |
| "default": false | |
| }, | |
| "endpoint": { | |
| "type": "string" | |
| }, | |
| "headers": { | |
| "type": "object", | |
| "additionalProperties": { | |
| "type": "string" | |
| } | |
| } | |
| }, | |
| "required": ["endpoint"], | |
| "additionalProperties": false | |
| } | |
| }, | |
| "additionalProperties": false | |
| }, | |
| "assertionTemplates": { | |
| "type": "object", | |
| "additionalProperties": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/tests/anyOf/1/items/anyOf/1/properties/assert/items/anyOf/0/properties/assert/items" | |
| } | |
| }, | |
| "evaluateOptions": { | |
| "type": "object", | |
| "properties": { | |
| "cache": { | |
| "type": "boolean" | |
| }, | |
| "delay": { | |
| "type": "number" | |
| }, | |
| "eventSource": { | |
| "type": "string" | |
| }, | |
| "generateSuggestions": { | |
| "type": "boolean" | |
| }, | |
| "interactiveProviders": { | |
| "type": "boolean" | |
| }, | |
| "maxConcurrency": { | |
| "type": "number" | |
| }, | |
| "repeat": { | |
| "type": "number" | |
| }, | |
| "showProgressBar": { | |
| "type": "boolean" | |
| }, | |
| "timeoutMs": { | |
| "type": "number" | |
| }, | |
| "maxEvalTimeMs": { | |
| "type": "number" | |
| }, | |
| "isRedteam": { | |
| "type": "boolean" | |
| } | |
| }, | |
| "additionalProperties": false | |
| }, | |
| "commandLineOptions": { | |
| "type": "object", | |
| "properties": { | |
| "description": { | |
| "type": "string" | |
| }, | |
| "prompts": { | |
| "type": "array", | |
| "items": { | |
| "type": "string" | |
| } | |
| }, | |
| "providers": { | |
| "type": "array", | |
| "items": { | |
| "type": "string" | |
| } | |
| }, | |
| "output": { | |
| "type": "array", | |
| "items": { | |
| "type": "string" | |
| } | |
| }, | |
| "maxConcurrency": { | |
| "type": "integer", | |
| "exclusiveMinimum": 0 | |
| }, | |
| "repeat": { | |
| "type": "integer", | |
| "exclusiveMinimum": 0 | |
| }, | |
| "delay": { | |
| "type": "integer", | |
| "minimum": 0, | |
| "default": 0 | |
| }, | |
| "vars": { | |
| "type": "string" | |
| }, | |
| "tests": { | |
| "type": "string" | |
| }, | |
| "config": { | |
| "type": "array", | |
| "items": { | |
| "type": "string" | |
| } | |
| }, | |
| "assertions": { | |
| "type": "string" | |
| }, | |
| "modelOutputs": { | |
| "type": "string" | |
| }, | |
| "verbose": { | |
| "type": "boolean" | |
| }, | |
| "grader": { | |
| "type": "string" | |
| }, | |
| "tableCellMaxLength": { | |
| "type": "integer", | |
| "exclusiveMinimum": 0 | |
| }, | |
| "write": { | |
| "type": "boolean" | |
| }, | |
| "cache": { | |
| "type": "boolean" | |
| }, | |
| "table": { | |
| "type": "boolean" | |
| }, | |
| "share": { | |
| "type": "boolean" | |
| }, | |
| "noShare": { | |
| "type": "boolean" | |
| }, | |
| "progressBar": { | |
| "type": "boolean" | |
| }, | |
| "watch": { | |
| "type": "boolean" | |
| }, | |
| "filterErrorsOnly": { | |
| "type": "string" | |
| }, | |
| "filterFailing": { | |
| "type": "string" | |
| }, | |
| "filterFirstN": { | |
| "type": "integer", | |
| "exclusiveMinimum": 0 | |
| }, | |
| "filterMetadata": { | |
| "type": "string" | |
| }, | |
| "filterPattern": { | |
| "type": "string" | |
| }, | |
| "filterProviders": { | |
| "type": "string" | |
| }, | |
| "filterSample": { | |
| "type": "integer", | |
| "exclusiveMinimum": 0 | |
| }, | |
| "filterTargets": { | |
| "type": "string" | |
| }, | |
| "var": { | |
| "type": "object", | |
| "additionalProperties": { | |
| "type": "string" | |
| } | |
| }, | |
| "generateSuggestions": { | |
| "type": "boolean" | |
| }, | |
| "promptPrefix": { | |
| "type": "string" | |
| }, | |
| "promptSuffix": { | |
| "type": "string" | |
| }, | |
| "retryErrors": { | |
| "type": "boolean" | |
| }, | |
| "envPath": { | |
| "type": "string" | |
| } | |
| }, | |
| "additionalProperties": false | |
| }, | |
| "targets": { | |
| "$ref": "#/definitions/PromptfooConfigSchema/properties/providers" | |
| } | |
| }, | |
| "required": ["prompts"], | |
| "additionalProperties": false | |
| } | |
| }, | |
| "$schema": "http://json-schema.org/draft-07/schema#" | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment