{"name":"evals.blah.dev API Specification","version":"0.1.1","inference":{"description":"Model endpoints must accept POST requests with the request schema and return the response schema.","request":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"prompt":{"description":"The prompt to send to the model","type":"string"},"max_tokens":{"description":"Maximum tokens to generate","default":2048,"type":"integer","exclusiveMinimum":0,"maximum":9007199254740991},"temperature":{"description":"Sampling temperature","default":0.7,"type":"number","minimum":0,"maximum":2}},"required":["prompt","max_tokens","temperature"],"additionalProperties":false},"response":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"text":{"description":"The generated text response","type":"string"},"model":{"description":"Model identifier","type":"string"},"usage":{"description":"Token usage statistics","type":"object","properties":{"prompt_tokens":{"type":"number"},"completion_tokens":{"type":"number"}},"additionalProperties":false}},"required":["text"],"additionalProperties":false}},"models":{"create":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"name":{"description":"Model display name","type":"string","minLength":1,"maxLength":100},"alias":{"description":"Short name used in the chatroom, e.g. 'alpha'. Falls back to name when unset.","anyOf":[{"type":"string","maxLength":20,"pattern":"^[a-z0-9][a-z0-9._-]*$"},{"type":"null"}]},"description":{"description":"Model description","default":"","type":"string","maxLength":1000},"inference_uri":{"description":"HTTP endpoint URL or AI SDK provider/model (e.g. 'anthropic/claude-sonnet-4-6', 'openai/gpt-4o')","type":"string","minLength":1},"api_key":{"description":"API key for the inference endpoint or provider","type":"string"},"is_official":{"description":"Whether this is an official model","default":false,"type":"boolean"},"default_temperature":{"description":"Default sampling temperature (0.00–1.10)","default":0.7,"type":"number","minimum":0,"maximum":1.1},"default_top_k":{"description":"Default top-k sampling (0.00–1.00)","default":0,"type":"number","minimum":0,"maximum":1},"default_max_tokens":{"description":"Maximum token limit (250–250,000)","default":16384,"type":"integer","minimum":250,"maximum":250000}},"required":["name","description","inference_uri","is_official","default_temperature","default_top_k","default_max_tokens"],"additionalProperties":false}},"evals":{"create":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"name":{"description":"Eval display name","type":"string","minLength":1,"maxLength":100},"description":{"description":"Eval description","default":"","type":"string","maxLength":1000},"prompt":{"description":"The prompt to send to models","type":"string","minLength":1},"expected_behavior":{"description":"Human-readable description of expected behavior","default":"","type":"string","maxLength":2000},"eval_type":{"type":"string","enum":["semantic","rubric","sycophancy"]},"eval_criteria":{"description":"JSON-encoded eval criteria","type":"string"}},"required":["name","description","prompt","expected_behavior","eval_type","eval_criteria"],"additionalProperties":false},"criteria":{"semantic":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"ideal_response":{"description":"The ideal response to compare against","type":"string"},"rubric":{"description":"Instructions for the LLM judge","default":"Rate how semantically similar the response is to the ideal response on a scale of 0-1.","type":"string"}},"required":["ideal_response","rubric"],"additionalProperties":false},"rubric":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"rubric":{"description":"Detailed grading rubric for the LLM judge","type":"string"},"max_score":{"description":"Maximum score","default":1,"type":"number","exclusiveMinimum":0}},"required":["rubric","max_score"],"additionalProperties":false}}},"artifacts":{"create":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":200},"description":{"default":"","type":"string","maxLength":2000},"artifact_type":{"default":"other","type":"string","enum":["weights","config","checkpoint","logs","dataset","other"]},"version":{"default":"","type":"string","maxLength":100}},"required":["name","description","artifact_type","version"],"additionalProperties":false},"update":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":200},"description":{"type":"string","maxLength":2000},"artifact_type":{"type":"string","enum":["weights","config","checkpoint","logs","dataset","other"]},"version":{"type":"string","maxLength":100}},"additionalProperties":false},"artifactType":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"string","enum":["weights","config","checkpoint","logs","dataset","other"]}}}