Create Evaluation Config

Python

import os
from scale_gp import SGPClient

client = SGPClient(
    api_key=os.environ.get("SGP_API_KEY"),  # This is the default and can be omitted
)
evaluation_config = client.evaluation_configs.create(
    account_id="account_id",
    question_set_id="question_set_id",
)
print(evaluation_config.id)

package main

import (
	"context"
	"fmt"

	"github.com/stainless-sdks/sgp-go"
	"github.com/stainless-sdks/sgp-go/option"
)

func main() {
	client := sgp.NewClient(
		option.WithAPIKey("My API Key"),
	)
	evaluationConfig, err := client.EvaluationConfigs.New(context.TODO(), sgp.EvaluationConfigNewParams{
		Body: sgp.EvaluationConfigNewParamsBodyAutoEvalEvaluationConfigRequest{
			AccountID:      sgp.F("account_id"),
			QuestionSetID:  sgp.F("question_set_id"),
			EvaluationType: sgp.F(sgp.EvaluationConfigNewParamsBodyAutoEvalEvaluationConfigRequestEvaluationTypeLlmAuto),
		},
	})
	if err != nil {
		panic(err.Error())
	}
	fmt.Printf("%+v\n", evaluationConfig.ID)
}

curl --request POST \
  --url https://api.egp.scale.com/v4/evaluation-configs \
  --header 'Content-Type: application/json' \
  --data '
{
  "question_set_id": "<string>",
  "account_id": "<string>",
  "evaluation_type": "llm_auto",
  "studio_project_id": "<string>",
  "auto_evaluation_model": "gpt-4-turbo-2024-04-09",
  "auto_evaluation_parameters": {
    "temperature": 1,
    "batch_size": 13
  }
}
'

const options = {
  method: 'POST',
  headers: {'Content-Type': 'application/json'},
  body: JSON.stringify({
    question_set_id: '<string>',
    account_id: '<string>',
    evaluation_type: 'llm_auto',
    studio_project_id: '<string>',
    auto_evaluation_model: 'gpt-4-turbo-2024-04-09',
    auto_evaluation_parameters: {temperature: 1, batch_size: 13}
  })
};

fetch('https://api.egp.scale.com/v4/evaluation-configs', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.egp.scale.com/v4/evaluation-configs",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'question_set_id' => '<string>',
    'account_id' => '<string>',
    'evaluation_type' => 'llm_auto',
    'studio_project_id' => '<string>',
    'auto_evaluation_model' => 'gpt-4-turbo-2024-04-09',
    'auto_evaluation_parameters' => [
        'temperature' => 1,
        'batch_size' => 13
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

HttpResponse<String> response = Unirest.post("https://api.egp.scale.com/v4/evaluation-configs")
  .header("Content-Type", "application/json")
  .body("{\n  \"question_set_id\": \"<string>\",\n  \"account_id\": \"<string>\",\n  \"evaluation_type\": \"llm_auto\",\n  \"studio_project_id\": \"<string>\",\n  \"auto_evaluation_model\": \"gpt-4-turbo-2024-04-09\",\n  \"auto_evaluation_parameters\": {\n    \"temperature\": 1,\n    \"batch_size\": 13\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.egp.scale.com/v4/evaluation-configs")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Content-Type"] = 'application/json'
request.body = "{\n  \"question_set_id\": \"<string>\",\n  \"account_id\": \"<string>\",\n  \"evaluation_type\": \"llm_auto\",\n  \"studio_project_id\": \"<string>\",\n  \"auto_evaluation_model\": \"gpt-4-turbo-2024-04-09\",\n  \"auto_evaluation_parameters\": {\n    \"temperature\": 1,\n    \"batch_size\": 13\n  }\n}"

response = http.request(request)
puts response.read_body

{
  "evaluation_type": "studio",
  "question_set_id": "<string>",
  "id": "<string>",
  "created_at": "2023-11-07T05:31:56Z",
  "account_id": "<string>",
  "created_by_user_id": "<string>",
  "created_by_identity_type": "user",
  "studio_project_id": "<string>",
  "auto_evaluation_model": "gpt-4-32k-0613",
  "auto_evaluation_parameters": {
    "temperature": 1,
    "batch_size": 13
  }
}

{
  "detail": [
    {
      "loc": [
        "<string>"
      ],
      "msg": "<string>",
      "type": "<string>",
      "input": "<unknown>",
      "ctx": {}
    }
  ]
}

POST

evaluation-configs

Python

import os
from scale_gp import SGPClient

client = SGPClient(
    api_key=os.environ.get("SGP_API_KEY"),  # This is the default and can be omitted
)
evaluation_config = client.evaluation_configs.create(
    account_id="account_id",
    question_set_id="question_set_id",
)
print(evaluation_config.id)

package main

import (
	"context"
	"fmt"

	"github.com/stainless-sdks/sgp-go"
	"github.com/stainless-sdks/sgp-go/option"
)

func main() {
	client := sgp.NewClient(
		option.WithAPIKey("My API Key"),
	)
	evaluationConfig, err := client.EvaluationConfigs.New(context.TODO(), sgp.EvaluationConfigNewParams{
		Body: sgp.EvaluationConfigNewParamsBodyAutoEvalEvaluationConfigRequest{
			AccountID:      sgp.F("account_id"),
			QuestionSetID:  sgp.F("question_set_id"),
			EvaluationType: sgp.F(sgp.EvaluationConfigNewParamsBodyAutoEvalEvaluationConfigRequestEvaluationTypeLlmAuto),
		},
	})
	if err != nil {
		panic(err.Error())
	}
	fmt.Printf("%+v\n", evaluationConfig.ID)
}

curl --request POST \
  --url https://api.egp.scale.com/v4/evaluation-configs \
  --header 'Content-Type: application/json' \
  --data '
{
  "question_set_id": "<string>",
  "account_id": "<string>",
  "evaluation_type": "llm_auto",
  "studio_project_id": "<string>",
  "auto_evaluation_model": "gpt-4-turbo-2024-04-09",
  "auto_evaluation_parameters": {
    "temperature": 1,
    "batch_size": 13
  }
}
'

const options = {
  method: 'POST',
  headers: {'Content-Type': 'application/json'},
  body: JSON.stringify({
    question_set_id: '<string>',
    account_id: '<string>',
    evaluation_type: 'llm_auto',
    studio_project_id: '<string>',
    auto_evaluation_model: 'gpt-4-turbo-2024-04-09',
    auto_evaluation_parameters: {temperature: 1, batch_size: 13}
  })
};

fetch('https://api.egp.scale.com/v4/evaluation-configs', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.egp.scale.com/v4/evaluation-configs",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'question_set_id' => '<string>',
    'account_id' => '<string>',
    'evaluation_type' => 'llm_auto',
    'studio_project_id' => '<string>',
    'auto_evaluation_model' => 'gpt-4-turbo-2024-04-09',
    'auto_evaluation_parameters' => [
        'temperature' => 1,
        'batch_size' => 13
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

HttpResponse<String> response = Unirest.post("https://api.egp.scale.com/v4/evaluation-configs")
  .header("Content-Type", "application/json")
  .body("{\n  \"question_set_id\": \"<string>\",\n  \"account_id\": \"<string>\",\n  \"evaluation_type\": \"llm_auto\",\n  \"studio_project_id\": \"<string>\",\n  \"auto_evaluation_model\": \"gpt-4-turbo-2024-04-09\",\n  \"auto_evaluation_parameters\": {\n    \"temperature\": 1,\n    \"batch_size\": 13\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.egp.scale.com/v4/evaluation-configs")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Content-Type"] = 'application/json'
request.body = "{\n  \"question_set_id\": \"<string>\",\n  \"account_id\": \"<string>\",\n  \"evaluation_type\": \"llm_auto\",\n  \"studio_project_id\": \"<string>\",\n  \"auto_evaluation_model\": \"gpt-4-turbo-2024-04-09\",\n  \"auto_evaluation_parameters\": {\n    \"temperature\": 1,\n    \"batch_size\": 13\n  }\n}"

response = http.request(request)
puts response.read_body

{
  "evaluation_type": "studio",
  "question_set_id": "<string>",
  "id": "<string>",
  "created_at": "2023-11-07T05:31:56Z",
  "account_id": "<string>",
  "created_by_user_id": "<string>",
  "created_by_identity_type": "user",
  "studio_project_id": "<string>",
  "auto_evaluation_model": "gpt-4-32k-0613",
  "auto_evaluation_parameters": {
    "temperature": 1,
    "batch_size": 13
  }
}

{
  "detail": [
    {
      "loc": [
        "<string>"
      ],
      "msg": "<string>",
      "type": "<string>",
      "input": "<unknown>",
      "ctx": {}
    }
  ]
}

Body

application/json

AutoEvalEvaluationConfigRequest
ManualEvaluationConfigRequest

question_set_id

string

required

account_id

string

required

The ID of the account that owns the given entity.

evaluation_type

default:llm_auto

Evaluation type

Allowed value: "llm_auto"

studio_project_id

string

auto_evaluation_model

enum<string>

default:gpt-4-turbo-2024-04-09

The name of the model to be used for auto-evaluation

Available options:

llama-3-1-70b-instruct,

gpt-4-turbo-2024-04-09,

llama-3-70b-instruct-bedrock,

gpt-4o,

gpt-4o-mini,

gpt-4.1,

gpt-4.1-mini,

gpt-4.1-nano,

gpt-5-nano,

gpt-5-mini,

gpt-5,

gpt-5.1,

gpt-5.2,

o1,

o3,

o3-mini,

o4-mini

auto_evaluation_parameters

AutoEvaluationParameters · object

Execution parameters for auto-evaluation

Show child attributes

Response

Successful Response

evaluation_type

enum<string>

required

Evaluation type

Available options:

studio,

llm_auto,

human,

llm_benchmark

question_set_id

string

required

string

required

The unique identifier of the entity.

created_at

string<date-time>

required

The date and time when the entity was created in ISO format.

account_id

string

required

The ID of the account that owns the given entity.

created_by_user_id

string

required

The user who originally created the entity.

created_by_identity_type

enum<string>

required

The type of identity that created the entity.

Available options:

user,

service_account

studio_project_id

string

auto_evaluation_model

enum<string>

The name of the model to be used for auto-evaluation

Available options:

gpt-4-32k-0613,

gpt-4-turbo-preview,

gpt-4-turbo-2024-04-09,

gpt-4o-2024-05-13,

gpt-4o,

gpt-4o-mini-2024-07-18,

gpt-4o-mini,

gpt-4.1,

gpt-4.1-mini,

gpt-4.1-nano,

gpt-5-nano,

gpt-5-mini,

gpt-5,

gpt-5.1,

gpt-5.2,

o1,

o1-mini,

o3,

o3-mini,

o3-mini-2025-01-31,

o4-mini,

gpt-oss-120b,

gpt-oss-20b,

llama-3-70b-instruct,

llama-3-1-70b-instruct,

llama-3-70b-instruct-bedrock

auto_evaluation_parameters

AutoEvaluationParameters · object

Execution parameters for auto-evaluation

Show child attributes

Knowledge Bases

Chunks

Agents

Completions

Chat Completions

Models

Users

Accounts

Organizations

Question Sets

Evaluations

Evaluation Configs

Evaluation Datasets

Studio Projects

Application Specs

Questions

Knowledge Base Data Sources

Model Templates V3 (Beta)

Model server

API Reference

Fine Tuning Jobs V3 (Beta)

Training Datasets V3 (Beta)

package deployments

Beta

Applications

ChatThreads

Interactions

MonitoringDashboard

Chat Themes

Sessions

account groups

Create Evaluation Config

Description

Details

Body

Response