Conversational Analysis Authoring - Get Model Evaluation Summary
Gets the evaluation summary of a trained model. The summary includes high level performance measurements of the model e.g., F1, Precision, Recall, etc.
GET {Endpoint}/language/authoring/analyze-conversations/projects/{projectName}/models/{trainedModelLabel}/evaluation/summary-result?api-version=2023-04-01URI Parameters
| Name | In | Required | Type | Description |
|---|---|---|---|---|
|
Endpoint
|
path | True |
string |
|
|
project
|
path | True |
string |
The name of the project to use. |
|
trained
|
path | True |
string |
The trained model label. |
|
api-version
|
query | True |
string |
The API version to use for this operation. |
Responses
| Name | Type | Description |
|---|---|---|
| 200 OK |
The request has succeeded. |
|
| Other Status Codes |
An unexpected error response. |
Security
Ocp-Apim-Subscription-Key
Type:
apiKey
In:
header
OAuth2Auth
Type:
oauth2
Flow:
accessCode
Authorization URL:
https://login.microsoftonline.com/common/oauth2/authorize
Token URL:
https://login.microsoftonline.com/common/oauth2/token
Scopes
| Name | Description |
|---|---|
| https://cognitiveservices.azure.com/.default |
Examples
Successful Get Model Evaluation Summary
Sample request
GET {Endpoint}/language/authoring/analyze-conversations/projects/EmailApp/models/model1/evaluation/summary-result?api-version=2023-04-01
Sample response
{
"entitiesEvaluation": {
"confusionMatrix": {
"Attachment": {
"Attachment": {
"normalizedValue": 100,
"rawValue": 3
}
},
"Category": {
"Category": {
"normalizedValue": 91.666664,
"rawValue": 11
},
"$none": {
"normalizedValue": 8.333333,
"rawValue": 1
}
},
"ContactName": {
"ContactName": {
"normalizedValue": 91.666664,
"rawValue": 22
},
"SenderName": {
"normalizedValue": 4.1666665,
"rawValue": 1
},
"$none": {
"normalizedValue": 4.1666665,
"rawValue": 1
}
},
"Date": {
"Date": {
"normalizedValue": 100,
"rawValue": 2
}
},
"EmailSubject": {
"EmailSubject": {
"normalizedValue": 93.33333,
"rawValue": 9.333334
},
"$none": {
"normalizedValue": 6.6666665,
"rawValue": 0.6666667
}
},
"FromRelationshipName": {
"FromRelationshipName": {
"normalizedValue": 100,
"rawValue": 1
}
},
"Line": {
"Line": {
"normalizedValue": 100,
"rawValue": 2
}
},
"Message": {
"Message": {
"normalizedValue": 81.2063,
"rawValue": 6.496504
},
"EmailSubject": {
"normalizedValue": 7.43007,
"rawValue": 0.5944056
},
"$none": {
"normalizedValue": 9.120047,
"rawValue": 0.7296037
},
"Date": {
"normalizedValue": 2.2435899,
"rawValue": 0.17948718
}
},
"OrderReference": {
"OrderReference": {
"normalizedValue": 100,
"rawValue": 17
}
},
"PositionReference": {
"$none": {
"normalizedValue": 100,
"rawValue": 1
}
},
"RelationshipName": {
"RelationshipName": {
"normalizedValue": 66.666664,
"rawValue": 2
},
"$none": {
"normalizedValue": 33.333332,
"rawValue": 1
}
},
"SearchTexts": {
"SearchTexts": {
"normalizedValue": 100,
"rawValue": 4
}
},
"SenderName": {
"SenderName": {
"normalizedValue": 88.888885,
"rawValue": 8
},
"ContactName": {
"normalizedValue": 11.111111,
"rawValue": 1
}
},
"Time": {
"$none": {
"normalizedValue": 100,
"rawValue": 2
}
},
"$none": {
"$none": {
"normalizedValue": 99.739265,
"rawValue": 162.575
},
"Category": {
"normalizedValue": 0.2607362,
"rawValue": 0.425
}
}
},
"entities": {
"ContactName": {
"f1": 0.9361702799797058,
"precision": 0.95652174949646,
"recall": 0.9166666865348816,
"truePositiveCount": 22,
"trueNegativeCount": 0,
"falsePositiveCount": 1,
"falseNegativeCount": 2
},
"Category": {
"f1": 0.8799999952316284,
"precision": 0.8461538553237915,
"recall": 0.9166666865348816,
"truePositiveCount": 11,
"trueNegativeCount": 0,
"falsePositiveCount": 2,
"falseNegativeCount": 1
},
"SenderName": {
"f1": 0.8888888955116272,
"precision": 0.8888888955116272,
"recall": 0.8888888955116272,
"truePositiveCount": 8,
"trueNegativeCount": 0,
"falsePositiveCount": 1,
"falseNegativeCount": 1
},
"EmailSubject": {
"f1": 0.8181817531585693,
"precision": 0.75,
"recall": 0.8999999761581421,
"truePositiveCount": 9,
"trueNegativeCount": 0,
"falsePositiveCount": 3,
"falseNegativeCount": 1
},
"Message": {
"f1": 0.75,
"precision": 0.75,
"recall": 0.75,
"truePositiveCount": 6,
"trueNegativeCount": 0,
"falsePositiveCount": 2,
"falseNegativeCount": 2
},
"Date": {
"f1": 0.800000011920929,
"precision": 0.6666666865348816,
"recall": 1,
"truePositiveCount": 2,
"trueNegativeCount": 0,
"falsePositiveCount": 1,
"falseNegativeCount": 0
},
"OrderReference": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 17,
"trueNegativeCount": 0,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"SearchTexts": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 4,
"trueNegativeCount": 0,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"Attachment": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 3,
"trueNegativeCount": 0,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"RelationshipName": {
"f1": 0.800000011920929,
"precision": 1,
"recall": 0.6666666865348816,
"truePositiveCount": 2,
"trueNegativeCount": 0,
"falsePositiveCount": 0,
"falseNegativeCount": 1
},
"Line": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 2,
"trueNegativeCount": 0,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"Time": {
"f1": 0,
"precision": 0,
"recall": 0,
"truePositiveCount": 0,
"trueNegativeCount": 0,
"falsePositiveCount": 0,
"falseNegativeCount": 2
},
"FromRelationshipName": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 1,
"trueNegativeCount": 0,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"PositionReference": {
"f1": 0,
"precision": 0,
"recall": 0,
"truePositiveCount": 0,
"trueNegativeCount": 0,
"falsePositiveCount": 0,
"falseNegativeCount": 1
}
},
"microF1": 0.8923077,
"microPrecision": 0.8969072,
"microRecall": 0.8877551,
"macroF1": 0.7766601,
"macroPrecision": 0.7755879,
"macroRecall": 0.78849214
},
"intentsEvaluation": {
"confusionMatrix": {
"AddFlag": {
"AddFlag": {
"normalizedValue": 100,
"rawValue": 6
}
},
"AddMore": {
"AddMore": {
"normalizedValue": 100,
"rawValue": 17
}
},
"Cancel": {
"Cancel": {
"normalizedValue": 100,
"rawValue": 9
}
},
"CheckMessages": {
"CheckMessages": {
"normalizedValue": 100,
"rawValue": 9
}
},
"Confirm": {
"Confirm": {
"normalizedValue": 100,
"rawValue": 4
}
},
"Delete": {
"Delete": {
"normalizedValue": 100,
"rawValue": 5
}
},
"Forward": {
"Forward": {
"normalizedValue": 100,
"rawValue": 6
}
},
"None": {
"None": {
"normalizedValue": 100,
"rawValue": 1
}
},
"QueryLastText": {
"QueryLastText": {
"normalizedValue": 100,
"rawValue": 6
}
},
"ReadAloud": {
"ReadAloud": {
"normalizedValue": 100,
"rawValue": 16
}
},
"Reply": {
"Reply": {
"normalizedValue": 100,
"rawValue": 6
}
},
"SearchMessages": {
"SearchMessages": {
"normalizedValue": 100,
"rawValue": 9
}
},
"SendEmail": {
"SendEmail": {
"normalizedValue": 100,
"rawValue": 20
}
},
"ShowNext": {
"ShowNext": {
"normalizedValue": 100,
"rawValue": 4
}
},
"ShowPrevious": {
"ShowPrevious": {
"normalizedValue": 100,
"rawValue": 3
}
}
},
"intents": {
"AddMore": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 17,
"trueNegativeCount": 104,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"Cancel": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 9,
"trueNegativeCount": 112,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"SendEmail": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 20,
"trueNegativeCount": 101,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"CheckMessages": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 9,
"trueNegativeCount": 112,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"AddFlag": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 6,
"trueNegativeCount": 115,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"Reply": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 6,
"trueNegativeCount": 115,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"ReadAloud": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 16,
"trueNegativeCount": 105,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"QueryLastText": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 6,
"trueNegativeCount": 115,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"SearchMessages": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 9,
"trueNegativeCount": 112,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"Delete": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 5,
"trueNegativeCount": 116,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"Forward": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 6,
"trueNegativeCount": 115,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"Confirm": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 4,
"trueNegativeCount": 117,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"ShowNext": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 4,
"trueNegativeCount": 117,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"ShowPrevious": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 3,
"trueNegativeCount": 118,
"falsePositiveCount": 0,
"falseNegativeCount": 0
},
"None": {
"f1": 1,
"precision": 1,
"recall": 1,
"truePositiveCount": 1,
"trueNegativeCount": 120,
"falsePositiveCount": 0,
"falseNegativeCount": 0
}
},
"microF1": 1,
"microPrecision": 1,
"microRecall": 1,
"macroF1": 1,
"macroPrecision": 1,
"macroRecall": 1
},
"evaluationOptions": {
"kind": "percentage",
"trainingSplitPercentage": 80,
"testingSplitPercentage": 20
}
}Definitions
| Name | Description |
|---|---|
|
Conversational |
Represents the evaluation result summary for the project's entities. |
|
Conversational |
Represents the evaluation result for an entity. |
|
Conversational |
Represents the options used running the evaluation. |
|
Conversational |
Represents the evaluation result summary. |
|
Conversational |
The evaluation summary for an intent. |
|
Conversational |
Represents the evaluation summary for the project's intents. |
| Error |
The error object. |
|
Error |
Human-readable error code. |
|
Error |
Error response. |
|
Evaluation |
|
|
Inner |
Human-readable error code. |
|
Inner |
An object containing more specific information about the error. As per Microsoft One API guidelines - https://github.com/Microsoft/api-guidelines/blob/vNext/Guidelines.md#7102-error-condition-responses. |
ConversationalAnalysisAuthoringEntitiesEvaluationSummary
Represents the evaluation result summary for the project's entities.
| Name | Type | Description |
|---|---|---|
| confusionMatrix |
object |
Represents the confusion matrix between two entities (the two entities can be the same). The matrix is between the entity that was labelled and the entity that was predicted. |
| entities |
<string,
Conversational |
Represents the entities evaluation summary. |
| macroF1 |
number |
Represents the macro F1. Expected value is a float between 0 and 1 inclusive. |
| macroPrecision |
number |
Represents the macro precision. Expected value is a float between 0 and 1 inclusive. |
| macroRecall |
number |
Represents the macro recall. Expected value is a float between 0 and 1 inclusive. |
| microF1 |
number |
Represents the micro F1. Expected value is a float between 0 and 1 inclusive. |
| microPrecision |
number |
Represents the micro precision. Expected value is a float between 0 and 1 inclusive. |
| microRecall |
number |
Represents the micro recall. Expected value is a float between 0 and 1 inclusive. |
ConversationalAnalysisAuthoringEntityEvaluationSummary
Represents the evaluation result for an entity.
| Name | Type | Description |
|---|---|---|
| f1 |
number |
Represents the model precision |
| falseNegativeCount |
integer |
Represents the count of false negative |
| falsePositiveCount |
integer |
Represents the count of false positive |
| precision |
number |
Represents the model recall |
| recall |
number |
Represents the model F1 score |
| trueNegativeCount |
integer |
Represents the count of true negative |
| truePositiveCount |
integer |
Represents the count of true positive |
ConversationalAnalysisAuthoringEvaluationOptions
Represents the options used running the evaluation.
| Name | Type | Description |
|---|---|---|
| kind |
Represents the evaluation kind. By default, the evaluation kind is set to percentage. |
|
| testingSplitPercentage |
integer |
Represents the testing dataset split percentage. Only needed in case the evaluation kind is percentage. |
| trainingSplitPercentage |
integer |
Represents the training dataset split percentage. Only needed in case the evaluation kind is percentage. |
ConversationalAnalysisAuthoringEvaluationSummary
Represents the evaluation result summary.
| Name | Type | Description |
|---|---|---|
| entitiesEvaluation |
Contains the data related to entities evaluation. |
|
| evaluationOptions |
The options that were used while running the evaluation. |
|
| intentsEvaluation |
Contains the data related to intents evaluation. |
ConversationalAnalysisAuthoringIntentEvaluationSummary
The evaluation summary for an intent.
| Name | Type | Description |
|---|---|---|
| f1 |
number |
Represents the model precision |
| falseNegativeCount |
integer |
Represents the count of false negative |
| falsePositiveCount |
integer |
Represents the count of false positive |
| precision |
number |
Represents the model recall |
| recall |
number |
Represents the model F1 score |
| trueNegativeCount |
integer |
Represents the count of true negative |
| truePositiveCount |
integer |
Represents the count of true positive |
ConversationalAnalysisAuthoringIntentsEvaluationSummary
Represents the evaluation summary for the project's intents.
| Name | Type | Description |
|---|---|---|
| confusionMatrix |
object |
Represents the confusion matrix between two intents (the two intents can be the same). The matrix is between the intent that was labelled and the intent that was predicted. |
| intents |
<string,
Conversational |
Represents the intents evaluation summary. |
| macroF1 |
number |
Represents the macro F1. Expected value is a float between 0 and 1 inclusive. |
| macroPrecision |
number |
Represents the macro precision. Expected value is a float between 0 and 1 inclusive. |
| macroRecall |
number |
Represents the macro recall. Expected value is a float between 0 and 1 inclusive. |
| microF1 |
number |
Represents the micro F1. Expected value is a float between 0 and 1 inclusive. |
| microPrecision |
number |
Represents the micro precision. Expected value is a float between 0 and 1 inclusive. |
| microRecall |
number |
Represents the micro recall. Expected value is a float between 0 and 1 inclusive. |
Error
The error object.
| Name | Type | Description |
|---|---|---|
| code |
One of a server-defined set of error codes. |
|
| details |
Error[] |
An array of details about specific errors that led to this reported error. |
| innererror |
An object containing more specific information than the current object about the error. |
|
| message |
string |
A human-readable representation of the error. |
| target |
string |
The target of the error. |
ErrorCode
Human-readable error code.
| Name | Type | Description |
|---|---|---|
| AzureCognitiveSearchIndexLimitReached |
string |
|
| AzureCognitiveSearchIndexNotFound |
string |
|
| AzureCognitiveSearchNotFound |
string |
|
| AzureCognitiveSearchThrottling |
string |
|
| Conflict |
string |
|
| Forbidden |
string |
|
| InternalServerError |
string |
|
| InvalidArgument |
string |
|
| InvalidRequest |
string |
|
| NotFound |
string |
|
| OperationNotFound |
string |
|
| ProjectNotFound |
string |
|
| QuotaExceeded |
string |
|
| ServiceUnavailable |
string |
|
| Timeout |
string |
|
| TooManyRequests |
string |
|
| Unauthorized |
string |
|
| Warning |
string |
ErrorResponse
Error response.
| Name | Type | Description |
|---|---|---|
| error |
The error object. |
EvaluationKind
| Name | Type | Description |
|---|---|---|
| manual |
string |
Split the data according to the chosen dataset for every example in the data. |
| percentage |
string |
Split the data into training and test sets according to user-defined percentages. |
InnerErrorCode
Human-readable error code.
| Name | Type | Description |
|---|---|---|
| AzureCognitiveSearchNotFound |
string |
|
| AzureCognitiveSearchThrottling |
string |
|
| EmptyRequest |
string |
|
| ExtractionFailure |
string |
|
| InvalidCountryHint |
string |
|
| InvalidDocument |
string |
|
| InvalidDocumentBatch |
string |
|
| InvalidParameterValue |
string |
|
| InvalidRequest |
string |
|
| InvalidRequestBodyFormat |
string |
|
| KnowledgeBaseNotFound |
string |
|
| MissingInputDocuments |
string |
|
| ModelVersionIncorrect |
string |
|
| UnsupportedLanguageCode |
string |
InnerErrorModel
An object containing more specific information about the error. As per Microsoft One API guidelines - https://github.com/Microsoft/api-guidelines/blob/vNext/Guidelines.md#7102-error-condition-responses.
| Name | Type | Description |
|---|---|---|
| code |
One of a server-defined set of error codes. |
|
| details |
object |
Error details. |
| innererror |
An object containing more specific information than the current object about the error. |
|
| message |
string |
Error message. |
| target |
string |
Error target. |