feat: sync deployed v1.0.0-rc.38-custom.42

Sync production source including routing, billing, Astra detection and public uptime. Update custom build instructions and license packaging.

service/responses_usage.go

@@ -0,0 +1,170 @@
+package service
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/QuantumNous/new-api/common"
+	relaycommon "github.com/QuantumNous/new-api/relay/common"
+	"github.com/QuantumNous/new-api/relaykit/dto"
+	"github.com/QuantumNous/new-api/relaykit/relayconvert"
+)
+
+// ResponsesUsageAccumulator owns the accounting facts for one Responses stream.
+// HTTP SSE and WebSocket transports feed the same events into it, then settle
+// Finish's usage through the normal text billing path, including interrupted
+// streams. Observe and Finish must be called by the same stream owner.
+type ResponsesUsageAccumulator struct {
+	info           *relaycommon.RelayInfo
+	usage          *dto.Usage
+	outputText     strings.Builder
+	imageCounter   relaycommon.ImageGenerationCallCounter
+	imageCommitted bool
+	started        bool
+	hasUsage bool
+	finished       bool
+}
+
+func NewResponsesUsageAccumulator(info *relaycommon.RelayInfo) *ResponsesUsageAccumulator {
+	return &ResponsesUsageAccumulator{info: info, usage: &dto.Usage{}}
+}
+
+func (a *ResponsesUsageAccumulator) Observe(event *dto.ResponsesStreamResponse) {
+	if a == nil || event == nil || a.finished {
+		return
+	}
+	if event.Response != nil {
+		a.info.ObserveResponseModel(event.Response.Model)
+	}
+	a.started = true
+	ObserveResponsesOutcome(a.info, event)
+	switch event.Type {
+	case "response.completed", "response.done", "response.failed", "response.incomplete", "response.cancelled", "response.canceled":
+		if event.Response != nil {
+			ApplyResponsesUsage(a.usage, event.Response.Usage)
+			if event.Response.Usage!=nil {a.hasUsage=true}
+			if a.outputText.Len() == 0 {
+				// Some upstreams carry the output only on the terminal event.
+				a.outputText.WriteString(relayconvert.ExtractOutputTextFromResponses(event.Response))
+			}
+		}
+		if a.imageCommitted {
+			return
+		}
+		failed := event.Type != "response.completed" && event.Type != "response.done"
+		if failed || (event.Response != nil && relaycommon.IsNonBillableResponsesStatus(event.Response.Status)) {
+			a.imageCounter.Reset()
+		} else if event.Response != nil {
+			for i := range event.Response.Output {
+				a.imageCounter.Observe(&event.Response.Output[i], &i)
+			}
+		}
+		a.imageCounter.Commit(a.info)
+		a.imageCommitted = true
+	case "response.output_text.delta", "response.function_call_arguments.delta",
+		"response.reasoning_summary_text.delta", "response.reasoning_text.delta", "response.refusal.delta":
+		// Every delta kind here is generated output that upstream bills as
+		// output tokens, so all of them feed the missing-usage estimate.
+		a.outputText.WriteString(event.Delta)
+	case dto.ResponsesOutputTypeItemDone:
+		if event.Item == nil {
+			return
+		}
+		switch event.Item.Type {
+		case dto.BuildInCallWebSearchCall, dto.BuildInCallFileSearchCall, dto.BuildInCallFunctionCall:
+			a.info.CountBillableToolCall(event.Item.Type, event.Item.Name)
+		case dto.ResponsesOutputTypeImageGenerationCall:
+			if !a.imageCommitted {
+				a.imageCounter.Observe(event.Item, event.OutputIndex)
+			}
+		}
+	}
+}
+
+func (a *ResponsesUsageAccumulator) Finish() *dto.Usage {
+	if a.finished {
+		return a.usage
+	}
+	a.finished = true
+	if a.info.DurableAttemptBilling && !a.hasUsage {a.usage.UsageSource="estimated"}
+	// A final image item can already have reached the client before the stream
+	// disconnects. Explicit failed/incomplete terminals reset and commit zero in
+	// Observe; otherwise retain completed tool usage even without a terminal.
+	if !a.imageCommitted {
+		a.imageCounter.Commit(a.info)
+		a.imageCommitted = true
+	}
+	if a.usage.CompletionTokens == 0 {
+		if output := a.outputText.String(); output != "" {
+			a.usage.CompletionTokens = CountTextToken(output, a.info.GetUpstreamModelName())
+		}
+	}
+	// Upstream bills the prompt as soon as it starts generating, so a stream
+	// that produced any event but no usage still owes its input tokens unless
+	// upstream reported an explicit failure.
+	billsPrompt := a.usage.CompletionTokens != 0 || (a.started && !a.info.StreamStatus.ResponseFailed())
+	if a.usage.PromptTokens == 0 && billsPrompt {
+		a.usage.PromptTokens = a.info.GetEstimatePromptTokens()
+	}
+	a.usage.TotalTokens = a.usage.PromptTokens + a.usage.CompletionTokens
+	if a.usage.BillingUsage != nil {
+		a.usage.BillingUsage = dto.CloneBillingUsageWithEstimatedCompletion(a.usage.BillingUsage, a.usage.CompletionTokens)
+	}
+	return a.usage
+}
+
+// ObserveResponsesOutcome records the protocol outcome of one Responses event
+// on the stream status for health classification. Only codes and types are
+// kept; messages never leave the event.
+func ObserveResponsesOutcome(info *relaycommon.RelayInfo, event *dto.ResponsesStreamResponse) {
+	if info == nil || info.StreamStatus == nil || event == nil {
+		return
+	}
+	var responseStatus string
+	if event.Response != nil {
+		_ = common.Unmarshal(event.Response.Status, &responseStatus)
+	}
+	switch {
+	case event.Type == "error" || event.Type == "response.failed" || event.Type == "response.error" || responseStatus == "failed":
+		code, errorType := event.Code, ""
+		if event.Response != nil {
+			if oaiErr := event.Response.GetOpenAIError(); oaiErr != nil {
+				if oaiErr.Code != nil {
+					code = fmt.Sprint(oaiErr.Code)
+				}
+				errorType = oaiErr.Type
+			}
+		}
+		info.StreamStatus.MarkFailed(code, errorType, 0)
+	case event.Type == "response.incomplete" || responseStatus == "incomplete":
+		reason := ""
+		if event.Response != nil && event.Response.IncompleteDetails != nil {
+			reason = event.Response.IncompleteDetails.Reason
+		}
+		info.StreamStatus.MarkIncomplete(reason)
+	case event.Type == "response.cancelled" || event.Type == "response.canceled" || responseStatus == "cancelled":
+		info.StreamStatus.MarkCancelled()
+	case event.Type == "response.completed" || event.Type == "response.done" || responseStatus == "completed":
+		info.StreamStatus.MarkCompleted()
+	}
+}
+
+func ApplyResponsesUsage(dst *dto.Usage, src *dto.Usage) {
+	if dst == nil || src == nil {
+		return
+	}
+	incoming := relayconvert.NormalizeResponsesUsage(src)
+	if src.InputTokensDetails != nil {
+		inputDetails := *src.InputTokensDetails
+		incoming.InputTokensDetails = &inputDetails
+	}
+	if src.OutputTokensDetails != nil {
+		incoming.CompletionTokenDetails = *src.OutputTokensDetails
+	}
+	incoming.PromptCacheHitTokens = src.PromptCacheHitTokens
+	dto.MergeUsageNonZero(dst, incoming)
+	outputDetails := dst.CompletionTokenDetails
+	if outputDetails != (dto.OutputTokenDetails{}) {
+		dst.OutputTokensDetails = &outputDetails
+	}
+}