feat: publish sanitized custom.97 source

relaykit/relayconvert/kitutil/mask.go

@@ -1,15 +1,25 @@
 package kitutil
 
 import (
+	"maps"
+	"net"
 	"net/url"
 	"regexp"
+	"slices"
 	"strings"
 )
 
 var (
-	maskURLPattern    = regexp.MustCompile(`(http|https)://[^\s/$.?#].[^\s]*`)
-	maskDomainPattern = regexp.MustCompile(`\b(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}\b`)
-	maskIPPattern     = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
+	// maskURLPattern stops at characters a URL cannot contain unescaped, such
+	// as double quotes and angle or curly brackets, so a URL quoted in an error
+	// keeps its delimiters. Apostrophes and parentheses are valid inside a URL;
+	// the caller trims them, like other punctuation, only when they trail it.
+	maskURLPattern = regexp.MustCompile(`(?:https?|wss?)://[^\s"<>{}]+`)
+	maskIPPattern  = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
+	// maskBracketedIPv6Pattern matches the bracketed form Go network errors use
+	// for IPv6 addresses, as in "dial tcp [2001:db8::1]:443", with an optional
+	// zone; the caller confirms the content parses as an IP.
+	maskBracketedIPv6Pattern = regexp.MustCompile(`\[([0-9A-Fa-f:.]*:[0-9A-Fa-f:.]*)(%[^\]\s]*)?\]`)
 	// maskApiKeyPattern matches patterns like 'api_key:xxx' or "api_key:xxx" to mask the API key value
 	maskApiKeyPattern = regexp.MustCompile(`(['"]?)api_key:([^\s'"]+)(['"]?)`)
 )
@@ -53,30 +63,111 @@ func maskHostForPlainDomain(domain string) string {
 	return stars + "." + strings.Join(tail, ".")
 }
 
-// MaskSensitiveInfo masks sensitive information like URLs, IPs, and domain names in a string
-// Example:
-// http://example.com -> http://***.com
-// https://api.test.org/v1/users/123?key=secret -> https://***.org/***/***/?key=***
-// https://sub.domain.co.uk/path/to/resource -> https://***.co.uk/***/***
-// 192.168.1.1 -> ***.***.***.***
-// openai.com -> ***.com
-// www.openai.com -> ***.***.com
-// api.openai.com -> ***.***.com
-func MaskSensitiveInfo(str string) string {
-	// Mask URLs
-	str = maskURLPattern.ReplaceAllStringFunc(str, func(urlStr string) string {
+// maskURLQuery keeps the parameter names of a raw query and masks the values.
+// Example: key=secret&alt=sse -> ?alt=***&key=***
+func maskURLQuery(rawQuery string) string {
+	if rawQuery == "" {
+		return ""
+	}
+	values, err := url.ParseQuery(rawQuery)
+	if err != nil {
+		// If can't parse query, just mask the whole query string
+		return "?***"
+	}
+	if len(values) == 0 {
+		return ""
+	}
+	maskedParams := make([]string, 0, len(values))
+	for _, key := range slices.Sorted(maps.Keys(values)) {
+		maskedParams = append(maskedParams, key+"=***")
+	}
+	return "?" + strings.Join(maskedParams, "&")
+}
+
+// MaskSensitiveInfo masks the upstream addresses in str and leaves every other
+// URL and domain readable. upstreamURLs are the channel base URL and the URL
+// the gateway actually requested, which differs for routes such as absolute
+// Advanced Custom paths; empty entries are skipped. Credentials and IPv4
+// addresses are masked regardless of the upstream: API key values, the query
+// values and userinfo of every URL, which carry secrets such as client_secret
+// or key on provider endpoints, and IPs, which in network errors include the
+// gateway's own address.
+// Example with upstream https://api.upstream.example.com:
+// https://api.upstream.example.com/v1/chat?key=secret -> https://***.com/***/***?key=***
+// lookup api.upstream.example.com -> lookup ***.***.***.com
+// dial tcp 192.168.1.1:443 -> dial tcp ***.***.***.***:443
+// https://docs.example.org/errors -> unchanged
+// https://auth.example.org/token?client_secret=s -> https://auth.example.org/token?client_secret=***
+func MaskSensitiveInfo(str string, upstreamURLs ...string) string {
+	// Mask API keys (e.g., "api_key:test-api-key-for-redaction" -> "api_key:***")
+	str = maskApiKeyPattern.ReplaceAllString(str, "${1}api_key:***${3}")
+
+	upstreamHosts := make([]string, 0, len(upstreamURLs))
+	for _, upstreamURL := range upstreamURLs {
+		upstreamURL = strings.TrimSpace(upstreamURL)
+		if upstreamURL == "" {
+			continue
+		}
+		if !strings.Contains(upstreamURL, "://") {
+			// A base URL saved without a scheme would otherwise parse as a path.
+			upstreamURL = "//" + upstreamURL
+		}
+		parsed, err := url.Parse(upstreamURL)
+		if err != nil {
+			continue
+		}
+		// A fully qualified host keeps its trailing root dot; errors may omit it.
+		if host := strings.TrimSuffix(parsed.Hostname(), "."); host != "" {
+			upstreamHosts = append(upstreamHosts, host)
+		}
+	}
+	// Mask URLs on an upstream host, and the query values of every other URL
+	str = maskURLPattern.ReplaceAllStringFunc(str, func(match string) string {
+		urlStr := match
+		for urlStr != "" {
+			last := urlStr[len(urlStr)-1]
+			if strings.IndexByte(".,;:!?'", last) < 0 && (last != ')' || strings.Count(urlStr, "(") >= strings.Count(urlStr, ")")) {
+				break
+			}
+			urlStr = urlStr[:len(urlStr)-1]
+		}
+		suffix := match[len(urlStr):]
 		u, err := url.Parse(urlStr)
 		if err != nil {
-			return urlStr
+			return match
+		}
+		host := strings.TrimSuffix(u.Hostname(), ".")
+		if !slices.ContainsFunc(upstreamHosts, func(upstreamHost string) bool { return strings.EqualFold(upstreamHost, host) }) {
+			if u.RawQuery == "" && u.User == nil {
+				return match
+			}
+			// Userinfo is a credential wherever the URL points.
+			if u.User != nil {
+				scheme, afterScheme, _ := strings.Cut(urlStr, "://")
+				authorityEnd := strings.IndexAny(afterScheme, "/?#")
+				if authorityEnd < 0 {
+					authorityEnd = len(afterScheme)
+				}
+				urlStr = scheme + "://***@" + afterScheme[strings.LastIndex(afterScheme[:authorityEnd], "@")+1:]
+			}
+			if u.RawQuery == "" {
+				return urlStr + suffix
+			}
+			beforeQuery, _, _ := strings.Cut(urlStr, "?")
+			fragment := ""
+			if u.Fragment != "" {
+				fragment = "#" + u.EscapedFragment()
+			}
+			return beforeQuery + maskURLQuery(u.RawQuery) + fragment + suffix
 		}
 
-		host := u.Host
-		if host == "" {
-			return urlStr
-		}
-
-		// Mask host with unified logic
 		maskedHost := maskHostForURL(host)
+		if ip := net.ParseIP(host); ip != nil && ip.To4() != nil {
+			maskedHost = "***.***.***.***"
+		}
+		if port := u.Port(); port != "" {
+			maskedHost += ":" + port
+		}
 
 		result := u.Scheme + "://" + maskedHost
 
@@ -96,36 +187,35 @@ func MaskSensitiveInfo(str string) string {
 			result += "/"
 		}
 
-		// Mask query parameters
-		if u.RawQuery != "" {
-			values, err := url.ParseQuery(u.RawQuery)
-			if err != nil {
-				// If can't parse query, just mask the whole query string
-				result += "?***"
-			} else {
-				maskedParams := make([]string, 0, len(values))
-				for key := range values {
-					maskedParams = append(maskedParams, key+"=***")
-				}
-				if len(maskedParams) > 0 {
-					result += "?" + strings.Join(maskedParams, "&")
-				}
-			}
+		return result + maskURLQuery(u.RawQuery) + suffix
+	})
+
+	// Mask IP addresses: in network errors they are the resolved upstream
+	// address or the gateway's own, so they are masked even without an
+	// upstream address, and an IP upstream host is covered too. IPv6 goes
+	// first so an embedded IPv4 such as [::ffff:10.0.0.1] still parses.
+	str = maskBracketedIPv6Pattern.ReplaceAllStringFunc(str, func(match string) string {
+		if net.ParseIP(maskBracketedIPv6Pattern.FindStringSubmatch(match)[1]) == nil {
+			return match
 		}
-
-		return result
+		return "[***]"
 	})
-
-	// Mask domain names without protocol (like openai.com, www.openai.com)
-	str = maskDomainPattern.ReplaceAllStringFunc(str, func(domain string) string {
-		return maskHostForPlainDomain(domain)
-	})
-
-	// Mask IP addresses
 	str = maskIPPattern.ReplaceAllString(str, "***.***.***.***")
+	if len(upstreamHosts) == 0 {
+		return str
+	}
 
-	// Mask API keys (e.g., "api_key:test-api-key-for-redaction" -> "api_key:***")
-	str = maskApiKeyPattern.ReplaceAllString(str, "${1}api_key:***${3}")
+	// Mask an upstream host where it appears without a scheme, as in
+	// "dial tcp: lookup api.example.com: no such host". Longer hosts go first
+	// so a subdomain upstream is masked whole before its parent base host.
+	slices.SortFunc(upstreamHosts, func(a, b string) int { return len(b) - len(a) })
+	for _, host := range upstreamHosts {
+		if net.ParseIP(host) != nil {
+			continue
+		}
+		hostPattern := regexp.MustCompile(`(?i)\b` + regexp.QuoteMeta(host) + `\b`)
+		str = hostPattern.ReplaceAllStringFunc(str, maskHostForPlainDomain)
+	}
 
 	return str
 }

relaykit/relayconvert/kitutil/mask_test.go

@@ -0,0 +1,136 @@
+package kitutil
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMaskSensitiveInfoMasksOnlyTheUpstreamAddress(t *testing.T) {
+	tests := []struct {
+		name         string
+		upstreamURLs []string
+		input        string
+		want         string
+	}{
+		{
+			name:         "no upstream address leaves hosts readable but masks IPs",
+			upstreamURLs: []string{"", ""},
+			input:        `Post "https://api.upstream.example.com/v1/chat/completions": dial tcp 10.0.0.1:443: connect: connection refused`,
+			want:         `Post "https://api.upstream.example.com/v1/chat/completions": dial tcp ***.***.***.***:443: connect: connection refused`,
+		},
+		{
+			name:         "bracketed IPv6 addresses are masked without an upstream address",
+			upstreamURLs: nil,
+			input:        `dial tcp [2a01:4f8::1]:443: i/o timeout; lookup api.example.com on [::1]:53; dial tcp [fe80::1%eth0]:443; dial tcp [::ffff:10.0.0.1]:443; items[0] and [1:2] stay`,
+			want:         `dial tcp [***]:443: i/o timeout; lookup api.example.com on [***]:53; dial tcp [***]:443; dial tcp [***]:443; items[0] and [1:2] stay`,
+		},
+		{
+			name:         "IPv6 upstream base URL in a WebSocket dial error",
+			upstreamURLs: []string{"http://[2001:db8::1]:8080"},
+			input:        `dial failed to ws://[2001:db8::1]:8080/v1/realtime: dial tcp [2001:db8::1]:8080: connect: connection refused`,
+			want:         `dial failed to ws://***:8080/***/***: dial tcp [***]:8080: connect: connection refused`,
+		},
+		{
+			name:         "no upstream address still masks API key values",
+			upstreamURLs: nil,
+			input:        `invalid api_key:test-api-key-for-redaction`,
+			want:         `invalid api_key:***`,
+		},
+		{
+			name:         "query values carry credentials on any host",
+			upstreamURLs: []string{"https://custom.base.example"},
+			input:        `Post "https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id=id-1&client_secret=secret-1": dial tcp: i/o timeout, see https://docs.example.org/errors?lang=en#retry`,
+			want:         `Post "https://aip.baidubce.com/oauth/2.0/token?client_id=***&client_secret=***&grant_type=***": dial tcp: i/o timeout, see https://docs.example.org/errors?lang=***#retry`,
+		},
+		{
+			name:         "parentheses inside a URL keep its query masked",
+			upstreamURLs: nil,
+			input:        `failed to download https://cdn.example/image(1).png?token=secret (status 403)`,
+			want:         `failed to download https://cdn.example/image(1).png?token=*** (status 403)`,
+		},
+		{
+			name:         "apostrophes inside a URL keep its query and userinfo masked",
+			upstreamURLs: nil,
+			input:        `failed to download https://cdn.example/John's-photo.png?token=secret from 'https://o'neil:pw@cdn.example/a?sig=1'`,
+			want:         `failed to download https://cdn.example/John's-photo.png?token=*** from 'https://***@cdn.example/a?sig=***'`,
+		},
+		{
+			name:         "userinfo is masked on any host",
+			upstreamURLs: nil,
+			input:        `downloading https://alice:secret@cdn.example/file.png and https://bob@cdn.example/a?sig=1`,
+			want:         `downloading https://***@cdn.example/file.png and https://***@cdn.example/a?sig=***`,
+		},
+		{
+			name:         "query values are masked without an upstream address",
+			upstreamURLs: nil,
+			input:        `request to https://generativelanguage.googleapis.com/v1beta/models?key=AIza-secret failed`,
+			want:         `request to https://generativelanguage.googleapis.com/v1beta/models?key=*** failed`,
+		},
+		{
+			name:         "upstream URL, its IP and other URLs",
+			upstreamURLs: []string{"https://api.upstream.example.com/openai"},
+			input:        `request to https://api.upstream.example.com/openai/v1/chat/completions?key=secret failed: dial tcp 10.0.0.1:443: i/o timeout, see https://docs.example.org/errors`,
+			want:         `request to https://***.com/***/***/***/***?key=*** failed: dial tcp ***.***.***.***:443: i/o timeout, see https://docs.example.org/errors`,
+		},
+		{
+			name:         "bare upstream host in a DNS error",
+			upstreamURLs: []string{"https://API.Upstream.example.com"},
+			input:        `dial tcp: lookup api.upstream.example.com: no such host`,
+			want:         `dial tcp: lookup ***.***.***.com: no such host`,
+		},
+		{
+			name:         "fully qualified base URL with a trailing dot",
+			upstreamURLs: []string{"https://private.example.com./v1"},
+			input:        `request to https://private.example.com./v1/responses failed: dial tcp: lookup private.example.com.: no such host`,
+			want:         `request to https://***.com/***/*** failed: dial tcp: lookup ***.***.com.: no such host`,
+		},
+		{
+			name:         "base URL saved without a scheme",
+			upstreamURLs: []string{"relay.example.co.uk"},
+			input:        `request to https://relay.example.co.uk/v1/messages failed: EOF`,
+			want:         `request to https://***.co.uk/***/*** failed: EOF`,
+		},
+		{
+			name:         "quoted URL keeps its delimiters and trailing punctuation",
+			upstreamURLs: []string{"https://api.upstream.example.com"},
+			input:        `Post "https://api.upstream.example.com/v1/chat/completions": EOF; retry https://api.upstream.example.com/v1/models.`,
+			want:         `Post "https://***.com/***/***/***": EOF; retry https://***.com/***/***.`,
+		},
+		{
+			name:         "port does not hide a country-code TLD",
+			upstreamURLs: []string{"https://relay.example.co.uk:8443"},
+			input:        `request to https://relay.example.co.uk:8443/v1/messages failed`,
+			want:         `request to https://***.co.uk:8443/***/*** failed`,
+		},
+		{
+			name:         "IP upstream host is fully masked",
+			upstreamURLs: []string{"http://10.0.0.1:8080"},
+			input:        `request to http://10.0.0.1:8080/v1/chat failed`,
+			want:         `request to http://***.***.***.***:8080/***/*** failed`,
+		},
+		{
+			name:         "requested origin of an absolute route besides the base URL",
+			upstreamURLs: []string{"https://api.upstream.example.com", "wss://realtime.private.example.net"},
+			input:        `dial failed to wss://realtime.private.example.net/v1/responses: dial tcp: lookup realtime.private.example.net: no such host`,
+			want:         `dial failed to wss://***.net/***/***: dial tcp: lookup ***.***.***.net: no such host`,
+		},
+		{
+			name:         "subdomain upstream is masked whole before its parent base host",
+			upstreamURLs: []string{"https://api.example.com", "wss://tenant.api.example.com"},
+			input:        `dial tcp: lookup tenant.api.example.com: no such host`,
+			want:         `dial tcp: lookup ***.***.***.com: no such host`,
+		},
+		{
+			name:         "other domains and dotted identifiers stay readable",
+			upstreamURLs: []string{"https://api.upstream.example.com"},
+			input:        `Unknown event response.output_text.delta from model gpt-4.1, contact support@example.org`,
+			want:         `Unknown event response.output_text.delta from model gpt-4.1, contact support@example.org`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, MaskSensitiveInfo(tt.input, tt.upstreamURLs...))
+		})
+	}
+}