feat: publish sanitized custom.97 source

router/access_token_scope_test.go

@@ -0,0 +1,202 @@
+package router
+
+import (
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/QuantumNous/new-api/common"
+	"github.com/QuantumNous/new-api/middleware"
+	"github.com/QuantumNous/new-api/service"
+	"github.com/QuantumNous/new-api/service/authz"
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// accessTokenExemptRoutes are /api routes that never reach dashboard
+// authentication, so a personal access token has no rule to satisfy there.
+var accessTokenExemptRoutes = []string{
+	// Anonymous pages and setup.
+	"GET /api/setup",
+	"POST /api/setup",
+	"GET /api/status",
+	"GET /api/uptime/status",
+	"GET /api/notice",
+	"GET /api/user-agreement",
+	"GET /api/privacy-policy",
+	"GET /api/about",
+	"GET /api/home_page_content",
+	"GET /api/ratio_config",
+	"GET /api/user/groups",
+
+	// Anonymous account entry points.
+	"GET /api/verification",
+	"GET /api/reset_password",
+	"POST /api/user/reset",
+	"POST /api/user/register",
+	"GET /api/user/login/encryption-key",
+	"POST /api/user/login",
+	"POST /api/user/login/2fa",
+	"POST /api/user/login/verify",
+	"POST /api/user/login/passkey/begin",
+	"POST /api/user/login/passkey/finish",
+	"POST /api/user/passkey/login/begin",
+	"POST /api/user/passkey/login/finish",
+	"GET /api/oauth/wechat",
+	"GET /api/oauth/telegram/login",
+	"GET /api/oauth/telegram/bind/:flow_token",
+
+	// Browser session cookie endpoints.
+	"POST /api/user/auth/refresh",
+	"POST /api/user/auth/logout",
+
+	// Payment callbacks.
+	"POST /api/stripe/webhook",
+	"POST /api/creem/webhook",
+	"POST /api/waffo/webhook",
+	"POST /api/waffo-pancake/webhook/:env",
+	"GET /api/user/epay/notify",
+	"POST /api/user/epay/notify",
+	"GET /api/subscription/epay/notify",
+	"POST /api/subscription/epay/notify",
+	"GET /api/subscription/epay/return",
+	"POST /api/subscription/epay/return",
+
+	// API key authentication (TokenAuthReadOnly).
+	"GET /api/usage/token/",
+	"GET /api/log/token",
+}
+
+// accessTokenHelperRoutes are declared through handlePermissionRoute outside
+// channelPermissionRoutes.
+var accessTokenHelperRoutes = map[string]authz.Permission{
+	"GET /api/task_plugin_options": authz.TaskPluginBind,
+	"GET /api/audit":               authz.AuditRead,
+}
+
+func newAccessTokenScopeTestEngine(t *testing.T) *gin.Engine {
+	t.Helper()
+	gin.SetMode(gin.TestMode)
+	engine := gin.New()
+	// A permission route that is also in the static table panics here.
+	require.NotPanics(t, func() {
+		SetApiRouter(engine)
+		SetDashboardRouter(engine)
+		SetRelayRouter(engine)
+		SetTaskPluginProtocolRouter(engine)
+		SetVideoRouter(engine)
+		SetTaskRouter(engine)
+	})
+	return engine
+}
+
+func TestAccessTokenRouteRulesCoverEveryDashboardRoute(t *testing.T) {
+	engine := newAccessTokenScopeTestEngine(t)
+	exempt := map[string]bool{}
+	for _, key := range accessTokenExemptRoutes {
+		exempt[key] = true
+	}
+	registered := map[string]bool{}
+	panel := map[string]bool{}
+	for _, route := range engine.Routes() {
+		if !strings.HasPrefix(route.Path, "/api/") && !strings.HasPrefix(route.Path, "/pg/") {
+			continue
+		}
+		key := route.Method + " " + route.Path
+		registered[key] = true
+		if !exempt[key] {
+			panel[key] = true
+		}
+	}
+	for _, key := range accessTokenExemptRoutes {
+		assert.True(t, registered[key], "exempt route %s is no longer registered", key)
+	}
+
+	declared := map[string]int{}
+	for _, key := range middleware.AccessTokenRouteRuleKeys() {
+		declared[key]++
+	}
+	for key := range panel {
+		assert.Equal(t, 1, declared[key], "panel route %s must be declared exactly once", key)
+	}
+	for key := range declared {
+		assert.True(t, panel[key], "declared route %s is not a registered panel route", key)
+	}
+
+	permissionRoutes := map[string]authz.Permission{}
+	for _, route := range channelPermissionRoutes {
+		permissionRoutes[route.method+" "+joinPaths("/api/channel", route.path)] = route.permission
+	}
+	for key, permission := range accessTokenHelperRoutes {
+		permissionRoutes[key] = permission
+	}
+	for key, permission := range permissionRoutes {
+		rule, ok := middleware.AccessTokenRouteRule(key)
+		require.True(t, ok, key)
+		assert.Equal(t, "scope", rule.Kind(), key)
+		assert.Equal(t, service.AccessTokenScopeOf(permission), rule.Scope(), key)
+	}
+}
+
+func TestAccessTokenCatalogCoversEveryPermission(t *testing.T) {
+	catalog := service.AccessTokenCatalog(1, common.RoleRootUser)
+	scopes := map[string]bool{}
+	resources := map[string]int{}
+	for _, group := range catalog {
+		for _, resource := range group.Resources {
+			resources[resource.Resource]++
+			for _, action := range resource.Actions {
+				scopes[resource.Resource+":"+action.Action] = true
+			}
+		}
+	}
+	for _, permission := range authz.AllPermissions() {
+		assert.True(t, scopes[service.AccessTokenScopeOf(permission)], "root catalog lacks %s", service.AccessTokenScopeOf(permission))
+	}
+	for resource, count := range resources {
+		assert.Equal(t, 1, count, "resource %s appears in more than one catalog entry", resource)
+	}
+}
+
+func TestAccessTokenVerificationScopesAreAllListed(t *testing.T) {
+	files, err := filepath.Glob("../service/*.go")
+	require.NoError(t, err)
+	fileSet := token.NewFileSet()
+	var scopes []string
+	for _, file := range files {
+		if strings.HasSuffix(file, "_test.go") {
+			continue
+		}
+		parsed, err := parser.ParseFile(fileSet, file, nil, 0)
+		require.NoError(t, err)
+		for _, declaration := range parsed.Decls {
+			general, ok := declaration.(*ast.GenDecl)
+			if !ok || general.Tok != token.CONST {
+				continue
+			}
+			for _, spec := range general.Specs {
+				value := spec.(*ast.ValueSpec)
+				for i, name := range value.Names {
+					if !strings.HasPrefix(name.Name, "VerificationScope") || i >= len(value.Values) {
+						continue
+					}
+					literal, ok := value.Values[i].(*ast.BasicLit)
+					require.True(t, ok, name.Name)
+					scope, err := strconv.Unquote(literal.Value)
+					require.NoError(t, err)
+					scopes = append(scopes, scope)
+				}
+			}
+		}
+	}
+	require.NotEmpty(t, scopes)
+	for _, scope := range scopes {
+		_, listed := service.AccessTokenVerificationScope(scope)
+		assert.True(t, listed, "verification scope %s has no access token rule", scope)
+	}
+}

router/api-router.go

@@ -1,6 +1,8 @@
 package router
 
 import (
+	"net/http"
+
 	"github.com/QuantumNous/new-api/controller"
 	"github.com/QuantumNous/new-api/middleware"
 	"github.com/QuantumNous/new-api/service/authz"
@@ -13,6 +15,8 @@ import (
 )
 
 func SetApiRouter(router *gin.Engine) {
+	// SSE must bypass the API gzip middleware to flush events immediately.
+	router.GET("/api/group-status/events", middleware.RouteTag("api"), middleware.GlobalAPIRateLimit(), controller.StreamGroupStatus)
 	apiRouter := router.Group("/api")
 	apiRouter.Use(middleware.RouteTag("api"))
 	apiRouter.Use(gzip.Gzip(gzip.DefaultCompression))
@@ -24,6 +28,11 @@ func SetApiRouter(router *gin.Engine) {
 		apiRouter.GET("/setup", controller.GetSetup)
 		apiRouter.POST("/setup", anonymousRequestBodyLimit, controller.PostSetup)
 		apiRouter.GET("/status", controller.GetStatus)
+		apiRouter.POST("/bot/balances", controller.BotBalanceQuery)
+		apiRouter.POST("/bot/chaintest", controller.BotChainTest)
+		apiRouter.GET("/bot/chaintest/:id", controller.BotChainTest)
+		apiRouter.POST("/bot/ciallo", controller.BotCiallo)
+		apiRouter.GET("/bot/ciallo/:id", controller.BotCiallo)
 		apiRouter.GET("/uptime/status", controller.GetGroupStatus)
 		apiRouter.GET("/group-status", controller.GetSiteGroupStatus)
 		apiRouter.GET("/uptime/admin", middleware.AdminAuth(), controller.GetStatusPageAdmin)
@@ -106,10 +115,17 @@ func SetApiRouter(router *gin.Engine) {
 				selfRoute.GET("/models", controller.GetUserModels)
 				selfRoute.PUT("/self", middleware.CriticalRateLimit(), middleware.DisableCache(), controller.UpdateSelf)
 				selfRoute.DELETE("/self", middleware.DisableCache(), controller.DeleteSelf)
-				selfRoute.GET("/token", middleware.CriticalRateLimit(), middleware.UserCriticalRateLimit("access-token"), middleware.DisableCache(), controller.GenerateAccessToken)
-				selfRoute.GET("/token/status", middleware.DisableCache(), controller.GetAccessTokenStatus)
-				selfRoute.POST("/token", middleware.CriticalRateLimit(), middleware.UserCriticalRateLimit("access-token"), middleware.DisableCache(), controller.GenerateAccessToken)
-				selfRoute.DELETE("/token", middleware.CriticalRateLimit(), middleware.UserCriticalRateLimit("access-token"), middleware.DisableCache(), controller.RevokeAccessToken)
+				accessTokenRoute := selfRoute.Group("/access_tokens")
+				accessTokenRoute.Use(middleware.DisableCache())
+				{
+					accessTokenRoute.GET("", controller.ListAccessTokens)
+					accessTokenRoute.GET("/catalog", controller.GetAccessTokenCatalog)
+					accessTokenRoute.GET("/scopes", controller.GetAccessTokenScopes)
+					accessTokenRoute.POST("", middleware.CriticalRateLimit(), middleware.UserCriticalRateLimit("access-token"), controller.CreateAccessToken)
+					accessTokenRoute.PATCH("/:id", middleware.CriticalRateLimit(), middleware.UserCriticalRateLimit("access-token"), controller.UpdateAccessToken)
+					accessTokenRoute.DELETE("/:id", middleware.CriticalRateLimit(), middleware.UserCriticalRateLimit("access-token"), controller.DeleteAccessToken)
+					accessTokenRoute.DELETE("/legacy", middleware.CriticalRateLimit(), middleware.UserCriticalRateLimit("access-token"), controller.RevokeLegacyAccessToken)
+				}
 				selfRoute.GET("/passkey", controller.PasskeyStatus)
 				selfRoute.POST("/passkey/register/begin", middleware.UserCriticalRateLimit("security-verification"), middleware.DisableCache(), controller.PasskeyRegisterBegin)
 				selfRoute.POST("/passkey/register/finish", middleware.UserCriticalRateLimit("security-verification"), middleware.DisableCache(), controller.PasskeyRegisterFinish)
@@ -276,7 +292,7 @@ func SetApiRouter(router *gin.Engine) {
 			taskPluginRoute.POST("/:key/dryrun", controller.DryRunTaskPlugin)
 			taskPluginRoute.DELETE("/:key/versions/:version", controller.DeleteTaskPluginVersion)
 		}
-		apiRouter.GET("/task_plugin_options", middleware.AdminAuth(), middleware.RequirePermission(authz.TaskPluginBind), controller.GetTaskPluginOptions)
+		handlePermissionRoute(apiRouter.Group("", middleware.AdminAuth()), http.MethodGet, "/task_plugin_options", authz.TaskPluginBind, controller.GetTaskPluginOptions)
 		registerChannelRoutes(apiRouter)
 		registerAuthzRoutes(apiRouter)
 		tokenRoute := apiRouter.Group("/token")
@@ -318,7 +334,7 @@ func SetApiRouter(router *gin.Engine) {
 			redemptionRoute.DELETE("/invalid", controller.DeleteInvalidRedemption)
 			redemptionRoute.DELETE("/:id", controller.DeleteRedemption)
 		}
-		apiRouter.GET("/audit", middleware.DisableCache(), middleware.AdminAuth(), middleware.RequirePermission(authz.AuditRead), controller.GetAuditLogs)
+		handlePermissionRoute(apiRouter.Group("", middleware.DisableCache(), middleware.AdminAuth()), http.MethodGet, "/audit", authz.AuditRead, controller.GetAuditLogs)
 		apiRouter.GET("/audit/self", middleware.DisableCache(), middleware.UserAuth(), controller.GetAuditLogs)
 		logRoute := apiRouter.Group("/log")
 		logRoute.GET("/attempts/:request_id", middleware.UserAuth(), controller.GetRelayAttempts)

router/channel-router.go

@@ -2,6 +2,7 @@ package router
 
 import (
 	"net/http"
+	"path"
 
 	"github.com/QuantumNous/new-api/controller"
 	"github.com/QuantumNous/new-api/middleware"
@@ -16,6 +17,26 @@ type permissionRoute struct {
 	handler    gin.HandlerFunc
 }
 
+// handlePermissionRoute registers a Casbin-guarded route and declares the same
+// permission as the access token scope that route requires.
+func handlePermissionRoute(group *gin.RouterGroup, method, relativePath string, permission authz.Permission, handlers ...gin.HandlerFunc) {
+	middleware.DeclareAccessTokenPermissionRoute(method, joinPaths(group.BasePath(), relativePath), permission)
+	group.Handle(method, relativePath, append([]gin.HandlerFunc{middleware.RequirePermission(permission)}, handlers...)...)
+}
+
+// joinPaths matches gin's route path joining, which keeps a trailing slash
+// from the relative path, so declared keys equal c.FullPath().
+func joinPaths(absolutePath, relativePath string) string {
+	if relativePath == "" {
+		return absolutePath
+	}
+	finalPath := path.Join(absolutePath, relativePath)
+	if relativePath[len(relativePath)-1] == '/' && finalPath[len(finalPath)-1] != '/' {
+		return finalPath + "/"
+	}
+	return finalPath
+}
+
 func registerChannelRoutes(apiRouter *gin.RouterGroup) {
 	channelRoute := apiRouter.Group("/channel")
 	channelRoute.Use(middleware.AdminAuth())
@@ -29,10 +50,7 @@ func registerChannelRoutes(apiRouter *gin.RouterGroup) {
 	)
 
 	for _, route := range channelPermissionRoutes {
-		channelRoute.Handle(route.method, route.path,
-			middleware.RequirePermission(route.permission),
-			route.handler,
-		)
+		handlePermissionRoute(channelRoute, route.method, route.path, route.permission, route.handler)
 	}
 }
 

router/channel_router_test.go

@@ -25,6 +25,8 @@ func TestChannelDefaultBaseURLsRequireReadPermission(t *testing.T) {
 }
 
 func TestChannelStatusRoutesUseExpectedPermissions(t *testing.T) {
+	assertChannelRoutePermission(t, http.MethodGet, "/priority/order", authz.ChannelRead, controller.GetChannelPriorityOrder)
+	assertChannelRoutePermission(t, http.MethodPost, "/priority/reorder", authz.ChannelWrite, controller.ReorderChannelPriority)
 	assertChannelRoutePermission(t, http.MethodGet, "/:id/vllm/status", authz.ChannelRead, controller.GetVLLMChannelStatus)
 	assertChannelRoutePermission(t, http.MethodGet, "/:id/sglang/status", authz.ChannelRead, controller.GetSGLangChannelStatus)
 	assertChannelRoutePermission(t, http.MethodPost, "/:id/status", authz.ChannelOperate, controller.UpdateChannelStatus)

router/main.go

@@ -23,7 +23,7 @@ func SetRouter(router *gin.Engine, assets WebAssets) {
 	frontendBaseUrl := os.Getenv("FRONTEND_BASE_URL")
 	if common.IsMasterNode && frontendBaseUrl != "" {
 		frontendBaseUrl = ""
-		common.SysLog("FRONTEND_BASE_URL is ignored on master node")
+		common.SysLog(common.LogText("FRONTEND_BASE_URL is ignored on master node"))
 	}
 	if frontendBaseUrl == "" {
 		SetWebRouter(router, assets, pluginDispatcher)

router/plugin-router.go

@@ -55,12 +55,12 @@ func SetPluginRouter(outer *gin.Engine) gin.HandlerFunc {
 	trustedProxies, _, err := common.ResolveTrustedProxies(os.Getenv("TRUSTED_PROXIES"))
 	dispatcher := &pluginRouteDispatcher{registry: jsplugin.DefaultRegistry}
 	if err != nil {
-		common.SysError("configure plugin router trusted proxies: " + err.Error())
+		common.SysError(common.LogText("configure plugin router trusted proxies: %s", err.Error()))
 		return dispatcher.dispatch
 	}
 	builder := newPluginGenerationBuilder(outer.Routes(), trustedProxies, productionPluginRouteHandlers)
 	if err = jsplugin.DefaultRegistry.SetGenerationPreparer(builder.prepare); err != nil {
-		common.SysError("build initial plugin router: " + err.Error())
+		common.SysError(common.LogText("build initial plugin router: %s", err.Error()))
 	}
 	return dispatcher.dispatch
 }
@@ -331,7 +331,7 @@ func pluginRouteRecovery() gin.HandlerFunc {
 			if recover() == nil {
 				return
 			}
-			common.SysError("panic recovered in plugin route")
+			common.SysError(common.LogText("panic recovered in plugin route"))
 			if pinnedValue, exists := c.Get(jsplugin.ContextKeyPinnedRoute); exists {
 				if pinned, ok := pinnedValue.(jsplugin.PinnedRoute); ok && pinned.Plugin != nil && pinned.Generation != nil {
 					logger.LogDebug(

router/plugin_router_test.go

@@ -907,7 +907,7 @@ func TestWebFallbackDoesNotCacheMissingAPIOrAssets(t *testing.T) {
 func TestSecurityRoutesDisableCachingBeforeAuthentication(t *testing.T) {
 	outer := gin.New()
 	SetApiRouter(outer)
-	for _, path := range []string{"/api/user/token/status", "/api/user/token", "/api/audit/self", "/api/audit"} {
+	for _, path := range []string{"/api/user/access_tokens", "/api/user/access_tokens/catalog", "/api/user/access_tokens/scopes", "/api/audit/self", "/api/audit"} {
 		t.Run(path, func(t *testing.T) {
 			response := performPluginRequest(outer, http.MethodGet, path)
 			assert.Equal(t, http.StatusUnauthorized, response.Code)

router/relay-router.go

@@ -116,16 +116,20 @@ func SetRelayRouter(router *gin.Engine) {
 			controller.Relay(c, types.RelayFormatOpenAIAlphaSearch)
 		})
 
-		// image related routes
+		// DeepSeek Search and Fetch API routes
+		httpRouter.POST("/tools/search", func(c *gin.Context) {
+			controller.Relay(c, types.RelayFormatDeepSeekTools)
+		})
+		httpRouter.POST("/tools/fetch", func(c *gin.Context) {
+			controller.Relay(c, types.RelayFormatDeepSeekTools)
+		})
+
+		// image related routes. /images/generations and /images/edits are
+		// host protocol endpoints (openai_image) registered by
+		// SetTaskPluginProtocolRouter; unclaimed models fall back to Relay.
 		httpRouter.POST("/edits", func(c *gin.Context) {
 			controller.Relay(c, types.RelayFormatOpenAIImage)
 		})
-		httpRouter.POST("/images/generations", func(c *gin.Context) {
-			controller.Relay(c, types.RelayFormatOpenAIImage)
-		})
-		httpRouter.POST("/images/edits", func(c *gin.Context) {
-			controller.Relay(c, types.RelayFormatOpenAIImage)
-		})
 
 		// embedding related routes
 		httpRouter.POST("/embeddings", func(c *gin.Context) {

router/task-plugin-protocol-router.go

@@ -34,6 +34,14 @@ func taskPluginProtocolHandlers(protocol, operation string) ([]gin.HandlerFunc,
 				controller.RelayTaskPluginEndpoint(c, func(c *gin.Context) { controller.Relay(c, types.RelayFormatOpenAIResponses) })
 			},
 		}, nil
+	case "openai_image.generate", "openai_image.edit":
+		return []gin.HandlerFunc{
+			middleware.RouteTag("relay"), middleware.SystemPerformanceCheck(), middleware.TokenAuth(),
+			middleware.ModelRequestRateLimit(), middleware.PinTaskPluginEndpoint(), middleware.PrepareTaskPluginEndpoint(), middleware.Distribute(),
+			func(c *gin.Context) {
+				controller.RelayTaskPluginEndpoint(c, func(c *gin.Context) { controller.Relay(c, types.RelayFormatOpenAIImage) })
+			},
+		}, nil
 	case "openai_video.create":
 		return []gin.HandlerFunc{
 			middleware.RouteTag("relay"), middleware.TokenAuth(), middleware.SystemPerformanceCheck(),

router/task_plugin_protocol_router_test.go

@@ -20,6 +20,8 @@ func TestHostProtocolRegistryDrivesProtocolRoutesOnce(t *testing.T) {
 		"GET /v1/videos/:task_id",
 		"GET /v1/videos/:task_id/content",
 		"HEAD /v1/videos/:task_id/content",
+		"POST /v1/images/generations",
+		"POST /v1/images/edits",
 	}
 	actual := make([]string, 0, len(engine.Routes()))
 	for _, route := range engine.Routes() {

router/web-router.go

@@ -9,7 +9,6 @@ import (
 	"github.com/QuantumNous/new-api/controller"
 	"github.com/QuantumNous/new-api/middleware"
 	"github.com/gin-contrib/gzip"
-	"github.com/gin-contrib/static"
 	"github.com/gin-gonic/gin"
 )
 
@@ -21,15 +20,21 @@ type WebAssets struct {
 
 func SetWebRouter(router *gin.Engine, assets WebAssets, pluginDispatcher gin.HandlerFunc) {
 	frontendFS := common.EmbedFolder(assets.BuildFS, "web/dist")
+	compress := gzip.Gzip(gzip.DefaultCompression)
 
 	router.NoRoute(
 		pluginDispatcher,
 		middleware.RouteTag("web"),
-		gzip.Gzip(gzip.DefaultCompression),
+		func(c *gin.Context) {
+			// ServeFrontendFiles sends build files already compressed.
+			if !frontendFS.Exists("/", c.Request.URL.Path) {
+				compress(c)
+			}
+		},
 		middleware.AccessTokenAudit(),
-		middleware.GlobalWebRateLimit(),
+		middleware.GlobalWebRateLimit(frontendFS),
 		middleware.Cache(),
-		static.Serve("/", frontendFS),
+		middleware.ServeFrontendFiles(frontendFS),
 		func(c *gin.Context) {
 			if strings.HasPrefix(c.Request.RequestURI, "/v1") || strings.HasPrefix(c.Request.RequestURI, "/api") || strings.HasPrefix(c.Request.RequestURI, "/assets") {
 				controller.RelayNotFound(c)