1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
23 "git.arvados.org/arvados.git/lib/config"
24 "git.arvados.org/arvados.git/lib/controller/rpc"
25 "git.arvados.org/arvados.git/sdk/go/arvados"
26 "git.arvados.org/arvados.git/sdk/go/arvadostest"
27 "git.arvados.org/arvados.git/sdk/go/auth"
28 "git.arvados.org/arvados.git/sdk/go/ctxlog"
29 "github.com/jmoiron/sqlx"
30 check "gopkg.in/check.v1"
33 // Gocheck boilerplate
34 func Test(t *testing.T) {
38 var _ = check.Suite(&OIDCLoginSuite{})
40 type OIDCLoginSuite struct {
41 cluster *arvados.Cluster
43 railsSpy *arvadostest.Proxy
44 fakeProvider *arvadostest.OIDCProvider
47 func (s *OIDCLoginSuite) TearDownSuite(c *check.C) {
48 // Undo any changes/additions to the user database so they
49 // don't affect subsequent tests.
50 arvadostest.ResetEnv()
51 c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
54 func (s *OIDCLoginSuite) SetUpTest(c *check.C) {
55 s.fakeProvider = arvadostest.NewOIDCProvider(c)
56 s.fakeProvider.AuthEmail = "active-user@arvados.local"
57 s.fakeProvider.AuthEmailVerified = true
58 s.fakeProvider.AuthName = "Fake User Name"
59 s.fakeProvider.ValidCode = fmt.Sprintf("abcdefgh-%d", time.Now().Unix())
60 s.fakeProvider.PeopleAPIResponse = map[string]interface{}{}
62 cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
63 c.Assert(err, check.IsNil)
64 s.cluster, err = cfg.GetCluster("")
65 c.Assert(err, check.IsNil)
66 s.cluster.Login.SSO.Enable = false
67 s.cluster.Login.Google.Enable = true
68 s.cluster.Login.Google.ClientID = "test%client$id"
69 s.cluster.Login.Google.ClientSecret = "test#client/secret"
70 s.cluster.Users.PreferDomainForUsername = "PreferDomainForUsername.example.com"
71 s.fakeProvider.ValidClientID = "test%client$id"
72 s.fakeProvider.ValidClientSecret = "test#client/secret"
74 s.localdb = NewConn(s.cluster)
75 c.Assert(s.localdb.loginController, check.FitsTypeOf, (*oidcLoginController)(nil))
76 s.localdb.loginController.(*oidcLoginController).Issuer = s.fakeProvider.Issuer.URL
77 s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakeProvider.PeopleAPI.URL
79 s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
80 *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
83 func (s *OIDCLoginSuite) TearDownTest(c *check.C) {
87 func (s *OIDCLoginSuite) TestGoogleLogout(c *check.C) {
88 resp, err := s.localdb.Logout(context.Background(), arvados.LogoutOptions{ReturnTo: "https://foo.example.com/bar"})
89 c.Check(err, check.IsNil)
90 c.Check(resp.RedirectLocation, check.Equals, "https://foo.example.com/bar")
93 func (s *OIDCLoginSuite) TestGoogleLogin_Start_Bogus(c *check.C) {
94 resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{})
95 c.Check(err, check.IsNil)
96 c.Check(resp.RedirectLocation, check.Equals, "")
97 c.Check(resp.HTML.String(), check.Matches, `.*missing return_to parameter.*`)
100 func (s *OIDCLoginSuite) TestGoogleLogin_Start(c *check.C) {
101 for _, remote := range []string{"", "zzzzz"} {
102 resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{Remote: remote, ReturnTo: "https://app.example.com/foo?bar"})
103 c.Check(err, check.IsNil)
104 target, err := url.Parse(resp.RedirectLocation)
105 c.Check(err, check.IsNil)
106 issuerURL, _ := url.Parse(s.fakeProvider.Issuer.URL)
107 c.Check(target.Host, check.Equals, issuerURL.Host)
109 c.Check(q.Get("client_id"), check.Equals, "test%client$id")
110 state := s.localdb.loginController.(*oidcLoginController).parseOAuth2State(q.Get("state"))
111 c.Check(state.verify([]byte(s.cluster.SystemRootToken)), check.Equals, true)
112 c.Check(state.Time, check.Not(check.Equals), 0)
113 c.Check(state.Remote, check.Equals, remote)
114 c.Check(state.ReturnTo, check.Equals, "https://app.example.com/foo?bar")
118 func (s *OIDCLoginSuite) TestGoogleLogin_InvalidCode(c *check.C) {
119 state := s.startLogin(c)
120 resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
121 Code: "first-try-a-bogus-code",
124 c.Check(err, check.IsNil)
125 c.Check(resp.RedirectLocation, check.Equals, "")
126 c.Check(resp.HTML.String(), check.Matches, `(?ms).*error in OAuth2 exchange.*cannot fetch token.*`)
129 func (s *OIDCLoginSuite) TestGoogleLogin_InvalidState(c *check.C) {
131 resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
132 Code: s.fakeProvider.ValidCode,
133 State: "bogus-state",
135 c.Check(err, check.IsNil)
136 c.Check(resp.RedirectLocation, check.Equals, "")
137 c.Check(resp.HTML.String(), check.Matches, `(?ms).*invalid OAuth2 state.*`)
140 func (s *OIDCLoginSuite) setupPeopleAPIError(c *check.C) {
141 s.fakeProvider.PeopleAPI = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
142 w.WriteHeader(http.StatusForbidden)
143 fmt.Fprintln(w, `Error 403: accessNotConfigured`)
145 s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakeProvider.PeopleAPI.URL
148 func (s *OIDCLoginSuite) TestGoogleLogin_PeopleAPIDisabled(c *check.C) {
149 s.localdb.loginController.(*oidcLoginController).UseGooglePeopleAPI = false
150 s.fakeProvider.AuthEmail = "joe.smith@primary.example.com"
151 s.setupPeopleAPIError(c)
152 state := s.startLogin(c)
153 _, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
154 Code: s.fakeProvider.ValidCode,
157 c.Check(err, check.IsNil)
158 authinfo := getCallbackAuthInfo(c, s.railsSpy)
159 c.Check(authinfo.Email, check.Equals, "joe.smith@primary.example.com")
162 func (s *OIDCLoginSuite) TestConfig(c *check.C) {
163 s.cluster.Login.Google.Enable = false
164 s.cluster.Login.OpenIDConnect.Enable = true
165 s.cluster.Login.OpenIDConnect.Issuer = "https://accounts.example.com/"
166 s.cluster.Login.OpenIDConnect.ClientID = "oidc-client-id"
167 s.cluster.Login.OpenIDConnect.ClientSecret = "oidc-client-secret"
168 localdb := NewConn(s.cluster)
169 ctrl := localdb.loginController.(*oidcLoginController)
170 c.Check(ctrl.Issuer, check.Equals, "https://accounts.example.com/")
171 c.Check(ctrl.ClientID, check.Equals, "oidc-client-id")
172 c.Check(ctrl.ClientSecret, check.Equals, "oidc-client-secret")
173 c.Check(ctrl.UseGooglePeopleAPI, check.Equals, false)
175 for _, enableAltEmails := range []bool{false, true} {
176 s.cluster.Login.OpenIDConnect.Enable = false
177 s.cluster.Login.Google.Enable = true
178 s.cluster.Login.Google.ClientID = "google-client-id"
179 s.cluster.Login.Google.ClientSecret = "google-client-secret"
180 s.cluster.Login.Google.AlternateEmailAddresses = enableAltEmails
181 localdb = NewConn(s.cluster)
182 ctrl = localdb.loginController.(*oidcLoginController)
183 c.Check(ctrl.Issuer, check.Equals, "https://accounts.google.com")
184 c.Check(ctrl.ClientID, check.Equals, "google-client-id")
185 c.Check(ctrl.ClientSecret, check.Equals, "google-client-secret")
186 c.Check(ctrl.UseGooglePeopleAPI, check.Equals, enableAltEmails)
190 func (s *OIDCLoginSuite) TestGoogleLogin_PeopleAPIError(c *check.C) {
191 s.setupPeopleAPIError(c)
192 state := s.startLogin(c)
193 resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
194 Code: s.fakeProvider.ValidCode,
197 c.Check(err, check.IsNil)
198 c.Check(resp.RedirectLocation, check.Equals, "")
201 func (s *OIDCLoginSuite) TestOIDCAuthorizer(c *check.C) {
202 s.cluster.Login.Google.Enable = false
203 s.cluster.Login.OpenIDConnect.Enable = true
204 json.Unmarshal([]byte(fmt.Sprintf("%q", s.fakeProvider.Issuer.URL)), &s.cluster.Login.OpenIDConnect.Issuer)
205 s.cluster.Login.OpenIDConnect.ClientID = "oidc#client#id"
206 s.cluster.Login.OpenIDConnect.ClientSecret = "oidc#client#secret"
207 s.fakeProvider.ValidClientID = "oidc#client#id"
208 s.fakeProvider.ValidClientSecret = "oidc#client#secret"
209 db := arvadostest.DB(c, s.cluster)
211 tokenCacheTTL = time.Millisecond
212 tokenCacheRaceWindow = time.Millisecond
214 oidcAuthorizer := OIDCAccessTokenAuthorizer(s.cluster, func(context.Context) (*sqlx.DB, error) { return db, nil })
215 accessToken := s.fakeProvider.ValidAccessToken()
217 mac := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
218 io.WriteString(mac, accessToken)
219 hmac := fmt.Sprintf("%x", mac.Sum(nil))
222 _, err := db.Exec(`delete from api_client_authorizations where api_token=$1`, hmac)
223 c.Check(err, check.IsNil)
228 ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{accessToken}})
230 oidcAuthorizer.WrapCalls(func(ctx context.Context, opts interface{}) (interface{}, error) {
231 creds, ok := auth.FromContext(ctx)
232 c.Assert(ok, check.Equals, true)
233 c.Assert(creds.Tokens, check.HasLen, 1)
234 c.Check(creds.Tokens[0], check.Equals, accessToken)
236 err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, hmac).Scan(&exp1)
237 c.Check(err, check.IsNil)
238 c.Check(exp1.Sub(time.Now()) > -time.Second, check.Equals, true)
239 c.Check(exp1.Sub(time.Now()) < time.Second, check.Equals, true)
243 // If the token is used again after the in-memory cache
244 // expires, oidcAuthorizer must re-checks the token and update
245 // the expires_at value in the database.
246 time.Sleep(3 * time.Millisecond)
247 oidcAuthorizer.WrapCalls(func(ctx context.Context, opts interface{}) (interface{}, error) {
249 err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, hmac).Scan(&exp)
250 c.Check(err, check.IsNil)
251 c.Check(exp.Sub(exp1) > 0, check.Equals, true)
252 c.Check(exp.Sub(exp1) < time.Second, check.Equals, true)
257 func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) {
258 s.cluster.Login.Google.Enable = false
259 s.cluster.Login.OpenIDConnect.Enable = true
260 json.Unmarshal([]byte(fmt.Sprintf("%q", s.fakeProvider.Issuer.URL)), &s.cluster.Login.OpenIDConnect.Issuer)
261 s.cluster.Login.OpenIDConnect.ClientID = "oidc#client#id"
262 s.cluster.Login.OpenIDConnect.ClientSecret = "oidc#client#secret"
263 s.fakeProvider.ValidClientID = "oidc#client#id"
264 s.fakeProvider.ValidClientSecret = "oidc#client#secret"
265 for _, trial := range []struct {
266 expectEmail string // "" if failure expected
270 expectEmail: "user@oidc.example.com",
272 c.Log("=== succeed because email_verified is false but not required")
273 s.fakeProvider.AuthEmail = "user@oidc.example.com"
274 s.fakeProvider.AuthEmailVerified = false
275 s.cluster.Login.OpenIDConnect.EmailClaim = "email"
276 s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = ""
277 s.cluster.Login.OpenIDConnect.UsernameClaim = ""
283 c.Log("=== fail because email_verified is false and required")
284 s.fakeProvider.AuthEmail = "user@oidc.example.com"
285 s.fakeProvider.AuthEmailVerified = false
286 s.cluster.Login.OpenIDConnect.EmailClaim = "email"
287 s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "email_verified"
288 s.cluster.Login.OpenIDConnect.UsernameClaim = ""
292 expectEmail: "user@oidc.example.com",
294 c.Log("=== succeed because email_verified is false but config uses custom 'verified' claim")
295 s.fakeProvider.AuthEmail = "user@oidc.example.com"
296 s.fakeProvider.AuthEmailVerified = false
297 s.cluster.Login.OpenIDConnect.EmailClaim = "email"
298 s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "alt_verified"
299 s.cluster.Login.OpenIDConnect.UsernameClaim = ""
303 expectEmail: "alt_email@example.com",
305 c.Log("=== succeed with custom 'email' and 'email_verified' claims")
306 s.fakeProvider.AuthEmail = "bad@wrong.example.com"
307 s.fakeProvider.AuthEmailVerified = false
308 s.cluster.Login.OpenIDConnect.EmailClaim = "alt_email"
309 s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "alt_verified"
310 s.cluster.Login.OpenIDConnect.UsernameClaim = "alt_username"
315 if s.railsSpy != nil {
318 s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
319 s.localdb = NewConn(s.cluster)
320 *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
322 state := s.startLogin(c)
323 resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
324 Code: s.fakeProvider.ValidCode,
327 c.Assert(err, check.IsNil)
328 if trial.expectEmail == "" {
329 c.Check(resp.HTML.String(), check.Matches, `(?ms).*Login error.*`)
330 c.Check(resp.RedirectLocation, check.Equals, "")
333 c.Check(resp.HTML.String(), check.Equals, "")
334 target, err := url.Parse(resp.RedirectLocation)
335 c.Assert(err, check.IsNil)
336 token := target.Query().Get("api_token")
337 c.Check(token, check.Matches, `v2/zzzzz-gj3su-.{15}/.{32,50}`)
338 authinfo := getCallbackAuthInfo(c, s.railsSpy)
339 c.Check(authinfo.Email, check.Equals, trial.expectEmail)
341 switch s.cluster.Login.OpenIDConnect.UsernameClaim {
343 c.Check(authinfo.Username, check.Equals, "desired-username")
345 c.Check(authinfo.Username, check.Equals, "")
347 c.Fail() // bad test case
352 func (s *OIDCLoginSuite) TestGoogleLogin_Success(c *check.C) {
353 state := s.startLogin(c)
354 resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
355 Code: s.fakeProvider.ValidCode,
358 c.Check(err, check.IsNil)
359 c.Check(resp.HTML.String(), check.Equals, "")
360 target, err := url.Parse(resp.RedirectLocation)
361 c.Check(err, check.IsNil)
362 c.Check(target.Host, check.Equals, "app.example.com")
363 c.Check(target.Path, check.Equals, "/foo")
364 token := target.Query().Get("api_token")
365 c.Check(token, check.Matches, `v2/zzzzz-gj3su-.{15}/.{32,50}`)
367 authinfo := getCallbackAuthInfo(c, s.railsSpy)
368 c.Check(authinfo.FirstName, check.Equals, "Fake User")
369 c.Check(authinfo.LastName, check.Equals, "Name")
370 c.Check(authinfo.Email, check.Equals, "active-user@arvados.local")
371 c.Check(authinfo.AlternateEmails, check.HasLen, 0)
373 // Try using the returned Arvados token.
374 c.Logf("trying an API call with new token %q", token)
375 ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{token}})
376 cl, err := s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1})
377 c.Check(cl.ItemsAvailable, check.Not(check.Equals), 0)
378 c.Check(cl.Items, check.Not(check.HasLen), 0)
379 c.Check(err, check.IsNil)
381 // Might as well check that bogus tokens aren't accepted.
382 badtoken := token + "plussomeboguschars"
383 c.Logf("trying an API call with mangled token %q", badtoken)
384 ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{badtoken}})
385 cl, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1})
386 c.Check(cl.Items, check.HasLen, 0)
387 c.Check(err, check.NotNil)
388 c.Check(err, check.ErrorMatches, `.*401 Unauthorized: Not logged in.*`)
391 func (s *OIDCLoginSuite) TestGoogleLogin_RealName(c *check.C) {
392 s.fakeProvider.AuthEmail = "joe.smith@primary.example.com"
393 s.fakeProvider.PeopleAPIResponse = map[string]interface{}{
394 "names": []map[string]interface{}{
396 "metadata": map[string]interface{}{"primary": false},
398 "familyName": "Smith",
401 "metadata": map[string]interface{}{"primary": true},
402 "givenName": "Joseph",
403 "familyName": "Psmith",
407 state := s.startLogin(c)
408 s.localdb.Login(context.Background(), arvados.LoginOptions{
409 Code: s.fakeProvider.ValidCode,
413 authinfo := getCallbackAuthInfo(c, s.railsSpy)
414 c.Check(authinfo.FirstName, check.Equals, "Joseph")
415 c.Check(authinfo.LastName, check.Equals, "Psmith")
418 func (s *OIDCLoginSuite) TestGoogleLogin_OIDCRealName(c *check.C) {
419 s.fakeProvider.AuthName = "Joe P. Smith"
420 s.fakeProvider.AuthEmail = "joe.smith@primary.example.com"
421 state := s.startLogin(c)
422 s.localdb.Login(context.Background(), arvados.LoginOptions{
423 Code: s.fakeProvider.ValidCode,
427 authinfo := getCallbackAuthInfo(c, s.railsSpy)
428 c.Check(authinfo.FirstName, check.Equals, "Joe P.")
429 c.Check(authinfo.LastName, check.Equals, "Smith")
432 // People API returns some additional email addresses.
433 func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses(c *check.C) {
434 s.fakeProvider.AuthEmail = "joe.smith@primary.example.com"
435 s.fakeProvider.PeopleAPIResponse = map[string]interface{}{
436 "emailAddresses": []map[string]interface{}{
438 "metadata": map[string]interface{}{"verified": true},
439 "value": "joe.smith@work.example.com",
442 "value": "joe.smith@unverified.example.com", // unverified, so this one will be ignored
445 "metadata": map[string]interface{}{"verified": true},
446 "value": "joe.smith@home.example.com",
450 state := s.startLogin(c)
451 s.localdb.Login(context.Background(), arvados.LoginOptions{
452 Code: s.fakeProvider.ValidCode,
456 authinfo := getCallbackAuthInfo(c, s.railsSpy)
457 c.Check(authinfo.Email, check.Equals, "joe.smith@primary.example.com")
458 c.Check(authinfo.AlternateEmails, check.DeepEquals, []string{"joe.smith@home.example.com", "joe.smith@work.example.com"})
461 // Primary address is not the one initially returned by oidc.
462 func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses_Primary(c *check.C) {
463 s.fakeProvider.AuthEmail = "joe.smith@alternate.example.com"
464 s.fakeProvider.PeopleAPIResponse = map[string]interface{}{
465 "emailAddresses": []map[string]interface{}{
467 "metadata": map[string]interface{}{"verified": true, "primary": true},
468 "value": "joe.smith@primary.example.com",
471 "metadata": map[string]interface{}{"verified": true},
472 "value": "joe.smith@alternate.example.com",
475 "metadata": map[string]interface{}{"verified": true},
476 "value": "jsmith+123@preferdomainforusername.example.com",
480 state := s.startLogin(c)
481 s.localdb.Login(context.Background(), arvados.LoginOptions{
482 Code: s.fakeProvider.ValidCode,
485 authinfo := getCallbackAuthInfo(c, s.railsSpy)
486 c.Check(authinfo.Email, check.Equals, "joe.smith@primary.example.com")
487 c.Check(authinfo.AlternateEmails, check.DeepEquals, []string{"joe.smith@alternate.example.com", "jsmith+123@preferdomainforusername.example.com"})
488 c.Check(authinfo.Username, check.Equals, "jsmith")
491 func (s *OIDCLoginSuite) TestGoogleLogin_NoPrimaryEmailAddress(c *check.C) {
492 s.fakeProvider.AuthEmail = "joe.smith@unverified.example.com"
493 s.fakeProvider.AuthEmailVerified = false
494 s.fakeProvider.PeopleAPIResponse = map[string]interface{}{
495 "emailAddresses": []map[string]interface{}{
497 "metadata": map[string]interface{}{"verified": true},
498 "value": "joe.smith@work.example.com",
501 "metadata": map[string]interface{}{"verified": true},
502 "value": "joe.smith@home.example.com",
506 state := s.startLogin(c)
507 s.localdb.Login(context.Background(), arvados.LoginOptions{
508 Code: s.fakeProvider.ValidCode,
512 authinfo := getCallbackAuthInfo(c, s.railsSpy)
513 c.Check(authinfo.Email, check.Equals, "joe.smith@work.example.com") // first verified email in People response
514 c.Check(authinfo.AlternateEmails, check.DeepEquals, []string{"joe.smith@home.example.com"})
515 c.Check(authinfo.Username, check.Equals, "")
518 func (s *OIDCLoginSuite) startLogin(c *check.C) (state string) {
519 // Initiate login, but instead of following the redirect to
520 // the provider, just grab state from the redirect URL.
521 resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{ReturnTo: "https://app.example.com/foo?bar"})
522 c.Check(err, check.IsNil)
523 target, err := url.Parse(resp.RedirectLocation)
524 c.Check(err, check.IsNil)
525 state = target.Query().Get("state")
526 c.Check(state, check.Not(check.Equals), "")
530 func getCallbackAuthInfo(c *check.C, railsSpy *arvadostest.Proxy) (authinfo rpc.UserSessionAuthInfo) {
531 for _, dump := range railsSpy.RequestDumps {
532 c.Logf("spied request: %q", dump)
533 split := bytes.Split(dump, []byte("\r\n\r\n"))
534 c.Assert(split, check.HasLen, 2)
535 hdr, body := string(split[0]), string(split[1])
536 if strings.Contains(hdr, "POST /auth/controller/callback") {
537 vs, err := url.ParseQuery(body)
538 c.Check(json.Unmarshal([]byte(vs.Get("auth_info")), &authinfo), check.IsNil)
539 c.Check(err, check.IsNil)
540 sort.Strings(authinfo.AlternateEmails)
544 c.Error("callback not found")