client_auth_test.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705
  1. // Copyright 2015 The etcd Authors
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package v2http
  15. import (
  16. "encoding/json"
  17. "errors"
  18. "fmt"
  19. "net/http"
  20. "net/http/httptest"
  21. "net/url"
  22. "path"
  23. "sort"
  24. "strings"
  25. "testing"
  26. "github.com/coreos/etcd/etcdserver/auth"
  27. )
  28. const goodPassword = "good"
  29. func mustJSONRequest(t *testing.T, method string, p string, body string) *http.Request {
  30. req, err := http.NewRequest(method, path.Join(authPrefix, p), strings.NewReader(body))
  31. if err != nil {
  32. t.Fatalf("Error making JSON request: %s %s %s\n", method, p, body)
  33. }
  34. req.Header.Set("Content-Type", "application/json")
  35. return req
  36. }
  37. type mockAuthStore struct {
  38. users map[string]*auth.User
  39. roles map[string]*auth.Role
  40. err error
  41. enabled bool
  42. }
  43. func (s *mockAuthStore) AllUsers() ([]string, error) {
  44. var us []string
  45. for u := range s.users {
  46. us = append(us, u)
  47. }
  48. sort.Strings(us)
  49. return us, s.err
  50. }
  51. func (s *mockAuthStore) GetUser(name string) (auth.User, error) {
  52. u, ok := s.users[name]
  53. if !ok {
  54. return auth.User{}, s.err
  55. }
  56. return *u, s.err
  57. }
  58. func (s *mockAuthStore) CreateOrUpdateUser(user auth.User) (out auth.User, created bool, err error) {
  59. if s.users == nil {
  60. out, err = s.CreateUser(user)
  61. return out, true, err
  62. }
  63. out, err = s.UpdateUser(user)
  64. return out, false, err
  65. }
  66. func (s *mockAuthStore) CreateUser(user auth.User) (auth.User, error) { return user, s.err }
  67. func (s *mockAuthStore) DeleteUser(name string) error { return s.err }
  68. func (s *mockAuthStore) UpdateUser(user auth.User) (auth.User, error) {
  69. return *s.users[user.User], s.err
  70. }
  71. func (s *mockAuthStore) AllRoles() ([]string, error) {
  72. return []string{"awesome", "guest", "root"}, s.err
  73. }
  74. func (s *mockAuthStore) GetRole(name string) (auth.Role, error) {
  75. r, ok := s.roles[name]
  76. if ok {
  77. return *r, s.err
  78. }
  79. return auth.Role{}, fmt.Errorf("%q does not exist (%v)", name, s.err)
  80. }
  81. func (s *mockAuthStore) CreateRole(role auth.Role) error { return s.err }
  82. func (s *mockAuthStore) DeleteRole(name string) error { return s.err }
  83. func (s *mockAuthStore) UpdateRole(role auth.Role) (auth.Role, error) {
  84. return *s.roles[role.Role], s.err
  85. }
  86. func (s *mockAuthStore) AuthEnabled() bool { return s.enabled }
  87. func (s *mockAuthStore) EnableAuth() error { return s.err }
  88. func (s *mockAuthStore) DisableAuth() error { return s.err }
  89. func (s *mockAuthStore) CheckPassword(user auth.User, password string) bool {
  90. return user.Password == password
  91. }
  92. func (s *mockAuthStore) HashPassword(password string) (string, error) {
  93. return password, nil
  94. }
  95. func TestAuthFlow(t *testing.T) {
  96. enableMapMu.Lock()
  97. enabledMap = make(map[capability]bool)
  98. enabledMap[authCapability] = true
  99. enableMapMu.Unlock()
  100. var testCases = []struct {
  101. req *http.Request
  102. store mockAuthStore
  103. wcode int
  104. wbody string
  105. }{
  106. {
  107. req: mustJSONRequest(t, "PUT", "users/alice", `{{{{{{{`),
  108. store: mockAuthStore{},
  109. wcode: http.StatusBadRequest,
  110. wbody: `{"message":"Invalid JSON in request body."}`,
  111. },
  112. {
  113. req: mustJSONRequest(t, "PUT", "users/alice", `{"user": "alice", "password": "goodpassword"}`),
  114. store: mockAuthStore{enabled: true},
  115. wcode: http.StatusUnauthorized,
  116. wbody: `{"message":"Insufficient credentials"}`,
  117. },
  118. // Users
  119. {
  120. req: mustJSONRequest(t, "GET", "users", ""),
  121. store: mockAuthStore{
  122. users: map[string]*auth.User{
  123. "alice": {
  124. User: "alice",
  125. Roles: []string{"alicerole", "guest"},
  126. Password: "wheeee",
  127. },
  128. "bob": {
  129. User: "bob",
  130. Roles: []string{"guest"},
  131. Password: "wheeee",
  132. },
  133. "root": {
  134. User: "root",
  135. Roles: []string{"root"},
  136. Password: "wheeee",
  137. },
  138. },
  139. roles: map[string]*auth.Role{
  140. "alicerole": {
  141. Role: "alicerole",
  142. },
  143. "guest": {
  144. Role: "guest",
  145. },
  146. "root": {
  147. Role: "root",
  148. },
  149. },
  150. },
  151. wcode: http.StatusOK,
  152. wbody: `{"users":[` +
  153. `{"user":"alice","roles":[` +
  154. `{"role":"alicerole","permissions":{"kv":{"read":null,"write":null}}},` +
  155. `{"role":"guest","permissions":{"kv":{"read":null,"write":null}}}` +
  156. `]},` +
  157. `{"user":"bob","roles":[{"role":"guest","permissions":{"kv":{"read":null,"write":null}}}]},` +
  158. `{"user":"root","roles":[{"role":"root","permissions":{"kv":{"read":null,"write":null}}}]}]}`,
  159. },
  160. {
  161. req: mustJSONRequest(t, "GET", "users/alice", ""),
  162. store: mockAuthStore{
  163. users: map[string]*auth.User{
  164. "alice": {
  165. User: "alice",
  166. Roles: []string{"alicerole"},
  167. Password: "wheeee",
  168. },
  169. },
  170. roles: map[string]*auth.Role{
  171. "alicerole": {
  172. Role: "alicerole",
  173. },
  174. },
  175. },
  176. wcode: http.StatusOK,
  177. wbody: `{"user":"alice","roles":[{"role":"alicerole","permissions":{"kv":{"read":null,"write":null}}}]}`,
  178. },
  179. {
  180. req: mustJSONRequest(t, "PUT", "users/alice", `{"user": "alice", "password": "goodpassword"}`),
  181. store: mockAuthStore{},
  182. wcode: http.StatusCreated,
  183. wbody: `{"user":"alice","roles":null}`,
  184. },
  185. {
  186. req: mustJSONRequest(t, "DELETE", "users/alice", ``),
  187. store: mockAuthStore{},
  188. wcode: http.StatusOK,
  189. wbody: ``,
  190. },
  191. {
  192. req: mustJSONRequest(t, "PUT", "users/alice", `{"user": "alice", "password": "goodpassword"}`),
  193. store: mockAuthStore{
  194. users: map[string]*auth.User{
  195. "alice": {
  196. User: "alice",
  197. Roles: []string{"alicerole", "guest"},
  198. Password: "wheeee",
  199. },
  200. },
  201. },
  202. wcode: http.StatusOK,
  203. wbody: `{"user":"alice","roles":["alicerole","guest"]}`,
  204. },
  205. {
  206. req: mustJSONRequest(t, "PUT", "users/alice", `{"user": "alice", "grant": ["alicerole"]}`),
  207. store: mockAuthStore{
  208. users: map[string]*auth.User{
  209. "alice": {
  210. User: "alice",
  211. Roles: []string{"alicerole", "guest"},
  212. Password: "wheeee",
  213. },
  214. },
  215. },
  216. wcode: http.StatusOK,
  217. wbody: `{"user":"alice","roles":["alicerole","guest"]}`,
  218. },
  219. {
  220. req: mustJSONRequest(t, "GET", "users/alice", ``),
  221. store: mockAuthStore{
  222. users: map[string]*auth.User{},
  223. err: auth.Error{Status: http.StatusNotFound, Errmsg: "auth: User alice doesn't exist."},
  224. },
  225. wcode: http.StatusNotFound,
  226. wbody: `{"message":"auth: User alice doesn't exist."}`,
  227. },
  228. {
  229. req: mustJSONRequest(t, "GET", "roles/manager", ""),
  230. store: mockAuthStore{
  231. roles: map[string]*auth.Role{
  232. "manager": {
  233. Role: "manager",
  234. },
  235. },
  236. },
  237. wcode: http.StatusOK,
  238. wbody: `{"role":"manager","permissions":{"kv":{"read":null,"write":null}}}`,
  239. },
  240. {
  241. req: mustJSONRequest(t, "DELETE", "roles/manager", ``),
  242. store: mockAuthStore{},
  243. wcode: http.StatusOK,
  244. wbody: ``,
  245. },
  246. {
  247. req: mustJSONRequest(t, "PUT", "roles/manager", `{"role":"manager","permissions":{"kv":{"read":[],"write":[]}}}`),
  248. store: mockAuthStore{},
  249. wcode: http.StatusCreated,
  250. wbody: `{"role":"manager","permissions":{"kv":{"read":[],"write":[]}}}`,
  251. },
  252. {
  253. req: mustJSONRequest(t, "PUT", "roles/manager", `{"role":"manager","revoke":{"kv":{"read":["foo"],"write":[]}}}`),
  254. store: mockAuthStore{
  255. roles: map[string]*auth.Role{
  256. "manager": {
  257. Role: "manager",
  258. },
  259. },
  260. },
  261. wcode: http.StatusOK,
  262. wbody: `{"role":"manager","permissions":{"kv":{"read":null,"write":null}}}`,
  263. },
  264. {
  265. req: mustJSONRequest(t, "GET", "roles", ""),
  266. store: mockAuthStore{
  267. roles: map[string]*auth.Role{
  268. "awesome": {
  269. Role: "awesome",
  270. },
  271. "guest": {
  272. Role: "guest",
  273. },
  274. "root": {
  275. Role: "root",
  276. },
  277. },
  278. },
  279. wcode: http.StatusOK,
  280. wbody: `{"roles":[{"role":"awesome","permissions":{"kv":{"read":null,"write":null}}},` +
  281. `{"role":"guest","permissions":{"kv":{"read":null,"write":null}}},` +
  282. `{"role":"root","permissions":{"kv":{"read":null,"write":null}}}]}`,
  283. },
  284. {
  285. req: mustJSONRequest(t, "GET", "enable", ""),
  286. store: mockAuthStore{
  287. enabled: true,
  288. },
  289. wcode: http.StatusOK,
  290. wbody: `{"enabled":true}`,
  291. },
  292. {
  293. req: mustJSONRequest(t, "PUT", "enable", ""),
  294. store: mockAuthStore{
  295. enabled: false,
  296. },
  297. wcode: http.StatusOK,
  298. wbody: ``,
  299. },
  300. {
  301. req: (func() *http.Request {
  302. req := mustJSONRequest(t, "DELETE", "enable", "")
  303. req.SetBasicAuth("root", "good")
  304. return req
  305. })(),
  306. store: mockAuthStore{
  307. enabled: true,
  308. users: map[string]*auth.User{
  309. "root": {
  310. User: "root",
  311. Password: goodPassword,
  312. Roles: []string{"root"},
  313. },
  314. },
  315. roles: map[string]*auth.Role{
  316. "root": {
  317. Role: "root",
  318. },
  319. },
  320. },
  321. wcode: http.StatusOK,
  322. wbody: ``,
  323. },
  324. {
  325. req: (func() *http.Request {
  326. req := mustJSONRequest(t, "DELETE", "enable", "")
  327. req.SetBasicAuth("root", "bad")
  328. return req
  329. })(),
  330. store: mockAuthStore{
  331. enabled: true,
  332. users: map[string]*auth.User{
  333. "root": {
  334. User: "root",
  335. Password: goodPassword,
  336. Roles: []string{"root"},
  337. },
  338. },
  339. roles: map[string]*auth.Role{
  340. "root": {
  341. Role: "guest",
  342. },
  343. },
  344. },
  345. wcode: http.StatusUnauthorized,
  346. wbody: `{"message":"Insufficient credentials"}`,
  347. },
  348. }
  349. for i, tt := range testCases {
  350. mux := http.NewServeMux()
  351. h := &authHandler{
  352. sec: &tt.store,
  353. cluster: &fakeCluster{id: 1},
  354. }
  355. handleAuth(mux, h)
  356. rw := httptest.NewRecorder()
  357. mux.ServeHTTP(rw, tt.req)
  358. if rw.Code != tt.wcode {
  359. t.Errorf("#%d: got code=%d, want %d", i, rw.Code, tt.wcode)
  360. }
  361. g := rw.Body.String()
  362. g = strings.TrimSpace(g)
  363. if g != tt.wbody {
  364. t.Errorf("#%d: got body=%s, want %s", i, g, tt.wbody)
  365. }
  366. }
  367. }
  368. func TestGetUserGrantedWithNonexistingRole(t *testing.T) {
  369. sh := &authHandler{
  370. sec: &mockAuthStore{
  371. users: map[string]*auth.User{
  372. "root": {
  373. User: "root",
  374. Roles: []string{"root", "foo"},
  375. },
  376. },
  377. roles: map[string]*auth.Role{
  378. "root": {
  379. Role: "root",
  380. },
  381. },
  382. },
  383. cluster: &fakeCluster{id: 1},
  384. }
  385. srv := httptest.NewServer(http.HandlerFunc(sh.baseUsers))
  386. defer srv.Close()
  387. req, err := http.NewRequest("GET", "", nil)
  388. if err != nil {
  389. t.Fatal(err)
  390. }
  391. req.URL, err = url.Parse(srv.URL)
  392. if err != nil {
  393. t.Fatal(err)
  394. }
  395. req.Header.Set("Content-Type", "application/json")
  396. cli := http.DefaultClient
  397. resp, err := cli.Do(req)
  398. if err != nil {
  399. t.Fatal(err)
  400. }
  401. defer resp.Body.Close()
  402. var uc usersCollections
  403. if err := json.NewDecoder(resp.Body).Decode(&uc); err != nil {
  404. t.Fatal(err)
  405. }
  406. if len(uc.Users) != 1 {
  407. t.Fatalf("expected 1 user, got %+v", uc.Users)
  408. }
  409. if uc.Users[0].User != "root" {
  410. t.Fatalf("expected 'root', got %q", uc.Users[0].User)
  411. }
  412. if len(uc.Users[0].Roles) != 1 {
  413. t.Fatalf("expected 1 role, got %+v", uc.Users[0].Roles)
  414. }
  415. if uc.Users[0].Roles[0].Role != "root" {
  416. t.Fatalf("expected 'root', got %q", uc.Users[0].Roles[0].Role)
  417. }
  418. }
  419. func mustAuthRequest(method, username, password string) *http.Request {
  420. req, err := http.NewRequest(method, "path", strings.NewReader(""))
  421. if err != nil {
  422. panic("Cannot make auth request: " + err.Error())
  423. }
  424. req.SetBasicAuth(username, password)
  425. return req
  426. }
  427. func TestPrefixAccess(t *testing.T) {
  428. var table = []struct {
  429. key string
  430. req *http.Request
  431. store *mockAuthStore
  432. hasRoot bool
  433. hasKeyPrefixAccess bool
  434. hasRecursiveAccess bool
  435. }{
  436. {
  437. key: "/foo",
  438. req: mustAuthRequest("GET", "root", "good"),
  439. store: &mockAuthStore{
  440. users: map[string]*auth.User{
  441. "root": {
  442. User: "root",
  443. Password: goodPassword,
  444. Roles: []string{"root"},
  445. },
  446. },
  447. roles: map[string]*auth.Role{
  448. "root": {
  449. Role: "root",
  450. },
  451. },
  452. enabled: true,
  453. },
  454. hasRoot: true,
  455. hasKeyPrefixAccess: true,
  456. hasRecursiveAccess: true,
  457. },
  458. {
  459. key: "/foo",
  460. req: mustAuthRequest("GET", "user", "good"),
  461. store: &mockAuthStore{
  462. users: map[string]*auth.User{
  463. "user": {
  464. User: "user",
  465. Password: goodPassword,
  466. Roles: []string{"foorole"},
  467. },
  468. },
  469. roles: map[string]*auth.Role{
  470. "foorole": {
  471. Role: "foorole",
  472. Permissions: auth.Permissions{
  473. KV: auth.RWPermission{
  474. Read: []string{"/foo"},
  475. Write: []string{"/foo"},
  476. },
  477. },
  478. },
  479. },
  480. enabled: true,
  481. },
  482. hasRoot: false,
  483. hasKeyPrefixAccess: true,
  484. hasRecursiveAccess: false,
  485. },
  486. {
  487. key: "/foo",
  488. req: mustAuthRequest("GET", "user", "good"),
  489. store: &mockAuthStore{
  490. users: map[string]*auth.User{
  491. "user": {
  492. User: "user",
  493. Password: goodPassword,
  494. Roles: []string{"foorole"},
  495. },
  496. },
  497. roles: map[string]*auth.Role{
  498. "foorole": {
  499. Role: "foorole",
  500. Permissions: auth.Permissions{
  501. KV: auth.RWPermission{
  502. Read: []string{"/foo*"},
  503. Write: []string{"/foo*"},
  504. },
  505. },
  506. },
  507. },
  508. enabled: true,
  509. },
  510. hasRoot: false,
  511. hasKeyPrefixAccess: true,
  512. hasRecursiveAccess: true,
  513. },
  514. {
  515. key: "/foo",
  516. req: mustAuthRequest("GET", "user", "bad"),
  517. store: &mockAuthStore{
  518. users: map[string]*auth.User{
  519. "user": {
  520. User: "user",
  521. Password: goodPassword,
  522. Roles: []string{"foorole"},
  523. },
  524. },
  525. roles: map[string]*auth.Role{
  526. "foorole": {
  527. Role: "foorole",
  528. Permissions: auth.Permissions{
  529. KV: auth.RWPermission{
  530. Read: []string{"/foo*"},
  531. Write: []string{"/foo*"},
  532. },
  533. },
  534. },
  535. },
  536. enabled: true,
  537. },
  538. hasRoot: false,
  539. hasKeyPrefixAccess: false,
  540. hasRecursiveAccess: false,
  541. },
  542. {
  543. key: "/foo",
  544. req: mustAuthRequest("GET", "user", "good"),
  545. store: &mockAuthStore{
  546. users: map[string]*auth.User{},
  547. err: errors.New("Not the user"),
  548. enabled: true,
  549. },
  550. hasRoot: false,
  551. hasKeyPrefixAccess: false,
  552. hasRecursiveAccess: false,
  553. },
  554. {
  555. key: "/foo",
  556. req: mustJSONRequest(t, "GET", "somepath", ""),
  557. store: &mockAuthStore{
  558. users: map[string]*auth.User{
  559. "user": {
  560. User: "user",
  561. Password: goodPassword,
  562. Roles: []string{"foorole"},
  563. },
  564. },
  565. roles: map[string]*auth.Role{
  566. "guest": {
  567. Role: "guest",
  568. Permissions: auth.Permissions{
  569. KV: auth.RWPermission{
  570. Read: []string{"/foo*"},
  571. Write: []string{"/foo*"},
  572. },
  573. },
  574. },
  575. },
  576. enabled: true,
  577. },
  578. hasRoot: false,
  579. hasKeyPrefixAccess: true,
  580. hasRecursiveAccess: true,
  581. },
  582. {
  583. key: "/bar",
  584. req: mustJSONRequest(t, "GET", "somepath", ""),
  585. store: &mockAuthStore{
  586. users: map[string]*auth.User{
  587. "user": {
  588. User: "user",
  589. Password: goodPassword,
  590. Roles: []string{"foorole"},
  591. },
  592. },
  593. roles: map[string]*auth.Role{
  594. "guest": {
  595. Role: "guest",
  596. Permissions: auth.Permissions{
  597. KV: auth.RWPermission{
  598. Read: []string{"/foo*"},
  599. Write: []string{"/foo*"},
  600. },
  601. },
  602. },
  603. },
  604. enabled: true,
  605. },
  606. hasRoot: false,
  607. hasKeyPrefixAccess: false,
  608. hasRecursiveAccess: false,
  609. },
  610. // check access for multiple roles
  611. {
  612. key: "/foo",
  613. req: mustAuthRequest("GET", "user", "good"),
  614. store: &mockAuthStore{
  615. users: map[string]*auth.User{
  616. "user": {
  617. User: "user",
  618. Password: goodPassword,
  619. Roles: []string{"role1", "role2"},
  620. },
  621. },
  622. roles: map[string]*auth.Role{
  623. "role1": {
  624. Role: "role1",
  625. },
  626. "role2": {
  627. Role: "role2",
  628. Permissions: auth.Permissions{
  629. KV: auth.RWPermission{
  630. Read: []string{"/foo"},
  631. Write: []string{"/foo"},
  632. },
  633. },
  634. },
  635. },
  636. enabled: true,
  637. },
  638. hasRoot: false,
  639. hasKeyPrefixAccess: true,
  640. hasRecursiveAccess: false,
  641. },
  642. {
  643. key: "/foo",
  644. req: (func() *http.Request {
  645. req := mustJSONRequest(t, "GET", "somepath", "")
  646. req.Header.Set("Authorization", "malformedencoding")
  647. return req
  648. })(),
  649. store: &mockAuthStore{
  650. enabled: true,
  651. users: map[string]*auth.User{
  652. "root": {
  653. User: "root",
  654. Password: goodPassword,
  655. Roles: []string{"root"},
  656. },
  657. },
  658. roles: map[string]*auth.Role{
  659. "guest": {
  660. Role: "guest",
  661. Permissions: auth.Permissions{
  662. KV: auth.RWPermission{
  663. Read: []string{"/foo*"},
  664. Write: []string{"/foo*"},
  665. },
  666. },
  667. },
  668. },
  669. },
  670. hasRoot: false,
  671. hasKeyPrefixAccess: false,
  672. hasRecursiveAccess: false,
  673. },
  674. }
  675. for i, tt := range table {
  676. if tt.hasRoot != hasRootAccess(tt.store, tt.req) {
  677. t.Errorf("#%d: hasRoot doesn't match (expected %v)", i, tt.hasRoot)
  678. }
  679. if tt.hasKeyPrefixAccess != hasKeyPrefixAccess(tt.store, tt.req, tt.key, false) {
  680. t.Errorf("#%d: hasKeyPrefixAccess doesn't match (expected %v)", i, tt.hasRoot)
  681. }
  682. if tt.hasRecursiveAccess != hasKeyPrefixAccess(tt.store, tt.req, tt.key, true) {
  683. t.Errorf("#%d: hasRecursiveAccess doesn't match (expected %v)", i, tt.hasRoot)
  684. }
  685. }
  686. }