client_auth_test.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703
  1. // Copyright 2015 The etcd Authors
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package v2http
  15. import (
  16. "encoding/json"
  17. "errors"
  18. "fmt"
  19. "net/http"
  20. "net/http/httptest"
  21. "net/url"
  22. "path"
  23. "sort"
  24. "strings"
  25. "testing"
  26. "github.com/coreos/etcd/etcdserver/api"
  27. "github.com/coreos/etcd/etcdserver/auth"
  28. )
  29. const goodPassword = "good"
  30. func mustJSONRequest(t *testing.T, method string, p string, body string) *http.Request {
  31. req, err := http.NewRequest(method, path.Join(authPrefix, p), strings.NewReader(body))
  32. if err != nil {
  33. t.Fatalf("Error making JSON request: %s %s %s\n", method, p, body)
  34. }
  35. req.Header.Set("Content-Type", "application/json")
  36. return req
  37. }
  38. type mockAuthStore struct {
  39. users map[string]*auth.User
  40. roles map[string]*auth.Role
  41. err error
  42. enabled bool
  43. }
  44. func (s *mockAuthStore) AllUsers() ([]string, error) {
  45. var us []string
  46. for u := range s.users {
  47. us = append(us, u)
  48. }
  49. sort.Strings(us)
  50. return us, s.err
  51. }
  52. func (s *mockAuthStore) GetUser(name string) (auth.User, error) {
  53. u, ok := s.users[name]
  54. if !ok {
  55. return auth.User{}, s.err
  56. }
  57. return *u, s.err
  58. }
  59. func (s *mockAuthStore) CreateOrUpdateUser(user auth.User) (out auth.User, created bool, err error) {
  60. if s.users == nil {
  61. out, err = s.CreateUser(user)
  62. return out, true, err
  63. }
  64. out, err = s.UpdateUser(user)
  65. return out, false, err
  66. }
  67. func (s *mockAuthStore) CreateUser(user auth.User) (auth.User, error) { return user, s.err }
  68. func (s *mockAuthStore) DeleteUser(name string) error { return s.err }
  69. func (s *mockAuthStore) UpdateUser(user auth.User) (auth.User, error) {
  70. return *s.users[user.User], s.err
  71. }
  72. func (s *mockAuthStore) AllRoles() ([]string, error) {
  73. return []string{"awesome", "guest", "root"}, s.err
  74. }
  75. func (s *mockAuthStore) GetRole(name string) (auth.Role, error) {
  76. r, ok := s.roles[name]
  77. if ok {
  78. return *r, s.err
  79. }
  80. return auth.Role{}, fmt.Errorf("%q does not exist (%v)", name, s.err)
  81. }
  82. func (s *mockAuthStore) CreateRole(role auth.Role) error { return s.err }
  83. func (s *mockAuthStore) DeleteRole(name string) error { return s.err }
  84. func (s *mockAuthStore) UpdateRole(role auth.Role) (auth.Role, error) {
  85. return *s.roles[role.Role], s.err
  86. }
  87. func (s *mockAuthStore) AuthEnabled() bool { return s.enabled }
  88. func (s *mockAuthStore) EnableAuth() error { return s.err }
  89. func (s *mockAuthStore) DisableAuth() error { return s.err }
  90. func (s *mockAuthStore) CheckPassword(user auth.User, password string) bool {
  91. return user.Password == password
  92. }
  93. func (s *mockAuthStore) HashPassword(password string) (string, error) {
  94. return password, nil
  95. }
  96. func TestAuthFlow(t *testing.T) {
  97. api.EnableCapability(api.AuthCapability)
  98. var testCases = []struct {
  99. req *http.Request
  100. store mockAuthStore
  101. wcode int
  102. wbody string
  103. }{
  104. {
  105. req: mustJSONRequest(t, "PUT", "users/alice", `{{{{{{{`),
  106. store: mockAuthStore{},
  107. wcode: http.StatusBadRequest,
  108. wbody: `{"message":"Invalid JSON in request body."}`,
  109. },
  110. {
  111. req: mustJSONRequest(t, "PUT", "users/alice", `{"user": "alice", "password": "goodpassword"}`),
  112. store: mockAuthStore{enabled: true},
  113. wcode: http.StatusUnauthorized,
  114. wbody: `{"message":"Insufficient credentials"}`,
  115. },
  116. // Users
  117. {
  118. req: mustJSONRequest(t, "GET", "users", ""),
  119. store: mockAuthStore{
  120. users: map[string]*auth.User{
  121. "alice": {
  122. User: "alice",
  123. Roles: []string{"alicerole", "guest"},
  124. Password: "wheeee",
  125. },
  126. "bob": {
  127. User: "bob",
  128. Roles: []string{"guest"},
  129. Password: "wheeee",
  130. },
  131. "root": {
  132. User: "root",
  133. Roles: []string{"root"},
  134. Password: "wheeee",
  135. },
  136. },
  137. roles: map[string]*auth.Role{
  138. "alicerole": {
  139. Role: "alicerole",
  140. },
  141. "guest": {
  142. Role: "guest",
  143. },
  144. "root": {
  145. Role: "root",
  146. },
  147. },
  148. },
  149. wcode: http.StatusOK,
  150. wbody: `{"users":[` +
  151. `{"user":"alice","roles":[` +
  152. `{"role":"alicerole","permissions":{"kv":{"read":null,"write":null}}},` +
  153. `{"role":"guest","permissions":{"kv":{"read":null,"write":null}}}` +
  154. `]},` +
  155. `{"user":"bob","roles":[{"role":"guest","permissions":{"kv":{"read":null,"write":null}}}]},` +
  156. `{"user":"root","roles":[{"role":"root","permissions":{"kv":{"read":null,"write":null}}}]}]}`,
  157. },
  158. {
  159. req: mustJSONRequest(t, "GET", "users/alice", ""),
  160. store: mockAuthStore{
  161. users: map[string]*auth.User{
  162. "alice": {
  163. User: "alice",
  164. Roles: []string{"alicerole"},
  165. Password: "wheeee",
  166. },
  167. },
  168. roles: map[string]*auth.Role{
  169. "alicerole": {
  170. Role: "alicerole",
  171. },
  172. },
  173. },
  174. wcode: http.StatusOK,
  175. wbody: `{"user":"alice","roles":[{"role":"alicerole","permissions":{"kv":{"read":null,"write":null}}}]}`,
  176. },
  177. {
  178. req: mustJSONRequest(t, "PUT", "users/alice", `{"user": "alice", "password": "goodpassword"}`),
  179. store: mockAuthStore{},
  180. wcode: http.StatusCreated,
  181. wbody: `{"user":"alice","roles":null}`,
  182. },
  183. {
  184. req: mustJSONRequest(t, "DELETE", "users/alice", ``),
  185. store: mockAuthStore{},
  186. wcode: http.StatusOK,
  187. wbody: ``,
  188. },
  189. {
  190. req: mustJSONRequest(t, "PUT", "users/alice", `{"user": "alice", "password": "goodpassword"}`),
  191. store: mockAuthStore{
  192. users: map[string]*auth.User{
  193. "alice": {
  194. User: "alice",
  195. Roles: []string{"alicerole", "guest"},
  196. Password: "wheeee",
  197. },
  198. },
  199. },
  200. wcode: http.StatusOK,
  201. wbody: `{"user":"alice","roles":["alicerole","guest"]}`,
  202. },
  203. {
  204. req: mustJSONRequest(t, "PUT", "users/alice", `{"user": "alice", "grant": ["alicerole"]}`),
  205. store: mockAuthStore{
  206. users: map[string]*auth.User{
  207. "alice": {
  208. User: "alice",
  209. Roles: []string{"alicerole", "guest"},
  210. Password: "wheeee",
  211. },
  212. },
  213. },
  214. wcode: http.StatusOK,
  215. wbody: `{"user":"alice","roles":["alicerole","guest"]}`,
  216. },
  217. {
  218. req: mustJSONRequest(t, "GET", "users/alice", ``),
  219. store: mockAuthStore{
  220. users: map[string]*auth.User{},
  221. err: auth.Error{Status: http.StatusNotFound, Errmsg: "auth: User alice doesn't exist."},
  222. },
  223. wcode: http.StatusNotFound,
  224. wbody: `{"message":"auth: User alice doesn't exist."}`,
  225. },
  226. {
  227. req: mustJSONRequest(t, "GET", "roles/manager", ""),
  228. store: mockAuthStore{
  229. roles: map[string]*auth.Role{
  230. "manager": {
  231. Role: "manager",
  232. },
  233. },
  234. },
  235. wcode: http.StatusOK,
  236. wbody: `{"role":"manager","permissions":{"kv":{"read":null,"write":null}}}`,
  237. },
  238. {
  239. req: mustJSONRequest(t, "DELETE", "roles/manager", ``),
  240. store: mockAuthStore{},
  241. wcode: http.StatusOK,
  242. wbody: ``,
  243. },
  244. {
  245. req: mustJSONRequest(t, "PUT", "roles/manager", `{"role":"manager","permissions":{"kv":{"read":[],"write":[]}}}`),
  246. store: mockAuthStore{},
  247. wcode: http.StatusCreated,
  248. wbody: `{"role":"manager","permissions":{"kv":{"read":[],"write":[]}}}`,
  249. },
  250. {
  251. req: mustJSONRequest(t, "PUT", "roles/manager", `{"role":"manager","revoke":{"kv":{"read":["foo"],"write":[]}}}`),
  252. store: mockAuthStore{
  253. roles: map[string]*auth.Role{
  254. "manager": {
  255. Role: "manager",
  256. },
  257. },
  258. },
  259. wcode: http.StatusOK,
  260. wbody: `{"role":"manager","permissions":{"kv":{"read":null,"write":null}}}`,
  261. },
  262. {
  263. req: mustJSONRequest(t, "GET", "roles", ""),
  264. store: mockAuthStore{
  265. roles: map[string]*auth.Role{
  266. "awesome": {
  267. Role: "awesome",
  268. },
  269. "guest": {
  270. Role: "guest",
  271. },
  272. "root": {
  273. Role: "root",
  274. },
  275. },
  276. },
  277. wcode: http.StatusOK,
  278. wbody: `{"roles":[{"role":"awesome","permissions":{"kv":{"read":null,"write":null}}},` +
  279. `{"role":"guest","permissions":{"kv":{"read":null,"write":null}}},` +
  280. `{"role":"root","permissions":{"kv":{"read":null,"write":null}}}]}`,
  281. },
  282. {
  283. req: mustJSONRequest(t, "GET", "enable", ""),
  284. store: mockAuthStore{
  285. enabled: true,
  286. },
  287. wcode: http.StatusOK,
  288. wbody: `{"enabled":true}`,
  289. },
  290. {
  291. req: mustJSONRequest(t, "PUT", "enable", ""),
  292. store: mockAuthStore{
  293. enabled: false,
  294. },
  295. wcode: http.StatusOK,
  296. wbody: ``,
  297. },
  298. {
  299. req: (func() *http.Request {
  300. req := mustJSONRequest(t, "DELETE", "enable", "")
  301. req.SetBasicAuth("root", "good")
  302. return req
  303. })(),
  304. store: mockAuthStore{
  305. enabled: true,
  306. users: map[string]*auth.User{
  307. "root": {
  308. User: "root",
  309. Password: goodPassword,
  310. Roles: []string{"root"},
  311. },
  312. },
  313. roles: map[string]*auth.Role{
  314. "root": {
  315. Role: "root",
  316. },
  317. },
  318. },
  319. wcode: http.StatusOK,
  320. wbody: ``,
  321. },
  322. {
  323. req: (func() *http.Request {
  324. req := mustJSONRequest(t, "DELETE", "enable", "")
  325. req.SetBasicAuth("root", "bad")
  326. return req
  327. })(),
  328. store: mockAuthStore{
  329. enabled: true,
  330. users: map[string]*auth.User{
  331. "root": {
  332. User: "root",
  333. Password: goodPassword,
  334. Roles: []string{"root"},
  335. },
  336. },
  337. roles: map[string]*auth.Role{
  338. "root": {
  339. Role: "guest",
  340. },
  341. },
  342. },
  343. wcode: http.StatusUnauthorized,
  344. wbody: `{"message":"Insufficient credentials"}`,
  345. },
  346. }
  347. for i, tt := range testCases {
  348. mux := http.NewServeMux()
  349. h := &authHandler{
  350. sec: &tt.store,
  351. cluster: &fakeCluster{id: 1},
  352. }
  353. handleAuth(mux, h)
  354. rw := httptest.NewRecorder()
  355. mux.ServeHTTP(rw, tt.req)
  356. if rw.Code != tt.wcode {
  357. t.Errorf("#%d: got code=%d, want %d", i, rw.Code, tt.wcode)
  358. }
  359. g := rw.Body.String()
  360. g = strings.TrimSpace(g)
  361. if g != tt.wbody {
  362. t.Errorf("#%d: got body=%s, want %s", i, g, tt.wbody)
  363. }
  364. }
  365. }
  366. func TestGetUserGrantedWithNonexistingRole(t *testing.T) {
  367. sh := &authHandler{
  368. sec: &mockAuthStore{
  369. users: map[string]*auth.User{
  370. "root": {
  371. User: "root",
  372. Roles: []string{"root", "foo"},
  373. },
  374. },
  375. roles: map[string]*auth.Role{
  376. "root": {
  377. Role: "root",
  378. },
  379. },
  380. },
  381. cluster: &fakeCluster{id: 1},
  382. }
  383. srv := httptest.NewServer(http.HandlerFunc(sh.baseUsers))
  384. defer srv.Close()
  385. req, err := http.NewRequest("GET", "", nil)
  386. if err != nil {
  387. t.Fatal(err)
  388. }
  389. req.URL, err = url.Parse(srv.URL)
  390. if err != nil {
  391. t.Fatal(err)
  392. }
  393. req.Header.Set("Content-Type", "application/json")
  394. cli := http.DefaultClient
  395. resp, err := cli.Do(req)
  396. if err != nil {
  397. t.Fatal(err)
  398. }
  399. defer resp.Body.Close()
  400. var uc usersCollections
  401. if err := json.NewDecoder(resp.Body).Decode(&uc); err != nil {
  402. t.Fatal(err)
  403. }
  404. if len(uc.Users) != 1 {
  405. t.Fatalf("expected 1 user, got %+v", uc.Users)
  406. }
  407. if uc.Users[0].User != "root" {
  408. t.Fatalf("expected 'root', got %q", uc.Users[0].User)
  409. }
  410. if len(uc.Users[0].Roles) != 1 {
  411. t.Fatalf("expected 1 role, got %+v", uc.Users[0].Roles)
  412. }
  413. if uc.Users[0].Roles[0].Role != "root" {
  414. t.Fatalf("expected 'root', got %q", uc.Users[0].Roles[0].Role)
  415. }
  416. }
  417. func mustAuthRequest(method, username, password string) *http.Request {
  418. req, err := http.NewRequest(method, "path", strings.NewReader(""))
  419. if err != nil {
  420. panic("Cannot make auth request: " + err.Error())
  421. }
  422. req.SetBasicAuth(username, password)
  423. return req
  424. }
  425. func TestPrefixAccess(t *testing.T) {
  426. var table = []struct {
  427. key string
  428. req *http.Request
  429. store *mockAuthStore
  430. hasRoot bool
  431. hasKeyPrefixAccess bool
  432. hasRecursiveAccess bool
  433. }{
  434. {
  435. key: "/foo",
  436. req: mustAuthRequest("GET", "root", "good"),
  437. store: &mockAuthStore{
  438. users: map[string]*auth.User{
  439. "root": {
  440. User: "root",
  441. Password: goodPassword,
  442. Roles: []string{"root"},
  443. },
  444. },
  445. roles: map[string]*auth.Role{
  446. "root": {
  447. Role: "root",
  448. },
  449. },
  450. enabled: true,
  451. },
  452. hasRoot: true,
  453. hasKeyPrefixAccess: true,
  454. hasRecursiveAccess: true,
  455. },
  456. {
  457. key: "/foo",
  458. req: mustAuthRequest("GET", "user", "good"),
  459. store: &mockAuthStore{
  460. users: map[string]*auth.User{
  461. "user": {
  462. User: "user",
  463. Password: goodPassword,
  464. Roles: []string{"foorole"},
  465. },
  466. },
  467. roles: map[string]*auth.Role{
  468. "foorole": {
  469. Role: "foorole",
  470. Permissions: auth.Permissions{
  471. KV: auth.RWPermission{
  472. Read: []string{"/foo"},
  473. Write: []string{"/foo"},
  474. },
  475. },
  476. },
  477. },
  478. enabled: true,
  479. },
  480. hasRoot: false,
  481. hasKeyPrefixAccess: true,
  482. hasRecursiveAccess: false,
  483. },
  484. {
  485. key: "/foo",
  486. req: mustAuthRequest("GET", "user", "good"),
  487. store: &mockAuthStore{
  488. users: map[string]*auth.User{
  489. "user": {
  490. User: "user",
  491. Password: goodPassword,
  492. Roles: []string{"foorole"},
  493. },
  494. },
  495. roles: map[string]*auth.Role{
  496. "foorole": {
  497. Role: "foorole",
  498. Permissions: auth.Permissions{
  499. KV: auth.RWPermission{
  500. Read: []string{"/foo*"},
  501. Write: []string{"/foo*"},
  502. },
  503. },
  504. },
  505. },
  506. enabled: true,
  507. },
  508. hasRoot: false,
  509. hasKeyPrefixAccess: true,
  510. hasRecursiveAccess: true,
  511. },
  512. {
  513. key: "/foo",
  514. req: mustAuthRequest("GET", "user", "bad"),
  515. store: &mockAuthStore{
  516. users: map[string]*auth.User{
  517. "user": {
  518. User: "user",
  519. Password: goodPassword,
  520. Roles: []string{"foorole"},
  521. },
  522. },
  523. roles: map[string]*auth.Role{
  524. "foorole": {
  525. Role: "foorole",
  526. Permissions: auth.Permissions{
  527. KV: auth.RWPermission{
  528. Read: []string{"/foo*"},
  529. Write: []string{"/foo*"},
  530. },
  531. },
  532. },
  533. },
  534. enabled: true,
  535. },
  536. hasRoot: false,
  537. hasKeyPrefixAccess: false,
  538. hasRecursiveAccess: false,
  539. },
  540. {
  541. key: "/foo",
  542. req: mustAuthRequest("GET", "user", "good"),
  543. store: &mockAuthStore{
  544. users: map[string]*auth.User{},
  545. err: errors.New("Not the user"),
  546. enabled: true,
  547. },
  548. hasRoot: false,
  549. hasKeyPrefixAccess: false,
  550. hasRecursiveAccess: false,
  551. },
  552. {
  553. key: "/foo",
  554. req: mustJSONRequest(t, "GET", "somepath", ""),
  555. store: &mockAuthStore{
  556. users: map[string]*auth.User{
  557. "user": {
  558. User: "user",
  559. Password: goodPassword,
  560. Roles: []string{"foorole"},
  561. },
  562. },
  563. roles: map[string]*auth.Role{
  564. "guest": {
  565. Role: "guest",
  566. Permissions: auth.Permissions{
  567. KV: auth.RWPermission{
  568. Read: []string{"/foo*"},
  569. Write: []string{"/foo*"},
  570. },
  571. },
  572. },
  573. },
  574. enabled: true,
  575. },
  576. hasRoot: false,
  577. hasKeyPrefixAccess: true,
  578. hasRecursiveAccess: true,
  579. },
  580. {
  581. key: "/bar",
  582. req: mustJSONRequest(t, "GET", "somepath", ""),
  583. store: &mockAuthStore{
  584. users: map[string]*auth.User{
  585. "user": {
  586. User: "user",
  587. Password: goodPassword,
  588. Roles: []string{"foorole"},
  589. },
  590. },
  591. roles: map[string]*auth.Role{
  592. "guest": {
  593. Role: "guest",
  594. Permissions: auth.Permissions{
  595. KV: auth.RWPermission{
  596. Read: []string{"/foo*"},
  597. Write: []string{"/foo*"},
  598. },
  599. },
  600. },
  601. },
  602. enabled: true,
  603. },
  604. hasRoot: false,
  605. hasKeyPrefixAccess: false,
  606. hasRecursiveAccess: false,
  607. },
  608. // check access for multiple roles
  609. {
  610. key: "/foo",
  611. req: mustAuthRequest("GET", "user", "good"),
  612. store: &mockAuthStore{
  613. users: map[string]*auth.User{
  614. "user": {
  615. User: "user",
  616. Password: goodPassword,
  617. Roles: []string{"role1", "role2"},
  618. },
  619. },
  620. roles: map[string]*auth.Role{
  621. "role1": {
  622. Role: "role1",
  623. },
  624. "role2": {
  625. Role: "role2",
  626. Permissions: auth.Permissions{
  627. KV: auth.RWPermission{
  628. Read: []string{"/foo"},
  629. Write: []string{"/foo"},
  630. },
  631. },
  632. },
  633. },
  634. enabled: true,
  635. },
  636. hasRoot: false,
  637. hasKeyPrefixAccess: true,
  638. hasRecursiveAccess: false,
  639. },
  640. {
  641. key: "/foo",
  642. req: (func() *http.Request {
  643. req := mustJSONRequest(t, "GET", "somepath", "")
  644. req.Header.Set("Authorization", "malformedencoding")
  645. return req
  646. })(),
  647. store: &mockAuthStore{
  648. enabled: true,
  649. users: map[string]*auth.User{
  650. "root": {
  651. User: "root",
  652. Password: goodPassword,
  653. Roles: []string{"root"},
  654. },
  655. },
  656. roles: map[string]*auth.Role{
  657. "guest": {
  658. Role: "guest",
  659. Permissions: auth.Permissions{
  660. KV: auth.RWPermission{
  661. Read: []string{"/foo*"},
  662. Write: []string{"/foo*"},
  663. },
  664. },
  665. },
  666. },
  667. },
  668. hasRoot: false,
  669. hasKeyPrefixAccess: false,
  670. hasRecursiveAccess: false,
  671. },
  672. }
  673. for i, tt := range table {
  674. if tt.hasRoot != hasRootAccess(tt.store, tt.req) {
  675. t.Errorf("#%d: hasRoot doesn't match (expected %v)", i, tt.hasRoot)
  676. }
  677. if tt.hasKeyPrefixAccess != hasKeyPrefixAccess(tt.store, tt.req, tt.key, false) {
  678. t.Errorf("#%d: hasKeyPrefixAccess doesn't match (expected %v)", i, tt.hasRoot)
  679. }
  680. if tt.hasRecursiveAccess != hasKeyPrefixAccess(tt.store, tt.req, tt.key, true) {
  681. t.Errorf("#%d: hasRecursiveAccess doesn't match (expected %v)", i, tt.hasRoot)
  682. }
  683. }
  684. }