etcd-tester: add retry logic on retriving lease info

getting lease and keys info through raw rpcs rarely experience error such as EOF. This is considered as a failure and causes tester to clean up.
however, they are just transient problem with temporary connection issue which should not be considered as a testing failure. so we add retry logic in case of transient failure.

FIX #6754

tools/functional-tester/etcd-tester/lease_stresser.go

@@ -258,12 +258,18 @@ func (ls *leaseStresser) getLeaseByID(ctx context.Context, leaseID int64) (*pb.L
 }
 
 func (ls *leaseStresser) hasLeaseExpired(ctx context.Context, leaseID int64) (bool, error) {
-	resp, err := ls.getLeaseByID(ctx, leaseID)
-	plog.Debugf("hasLeaseExpired %v resp %v error (%v)", leaseID, resp, err)
-	if rpctypes.Error(err) == rpctypes.ErrLeaseNotFound {
-		return true, nil
+	// keep retrying until lease's state is known or ctx is being canceled
+	for ctx.Err() == nil {
+		resp, err := ls.getLeaseByID(ctx, leaseID)
+		if err == nil {
+			return false, nil
+		}
+		if rpctypes.Error(err) == rpctypes.ErrLeaseNotFound {
+			return true, nil
+		}
+		plog.Warningf("hasLeaseExpired %v resp %v error (%v)", leaseID, resp, err)
 	}
-	return false, err
+	return false, ctx.Err()
 }
 
 // The keys attached to the lease has the format of "<leaseID>_<idx>" where idx is the ordering key creation