|
@@ -45,49 +45,49 @@ func (tt *tester) runLoop() {
|
|
|
tt.status.setCase(j)
|
|
tt.status.setCase(j)
|
|
|
|
|
|
|
|
if err := tt.cluster.WaitHealth(); err != nil {
|
|
if err := tt.cluster.WaitHealth(); err != nil {
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] wait full health error: %v", i, j, err)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] wait full health error: %v", i, j, err)
|
|
|
if err := tt.cleanup(i, j); err != nil {
|
|
if err := tt.cleanup(i, j); err != nil {
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] cleanup error: %v", i, j, err)
|
|
|
return
|
|
return
|
|
|
}
|
|
}
|
|
|
continue
|
|
continue
|
|
|
}
|
|
}
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] start failure %s", i, j, f.Desc())
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] start failure %s", i, j, f.Desc())
|
|
|
|
|
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] start injecting failure...", i, j)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] start injecting failure...", i, j)
|
|
|
if err := f.Inject(tt.cluster, i); err != nil {
|
|
if err := f.Inject(tt.cluster, i); err != nil {
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] injection error: %v", i, j, err)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] injection error: %v", i, j, err)
|
|
|
if err := tt.cleanup(i, j); err != nil {
|
|
if err := tt.cleanup(i, j); err != nil {
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] cleanup error: %v", i, j, err)
|
|
|
return
|
|
return
|
|
|
}
|
|
}
|
|
|
continue
|
|
continue
|
|
|
}
|
|
}
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] injected failure", i, j)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] injected failure", i, j)
|
|
|
|
|
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] start recovering failure...", i, j)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] start recovering failure...", i, j)
|
|
|
if err := f.Recover(tt.cluster, i); err != nil {
|
|
if err := f.Recover(tt.cluster, i); err != nil {
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] recovery error: %v", i, j, err)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] recovery error: %v", i, j, err)
|
|
|
if err := tt.cleanup(i, j); err != nil {
|
|
if err := tt.cleanup(i, j); err != nil {
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] cleanup error: %v", i, j, err)
|
|
|
return
|
|
return
|
|
|
}
|
|
}
|
|
|
continue
|
|
continue
|
|
|
}
|
|
}
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] recovered failure", i, j)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] recovered failure", i, j)
|
|
|
|
|
|
|
|
if tt.cluster.v2Only {
|
|
if tt.cluster.v2Only {
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] succeed!", i, j)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] succeed!", i, j)
|
|
|
continue
|
|
continue
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] canceling the stressers...", i, j)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] canceling the stressers...", i, j)
|
|
|
for _, s := range tt.cluster.Stressers {
|
|
for _, s := range tt.cluster.Stressers {
|
|
|
s.Cancel()
|
|
s.Cancel()
|
|
|
}
|
|
}
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] canceled stressers", i, j)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] canceled stressers", i, j)
|
|
|
|
|
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] checking current revisions...", i, j)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] checking current revisions...", i, j)
|
|
|
var (
|
|
var (
|
|
|
revs map[string]int64
|
|
revs map[string]int64
|
|
|
hashes map[string]int64
|
|
hashes map[string]int64
|
|
@@ -99,55 +99,55 @@ func (tt *tester) runLoop() {
|
|
|
|
|
|
|
|
revs, hashes, rerr = tt.cluster.getRevisionHash()
|
|
revs, hashes, rerr = tt.cluster.getRevisionHash()
|
|
|
if rerr != nil {
|
|
if rerr != nil {
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d.%d] failed to get current revisions (%v)", i, j, k, rerr)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d.%d] failed to get current revisions (%v)", i, j, k, rerr)
|
|
|
continue
|
|
continue
|
|
|
}
|
|
}
|
|
|
if currentRevision, ok = getSameValue(revs); ok {
|
|
if currentRevision, ok = getSameValue(revs); ok {
|
|
|
break
|
|
break
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d.%d] inconsistent current revisions %+v", i, j, k, revs)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d.%d] inconsistent current revisions %+v", i, j, k, revs)
|
|
|
}
|
|
}
|
|
|
if !ok || rerr != nil {
|
|
if !ok || rerr != nil {
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] checking current revisions failed (%v)", i, j, revs)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] checking current revisions failed (%v)", i, j, revs)
|
|
|
if err := tt.cleanup(i, j); err != nil {
|
|
if err := tt.cleanup(i, j); err != nil {
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] cleanup error: %v", i, j, err)
|
|
|
return
|
|
return
|
|
|
}
|
|
}
|
|
|
continue
|
|
continue
|
|
|
}
|
|
}
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] all members are consistent with current revisions", i, j)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] all members are consistent with current revisions", i, j)
|
|
|
|
|
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] checking current storage hashes...", i, j)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] checking current storage hashes...", i, j)
|
|
|
if _, ok = getSameValue(hashes); !ok {
|
|
if _, ok = getSameValue(hashes); !ok {
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] checking current storage hashes failed (%v)", i, j, hashes)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] checking current storage hashes failed (%v)", i, j, hashes)
|
|
|
if err := tt.cleanup(i, j); err != nil {
|
|
if err := tt.cleanup(i, j); err != nil {
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] cleanup error: %v", i, j, err)
|
|
|
return
|
|
return
|
|
|
}
|
|
}
|
|
|
continue
|
|
continue
|
|
|
}
|
|
}
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] all members are consistent with storage hashes", i, j)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] all members are consistent with storage hashes", i, j)
|
|
|
|
|
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] restarting the stressers...", i, j)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] restarting the stressers...", i, j)
|
|
|
for _, s := range tt.cluster.Stressers {
|
|
for _, s := range tt.cluster.Stressers {
|
|
|
go s.Stress()
|
|
go s.Stress()
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] succeed!", i, j)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] succeed!", i, j)
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
revToCompact := max(0, currentRevision-10000)
|
|
revToCompact := max(0, currentRevision-10000)
|
|
|
- plog.Printf("etcd-tester: [round#%d] compacting storage at %d (current revision %d)", i, revToCompact, currentRevision)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d] compacting storage at %d (current revision %d)", i, revToCompact, currentRevision)
|
|
|
if err := tt.cluster.compactKV(revToCompact); err != nil {
|
|
if err := tt.cluster.compactKV(revToCompact); err != nil {
|
|
|
- plog.Printf("etcd-tester: [round#%d] compactKV error (%v)", i, err)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d] compactKV error (%v)", i, err)
|
|
|
if err := tt.cleanup(i, 0); err != nil {
|
|
if err := tt.cleanup(i, 0); err != nil {
|
|
|
- plog.Printf("etcd-tester: [round#%d] cleanup error: %v", i, err)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d] cleanup error: %v", i, err)
|
|
|
return
|
|
return
|
|
|
}
|
|
}
|
|
|
continue
|
|
continue
|
|
|
}
|
|
}
|
|
|
- plog.Printf("etcd-tester: [round#%d] compacted storage", i)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d] compacted storage", i)
|
|
|
|
|
|
|
|
// TODO: make sure compaction is finished
|
|
// TODO: make sure compaction is finished
|
|
|
time.Sleep(30 * time.Second)
|
|
time.Sleep(30 * time.Second)
|
|
@@ -158,7 +158,7 @@ func (tt *tester) cleanup(i, j int) error {
|
|
|
roundFailedTotalCounter.Inc()
|
|
roundFailedTotalCounter.Inc()
|
|
|
caseFailedTotalCounter.WithLabelValues(tt.failures[j].Desc()).Inc()
|
|
caseFailedTotalCounter.WithLabelValues(tt.failures[j].Desc()).Inc()
|
|
|
|
|
|
|
|
- plog.Printf("etcd-tester: [round#%d case#%d] cleaning up...", i, j)
|
|
|
|
|
|
|
+ plog.Printf("[round#%d case#%d] cleaning up...", i, j)
|
|
|
if err := tt.cluster.Cleanup(); err != nil {
|
|
if err := tt.cluster.Cleanup(); err != nil {
|
|
|
return err
|
|
return err
|
|
|
}
|
|
}
|