etcd-io__etcd/rafthttp/pipeline.go

// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rafthttp

import (
	"bytes"
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	"github.com/coreos/etcd/etcdserver/stats"
	"github.com/coreos/etcd/pkg/pbutil"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
)

const (
	connPerPipeline = 4
	// pipelineBufSize is the size of pipeline buffer, which helps hold the
	// temporary network latency.
	// The size ensures that pipeline does not drop messages when the network
	// is out of work for less than 1 second in good path.
	pipelineBufSize = 64
)

type pipeline struct {
	id  types.ID
	cid types.ID

	tr http.RoundTripper
	// the url this pipeline sends to
	u      string
	fs     *stats.FollowerStats
	errorc chan error

	q chan *raftpb.Message
	// wait for the handling routines
	wg sync.WaitGroup
	sync.Mutex
	// if the last send was successful, the pipeline is active.
	// Or it is inactive
	active  bool
	errored error
}

func newPipeline(tr http.RoundTripper, u string, id, cid types.ID, fs *stats.FollowerStats, errorc chan error) *pipeline {
	p := &pipeline{
		id:     id,
		cid:    cid,
		tr:     tr,
		u:      u,
		fs:     fs,
		errorc: errorc,
		q:      make(chan *raftpb.Message, pipelineBufSize),
		active: true,
	}
	p.wg.Add(connPerPipeline)
	for i := 0; i < connPerPipeline; i++ {
		go p.handle()
	}
	return p
}

func (p *pipeline) update(u string) { p.u = u }

func (p *pipeline) send(m raftpb.Message) error {
	// TODO: don't block. we should be able to have 1000s
	// of messages out at a time.
	select {
	case p.q <- &m:
		return nil
	default:
		log.Printf("pipeline: dropping %s because maximal number %d of pipeline buffer entries to %s has been reached",
			m.Type, pipelineBufSize, p.u)
		return fmt.Errorf("reach maximal serving")
	}
}

func (p *pipeline) stop() {
	close(p.q)
	p.wg.Wait()
}

func (p *pipeline) handle() {
	defer p.wg.Done()
	for m := range p.q {
		start := time.Now()
		err := p.pipeline(pbutil.MustMarshal(m))
		end := time.Now()

		p.Lock()
		if err != nil {
			if p.errored == nil || p.errored.Error() != err.Error() {
				log.Printf("pipeline: error posting to %s: %v", p.id, err)
				p.errored = err
			}
			if p.active {
				log.Printf("pipeline: the connection with %s became inactive", p.id)
				p.active = false
			}
			if m.Type == raftpb.MsgApp {
				p.fs.Fail()
			}
		} else {
			if !p.active {
				log.Printf("pipeline: the connection with %s became active", p.id)
				p.active = true
				p.errored = nil
			}
			if m.Type == raftpb.MsgApp {
				p.fs.Succ(end.Sub(start))
			}
		}
		p.Unlock()
	}
}

// post POSTs a data payload to a url. Returns nil if the POST succeeds,
// error on any failure.
func (p *pipeline) pipeline(data []byte) error {
	p.Lock()
	req, err := http.NewRequest("POST", p.u, bytes.NewBuffer(data))
	p.Unlock()
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/protobuf")
	req.Header.Set("X-Etcd-Cluster-ID", p.cid.String())
	resp, err := p.tr.RoundTrip(req)
	if err != nil {
		return err
	}
	resp.Body.Close()

	switch resp.StatusCode {
	case http.StatusPreconditionFailed:
		err := fmt.Errorf("conflicting cluster ID with the target cluster (%s != %s)", resp.Header.Get("X-Etcd-Cluster-ID"), p.cid)
		select {
		case p.errorc <- err:
		default:
		}
		return nil
	case http.StatusForbidden:
		err := fmt.Errorf("the member has been permanently removed from the cluster")
		select {
		case p.errorc <- err:
		default:
		}
		return nil
	case http.StatusNoContent:
		return nil
	default:
		return fmt.Errorf("unexpected http status %s while posting to %q", http.StatusText(resp.StatusCode), req.URL.String())
	}
}