config.go 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604
  1. package sarama
  2. import (
  3. "compress/gzip"
  4. "crypto/tls"
  5. "fmt"
  6. "io/ioutil"
  7. "net"
  8. "regexp"
  9. "time"
  10. "github.com/rcrowley/go-metrics"
  11. )
  12. const defaultClientID = "sarama"
  13. var validID = regexp.MustCompile(`\A[A-Za-z0-9._-]+\z`)
  14. // Config is used to pass multiple configuration options to Sarama's constructors.
  15. type Config struct {
  16. // Admin is the namespace for ClusterAdmin properties used by the administrative Kafka client.
  17. Admin struct {
  18. // The maximum duration the administrative Kafka client will wait for ClusterAdmin operations,
  19. // including topics, brokers, configurations and ACLs (defaults to 3 seconds).
  20. Timeout time.Duration
  21. }
  22. // Net is the namespace for network-level properties used by the Broker, and
  23. // shared by the Client/Producer/Consumer.
  24. Net struct {
  25. // How many outstanding requests a connection is allowed to have before
  26. // sending on it blocks (default 5).
  27. MaxOpenRequests int
  28. // All three of the below configurations are similar to the
  29. // `socket.timeout.ms` setting in JVM kafka. All of them default
  30. // to 30 seconds.
  31. DialTimeout time.Duration // How long to wait for the initial connection.
  32. ReadTimeout time.Duration // How long to wait for a response.
  33. WriteTimeout time.Duration // How long to wait for a transmit.
  34. TLS struct {
  35. // Whether or not to use TLS when connecting to the broker
  36. // (defaults to false).
  37. Enable bool
  38. // The TLS configuration to use for secure connections if
  39. // enabled (defaults to nil).
  40. Config *tls.Config
  41. }
  42. // SASL based authentication with broker. While there are multiple SASL authentication methods
  43. // the current implementation is limited to plaintext (SASL/PLAIN) authentication
  44. SASL struct {
  45. // Whether or not to use SASL authentication when connecting to the broker
  46. // (defaults to false).
  47. Enable bool
  48. // SASLMechanism is the name of the enabled SASL mechanism.
  49. // Possible values: OAUTHBEARER, PLAIN (defaults to PLAIN).
  50. Mechanism SASLMechanism
  51. // Whether or not to send the Kafka SASL handshake first if enabled
  52. // (defaults to true). You should only set this to false if you're using
  53. // a non-Kafka SASL proxy.
  54. Handshake bool
  55. //username and password for SASL/PLAIN authentication
  56. User string
  57. Password string
  58. // TokenProvider is a user-defined callback for generating
  59. // access tokens for SASL/OAUTHBEARER auth. See the
  60. // AccessTokenProvider interface docs for proper implementation
  61. // guidelines.
  62. TokenProvider AccessTokenProvider
  63. }
  64. // KeepAlive specifies the keep-alive period for an active network connection.
  65. // If zero, keep-alives are disabled. (default is 0: disabled).
  66. KeepAlive time.Duration
  67. // LocalAddr is the local address to use when dialing an
  68. // address. The address must be of a compatible type for the
  69. // network being dialed.
  70. // If nil, a local address is automatically chosen.
  71. LocalAddr net.Addr
  72. }
  73. // Metadata is the namespace for metadata management properties used by the
  74. // Client, and shared by the Producer/Consumer.
  75. Metadata struct {
  76. Retry struct {
  77. // The total number of times to retry a metadata request when the
  78. // cluster is in the middle of a leader election (default 3).
  79. Max int
  80. // How long to wait for leader election to occur before retrying
  81. // (default 250ms). Similar to the JVM's `retry.backoff.ms`.
  82. Backoff time.Duration
  83. }
  84. // How frequently to refresh the cluster metadata in the background.
  85. // Defaults to 10 minutes. Set to 0 to disable. Similar to
  86. // `topic.metadata.refresh.interval.ms` in the JVM version.
  87. RefreshFrequency time.Duration
  88. // Whether to maintain a full set of metadata for all topics, or just
  89. // the minimal set that has been necessary so far. The full set is simpler
  90. // and usually more convenient, but can take up a substantial amount of
  91. // memory if you have many topics and partitions. Defaults to true.
  92. Full bool
  93. }
  94. // Producer is the namespace for configuration related to producing messages,
  95. // used by the Producer.
  96. Producer struct {
  97. // The maximum permitted size of a message (defaults to 1000000). Should be
  98. // set equal to or smaller than the broker's `message.max.bytes`.
  99. MaxMessageBytes int
  100. // The level of acknowledgement reliability needed from the broker (defaults
  101. // to WaitForLocal). Equivalent to the `request.required.acks` setting of the
  102. // JVM producer.
  103. RequiredAcks RequiredAcks
  104. // The maximum duration the broker will wait the receipt of the number of
  105. // RequiredAcks (defaults to 10 seconds). This is only relevant when
  106. // RequiredAcks is set to WaitForAll or a number > 1. Only supports
  107. // millisecond resolution, nanoseconds will be truncated. Equivalent to
  108. // the JVM producer's `request.timeout.ms` setting.
  109. Timeout time.Duration
  110. // The type of compression to use on messages (defaults to no compression).
  111. // Similar to `compression.codec` setting of the JVM producer.
  112. Compression CompressionCodec
  113. // The level of compression to use on messages. The meaning depends
  114. // on the actual compression type used and defaults to default compression
  115. // level for the codec.
  116. CompressionLevel int
  117. // Generates partitioners for choosing the partition to send messages to
  118. // (defaults to hashing the message key). Similar to the `partitioner.class`
  119. // setting for the JVM producer.
  120. Partitioner PartitionerConstructor
  121. // If enabled, the producer will ensure that exactly one copy of each message is
  122. // written.
  123. Idempotent bool
  124. // Return specifies what channels will be populated. If they are set to true,
  125. // you must read from the respective channels to prevent deadlock. If,
  126. // however, this config is used to create a `SyncProducer`, both must be set
  127. // to true and you shall not read from the channels since the producer does
  128. // this internally.
  129. Return struct {
  130. // If enabled, successfully delivered messages will be returned on the
  131. // Successes channel (default disabled).
  132. Successes bool
  133. // If enabled, messages that failed to deliver will be returned on the
  134. // Errors channel, including error (default enabled).
  135. Errors bool
  136. }
  137. // The following config options control how often messages are batched up and
  138. // sent to the broker. By default, messages are sent as fast as possible, and
  139. // all messages received while the current batch is in-flight are placed
  140. // into the subsequent batch.
  141. Flush struct {
  142. // The best-effort number of bytes needed to trigger a flush. Use the
  143. // global sarama.MaxRequestSize to set a hard upper limit.
  144. Bytes int
  145. // The best-effort number of messages needed to trigger a flush. Use
  146. // `MaxMessages` to set a hard upper limit.
  147. Messages int
  148. // The best-effort frequency of flushes. Equivalent to
  149. // `queue.buffering.max.ms` setting of JVM producer.
  150. Frequency time.Duration
  151. // The maximum number of messages the producer will send in a single
  152. // broker request. Defaults to 0 for unlimited. Similar to
  153. // `queue.buffering.max.messages` in the JVM producer.
  154. MaxMessages int
  155. }
  156. Retry struct {
  157. // The total number of times to retry sending a message (default 3).
  158. // Similar to the `message.send.max.retries` setting of the JVM producer.
  159. Max int
  160. // How long to wait for the cluster to settle between retries
  161. // (default 100ms). Similar to the `retry.backoff.ms` setting of the
  162. // JVM producer.
  163. Backoff time.Duration
  164. }
  165. }
  166. // Consumer is the namespace for configuration related to consuming messages,
  167. // used by the Consumer.
  168. Consumer struct {
  169. // Group is the namespace for configuring consumer group.
  170. Group struct {
  171. Session struct {
  172. // The timeout used to detect consumer failures when using Kafka's group management facility.
  173. // The consumer sends periodic heartbeats to indicate its liveness to the broker.
  174. // If no heartbeats are received by the broker before the expiration of this session timeout,
  175. // then the broker will remove this consumer from the group and initiate a rebalance.
  176. // Note that the value must be in the allowable range as configured in the broker configuration
  177. // by `group.min.session.timeout.ms` and `group.max.session.timeout.ms` (default 10s)
  178. Timeout time.Duration
  179. }
  180. Heartbeat struct {
  181. // The expected time between heartbeats to the consumer coordinator when using Kafka's group
  182. // management facilities. Heartbeats are used to ensure that the consumer's session stays active and
  183. // to facilitate rebalancing when new consumers join or leave the group.
  184. // The value must be set lower than Consumer.Group.Session.Timeout, but typically should be set no
  185. // higher than 1/3 of that value.
  186. // It can be adjusted even lower to control the expected time for normal rebalances (default 3s)
  187. Interval time.Duration
  188. }
  189. Rebalance struct {
  190. // Strategy for allocating topic partitions to members (default BalanceStrategyRange)
  191. Strategy BalanceStrategy
  192. // The maximum allowed time for each worker to join the group once a rebalance has begun.
  193. // This is basically a limit on the amount of time needed for all tasks to flush any pending
  194. // data and commit offsets. If the timeout is exceeded, then the worker will be removed from
  195. // the group, which will cause offset commit failures (default 60s).
  196. Timeout time.Duration
  197. Retry struct {
  198. // When a new consumer joins a consumer group the set of consumers attempt to "rebalance"
  199. // the load to assign partitions to each consumer. If the set of consumers changes while
  200. // this assignment is taking place the rebalance will fail and retry. This setting controls
  201. // the maximum number of attempts before giving up (default 4).
  202. Max int
  203. // Backoff time between retries during rebalance (default 2s)
  204. Backoff time.Duration
  205. }
  206. }
  207. Member struct {
  208. // Custom metadata to include when joining the group. The user data for all joined members
  209. // can be retrieved by sending a DescribeGroupRequest to the broker that is the
  210. // coordinator for the group.
  211. UserData []byte
  212. }
  213. }
  214. Retry struct {
  215. // How long to wait after a failing to read from a partition before
  216. // trying again (default 2s).
  217. Backoff time.Duration
  218. }
  219. // Fetch is the namespace for controlling how many bytes are retrieved by any
  220. // given request.
  221. Fetch struct {
  222. // The minimum number of message bytes to fetch in a request - the broker
  223. // will wait until at least this many are available. The default is 1,
  224. // as 0 causes the consumer to spin when no messages are available.
  225. // Equivalent to the JVM's `fetch.min.bytes`.
  226. Min int32
  227. // The default number of message bytes to fetch from the broker in each
  228. // request (default 1MB). This should be larger than the majority of
  229. // your messages, or else the consumer will spend a lot of time
  230. // negotiating sizes and not actually consuming. Similar to the JVM's
  231. // `fetch.message.max.bytes`.
  232. Default int32
  233. // The maximum number of message bytes to fetch from the broker in a
  234. // single request. Messages larger than this will return
  235. // ErrMessageTooLarge and will not be consumable, so you must be sure
  236. // this is at least as large as your largest message. Defaults to 0
  237. // (no limit). Similar to the JVM's `fetch.message.max.bytes`. The
  238. // global `sarama.MaxResponseSize` still applies.
  239. Max int32
  240. }
  241. // The maximum amount of time the broker will wait for Consumer.Fetch.Min
  242. // bytes to become available before it returns fewer than that anyways. The
  243. // default is 250ms, since 0 causes the consumer to spin when no events are
  244. // available. 100-500ms is a reasonable range for most cases. Kafka only
  245. // supports precision up to milliseconds; nanoseconds will be truncated.
  246. // Equivalent to the JVM's `fetch.wait.max.ms`.
  247. MaxWaitTime time.Duration
  248. // The maximum amount of time the consumer expects a message takes to
  249. // process for the user. If writing to the Messages channel takes longer
  250. // than this, that partition will stop fetching more messages until it
  251. // can proceed again.
  252. // Note that, since the Messages channel is buffered, the actual grace time is
  253. // (MaxProcessingTime * ChanneBufferSize). Defaults to 100ms.
  254. // If a message is not written to the Messages channel between two ticks
  255. // of the expiryTicker then a timeout is detected.
  256. // Using a ticker instead of a timer to detect timeouts should typically
  257. // result in many fewer calls to Timer functions which may result in a
  258. // significant performance improvement if many messages are being sent
  259. // and timeouts are infrequent.
  260. // The disadvantage of using a ticker instead of a timer is that
  261. // timeouts will be less accurate. That is, the effective timeout could
  262. // be between `MaxProcessingTime` and `2 * MaxProcessingTime`. For
  263. // example, if `MaxProcessingTime` is 100ms then a delay of 180ms
  264. // between two messages being sent may not be recognized as a timeout.
  265. MaxProcessingTime time.Duration
  266. // Return specifies what channels will be populated. If they are set to true,
  267. // you must read from them to prevent deadlock.
  268. Return struct {
  269. // If enabled, any errors that occurred while consuming are returned on
  270. // the Errors channel (default disabled).
  271. Errors bool
  272. }
  273. // Offsets specifies configuration for how and when to commit consumed
  274. // offsets. This currently requires the manual use of an OffsetManager
  275. // but will eventually be automated.
  276. Offsets struct {
  277. // How frequently to commit updated offsets. Defaults to 1s.
  278. CommitInterval time.Duration
  279. // The initial offset to use if no offset was previously committed.
  280. // Should be OffsetNewest or OffsetOldest. Defaults to OffsetNewest.
  281. Initial int64
  282. // The retention duration for committed offsets. If zero, disabled
  283. // (in which case the `offsets.retention.minutes` option on the
  284. // broker will be used). Kafka only supports precision up to
  285. // milliseconds; nanoseconds will be truncated. Requires Kafka
  286. // broker version 0.9.0 or later.
  287. // (default is 0: disabled).
  288. Retention time.Duration
  289. Retry struct {
  290. // The total number of times to retry failing commit
  291. // requests during OffsetManager shutdown (default 3).
  292. Max int
  293. }
  294. }
  295. }
  296. // A user-provided string sent with every request to the brokers for logging,
  297. // debugging, and auditing purposes. Defaults to "sarama", but you should
  298. // probably set it to something specific to your application.
  299. ClientID string
  300. // The number of events to buffer in internal and external channels. This
  301. // permits the producer and consumer to continue processing some messages
  302. // in the background while user code is working, greatly improving throughput.
  303. // Defaults to 256.
  304. ChannelBufferSize int
  305. // The version of Kafka that Sarama will assume it is running against.
  306. // Defaults to the oldest supported stable version. Since Kafka provides
  307. // backwards-compatibility, setting it to a version older than you have
  308. // will not break anything, although it may prevent you from using the
  309. // latest features. Setting it to a version greater than you are actually
  310. // running may lead to random breakage.
  311. Version KafkaVersion
  312. // The registry to define metrics into.
  313. // Defaults to a local registry.
  314. // If you want to disable metrics gathering, set "metrics.UseNilMetrics" to "true"
  315. // prior to starting Sarama.
  316. // See Examples on how to use the metrics registry
  317. MetricRegistry metrics.Registry
  318. }
  319. // NewConfig returns a new configuration instance with sane defaults.
  320. func NewConfig() *Config {
  321. c := &Config{}
  322. c.Admin.Timeout = 3 * time.Second
  323. c.Net.MaxOpenRequests = 5
  324. c.Net.DialTimeout = 30 * time.Second
  325. c.Net.ReadTimeout = 30 * time.Second
  326. c.Net.WriteTimeout = 30 * time.Second
  327. c.Net.SASL.Handshake = true
  328. c.Metadata.Retry.Max = 3
  329. c.Metadata.Retry.Backoff = 250 * time.Millisecond
  330. c.Metadata.RefreshFrequency = 10 * time.Minute
  331. c.Metadata.Full = true
  332. c.Producer.MaxMessageBytes = 1000000
  333. c.Producer.RequiredAcks = WaitForLocal
  334. c.Producer.Timeout = 10 * time.Second
  335. c.Producer.Partitioner = NewHashPartitioner
  336. c.Producer.Retry.Max = 3
  337. c.Producer.Retry.Backoff = 100 * time.Millisecond
  338. c.Producer.Return.Errors = true
  339. c.Producer.CompressionLevel = CompressionLevelDefault
  340. c.Consumer.Fetch.Min = 1
  341. c.Consumer.Fetch.Default = 1024 * 1024
  342. c.Consumer.Retry.Backoff = 2 * time.Second
  343. c.Consumer.MaxWaitTime = 250 * time.Millisecond
  344. c.Consumer.MaxProcessingTime = 100 * time.Millisecond
  345. c.Consumer.Return.Errors = false
  346. c.Consumer.Offsets.CommitInterval = 1 * time.Second
  347. c.Consumer.Offsets.Initial = OffsetNewest
  348. c.Consumer.Offsets.Retry.Max = 3
  349. c.Consumer.Group.Session.Timeout = 10 * time.Second
  350. c.Consumer.Group.Heartbeat.Interval = 3 * time.Second
  351. c.Consumer.Group.Rebalance.Strategy = BalanceStrategyRange
  352. c.Consumer.Group.Rebalance.Timeout = 60 * time.Second
  353. c.Consumer.Group.Rebalance.Retry.Max = 4
  354. c.Consumer.Group.Rebalance.Retry.Backoff = 2 * time.Second
  355. c.ClientID = defaultClientID
  356. c.ChannelBufferSize = 256
  357. c.Version = MinVersion
  358. c.MetricRegistry = metrics.NewRegistry()
  359. return c
  360. }
  361. // Validate checks a Config instance. It will return a
  362. // ConfigurationError if the specified values don't make sense.
  363. func (c *Config) Validate() error {
  364. // some configuration values should be warned on but not fail completely, do those first
  365. if c.Net.TLS.Enable == false && c.Net.TLS.Config != nil {
  366. Logger.Println("Net.TLS is disabled but a non-nil configuration was provided.")
  367. }
  368. if c.Net.SASL.Enable == false {
  369. if c.Net.SASL.User != "" {
  370. Logger.Println("Net.SASL is disabled but a non-empty username was provided.")
  371. }
  372. if c.Net.SASL.Password != "" {
  373. Logger.Println("Net.SASL is disabled but a non-empty password was provided.")
  374. }
  375. }
  376. if c.Producer.RequiredAcks > 1 {
  377. Logger.Println("Producer.RequiredAcks > 1 is deprecated and will raise an exception with kafka >= 0.8.2.0.")
  378. }
  379. if c.Producer.MaxMessageBytes >= int(MaxRequestSize) {
  380. Logger.Println("Producer.MaxMessageBytes must be smaller than MaxRequestSize; it will be ignored.")
  381. }
  382. if c.Producer.Flush.Bytes >= int(MaxRequestSize) {
  383. Logger.Println("Producer.Flush.Bytes must be smaller than MaxRequestSize; it will be ignored.")
  384. }
  385. if (c.Producer.Flush.Bytes > 0 || c.Producer.Flush.Messages > 0) && c.Producer.Flush.Frequency == 0 {
  386. Logger.Println("Producer.Flush: Bytes or Messages are set, but Frequency is not; messages may not get flushed.")
  387. }
  388. if c.Producer.Timeout%time.Millisecond != 0 {
  389. Logger.Println("Producer.Timeout only supports millisecond resolution; nanoseconds will be truncated.")
  390. }
  391. if c.Consumer.MaxWaitTime < 100*time.Millisecond {
  392. Logger.Println("Consumer.MaxWaitTime is very low, which can cause high CPU and network usage. See documentation for details.")
  393. }
  394. if c.Consumer.MaxWaitTime%time.Millisecond != 0 {
  395. Logger.Println("Consumer.MaxWaitTime only supports millisecond precision; nanoseconds will be truncated.")
  396. }
  397. if c.Consumer.Offsets.Retention%time.Millisecond != 0 {
  398. Logger.Println("Consumer.Offsets.Retention only supports millisecond precision; nanoseconds will be truncated.")
  399. }
  400. if c.Consumer.Group.Session.Timeout%time.Millisecond != 0 {
  401. Logger.Println("Consumer.Group.Session.Timeout only supports millisecond precision; nanoseconds will be truncated.")
  402. }
  403. if c.Consumer.Group.Heartbeat.Interval%time.Millisecond != 0 {
  404. Logger.Println("Consumer.Group.Heartbeat.Interval only supports millisecond precision; nanoseconds will be truncated.")
  405. }
  406. if c.Consumer.Group.Rebalance.Timeout%time.Millisecond != 0 {
  407. Logger.Println("Consumer.Group.Rebalance.Timeout only supports millisecond precision; nanoseconds will be truncated.")
  408. }
  409. if c.ClientID == defaultClientID {
  410. Logger.Println("ClientID is the default of 'sarama', you should consider setting it to something application-specific.")
  411. }
  412. // validate Net values
  413. switch {
  414. case c.Net.MaxOpenRequests <= 0:
  415. return ConfigurationError("Net.MaxOpenRequests must be > 0")
  416. case c.Net.DialTimeout <= 0:
  417. return ConfigurationError("Net.DialTimeout must be > 0")
  418. case c.Net.ReadTimeout <= 0:
  419. return ConfigurationError("Net.ReadTimeout must be > 0")
  420. case c.Net.WriteTimeout <= 0:
  421. return ConfigurationError("Net.WriteTimeout must be > 0")
  422. case c.Net.KeepAlive < 0:
  423. return ConfigurationError("Net.KeepAlive must be >= 0")
  424. case c.Net.SASL.Enable:
  425. // For backwards compatibility, empty mechanism value defaults to PLAIN
  426. isSASLPlain := len(c.Net.SASL.Mechanism) == 0 || c.Net.SASL.Mechanism == SASLTypePlaintext
  427. if isSASLPlain {
  428. if c.Net.SASL.User == "" {
  429. return ConfigurationError("Net.SASL.User must not be empty when SASL is enabled")
  430. }
  431. if c.Net.SASL.Password == "" {
  432. return ConfigurationError("Net.SASL.Password must not be empty when SASL is enabled")
  433. }
  434. } else if c.Net.SASL.Mechanism == SASLTypeOAuth {
  435. if c.Net.SASL.TokenProvider == nil {
  436. return ConfigurationError("An AccessTokenProvider instance must be provided to Net.SASL.User.TokenProvider")
  437. }
  438. } else {
  439. msg := fmt.Sprintf("The SASL mechanism configuration is invalid. Possible values are `%s` and `%s`",
  440. SASLTypeOAuth, SASLTypePlaintext)
  441. return ConfigurationError(msg)
  442. }
  443. }
  444. // validate the Admin values
  445. switch {
  446. case c.Admin.Timeout <= 0:
  447. return ConfigurationError("Admin.Timeout must be > 0")
  448. }
  449. // validate the Metadata values
  450. switch {
  451. case c.Metadata.Retry.Max < 0:
  452. return ConfigurationError("Metadata.Retry.Max must be >= 0")
  453. case c.Metadata.Retry.Backoff < 0:
  454. return ConfigurationError("Metadata.Retry.Backoff must be >= 0")
  455. case c.Metadata.RefreshFrequency < 0:
  456. return ConfigurationError("Metadata.RefreshFrequency must be >= 0")
  457. }
  458. // validate the Producer values
  459. switch {
  460. case c.Producer.MaxMessageBytes <= 0:
  461. return ConfigurationError("Producer.MaxMessageBytes must be > 0")
  462. case c.Producer.RequiredAcks < -1:
  463. return ConfigurationError("Producer.RequiredAcks must be >= -1")
  464. case c.Producer.Timeout <= 0:
  465. return ConfigurationError("Producer.Timeout must be > 0")
  466. case c.Producer.Partitioner == nil:
  467. return ConfigurationError("Producer.Partitioner must not be nil")
  468. case c.Producer.Flush.Bytes < 0:
  469. return ConfigurationError("Producer.Flush.Bytes must be >= 0")
  470. case c.Producer.Flush.Messages < 0:
  471. return ConfigurationError("Producer.Flush.Messages must be >= 0")
  472. case c.Producer.Flush.Frequency < 0:
  473. return ConfigurationError("Producer.Flush.Frequency must be >= 0")
  474. case c.Producer.Flush.MaxMessages < 0:
  475. return ConfigurationError("Producer.Flush.MaxMessages must be >= 0")
  476. case c.Producer.Flush.MaxMessages > 0 && c.Producer.Flush.MaxMessages < c.Producer.Flush.Messages:
  477. return ConfigurationError("Producer.Flush.MaxMessages must be >= Producer.Flush.Messages when set")
  478. case c.Producer.Retry.Max < 0:
  479. return ConfigurationError("Producer.Retry.Max must be >= 0")
  480. case c.Producer.Retry.Backoff < 0:
  481. return ConfigurationError("Producer.Retry.Backoff must be >= 0")
  482. }
  483. if c.Producer.Compression == CompressionLZ4 && !c.Version.IsAtLeast(V0_10_0_0) {
  484. return ConfigurationError("lz4 compression requires Version >= V0_10_0_0")
  485. }
  486. if c.Producer.Compression == CompressionGZIP {
  487. if c.Producer.CompressionLevel != CompressionLevelDefault {
  488. if _, err := gzip.NewWriterLevel(ioutil.Discard, c.Producer.CompressionLevel); err != nil {
  489. return ConfigurationError(fmt.Sprintf("gzip compression does not work with level %d: %v", c.Producer.CompressionLevel, err))
  490. }
  491. }
  492. }
  493. if c.Producer.Idempotent {
  494. if !c.Version.IsAtLeast(V0_11_0_0) {
  495. return ConfigurationError("Idempotent producer requires Version >= V0_11_0_0")
  496. }
  497. if c.Producer.Retry.Max == 0 {
  498. return ConfigurationError("Idempotent producer requires Producer.Retry.Max >= 1")
  499. }
  500. if c.Producer.RequiredAcks != WaitForAll {
  501. return ConfigurationError("Idempotent producer requires Producer.RequiredAcks to be WaitForAll")
  502. }
  503. if c.Net.MaxOpenRequests > 1 {
  504. return ConfigurationError("Idempotent producer requires Net.MaxOpenRequests to be 1")
  505. }
  506. }
  507. // validate the Consumer values
  508. switch {
  509. case c.Consumer.Fetch.Min <= 0:
  510. return ConfigurationError("Consumer.Fetch.Min must be > 0")
  511. case c.Consumer.Fetch.Default <= 0:
  512. return ConfigurationError("Consumer.Fetch.Default must be > 0")
  513. case c.Consumer.Fetch.Max < 0:
  514. return ConfigurationError("Consumer.Fetch.Max must be >= 0")
  515. case c.Consumer.MaxWaitTime < 1*time.Millisecond:
  516. return ConfigurationError("Consumer.MaxWaitTime must be >= 1ms")
  517. case c.Consumer.MaxProcessingTime <= 0:
  518. return ConfigurationError("Consumer.MaxProcessingTime must be > 0")
  519. case c.Consumer.Retry.Backoff < 0:
  520. return ConfigurationError("Consumer.Retry.Backoff must be >= 0")
  521. case c.Consumer.Offsets.CommitInterval <= 0:
  522. return ConfigurationError("Consumer.Offsets.CommitInterval must be > 0")
  523. case c.Consumer.Offsets.Initial != OffsetOldest && c.Consumer.Offsets.Initial != OffsetNewest:
  524. return ConfigurationError("Consumer.Offsets.Initial must be OffsetOldest or OffsetNewest")
  525. case c.Consumer.Offsets.Retry.Max < 0:
  526. return ConfigurationError("Consumer.Offsets.Retry.Max must be >= 0")
  527. }
  528. // validate the Consumer Group values
  529. switch {
  530. case c.Consumer.Group.Session.Timeout <= 2*time.Millisecond:
  531. return ConfigurationError("Consumer.Group.Session.Timeout must be >= 2ms")
  532. case c.Consumer.Group.Heartbeat.Interval < 1*time.Millisecond:
  533. return ConfigurationError("Consumer.Group.Heartbeat.Interval must be >= 1ms")
  534. case c.Consumer.Group.Heartbeat.Interval >= c.Consumer.Group.Session.Timeout:
  535. return ConfigurationError("Consumer.Group.Heartbeat.Interval must be < Consumer.Group.Session.Timeout")
  536. case c.Consumer.Group.Rebalance.Strategy == nil:
  537. return ConfigurationError("Consumer.Group.Rebalance.Strategy must not be empty")
  538. case c.Consumer.Group.Rebalance.Timeout <= time.Millisecond:
  539. return ConfigurationError("Consumer.Group.Rebalance.Timeout must be >= 1ms")
  540. case c.Consumer.Group.Rebalance.Retry.Max < 0:
  541. return ConfigurationError("Consumer.Group.Rebalance.Retry.Max must be >= 0")
  542. case c.Consumer.Group.Rebalance.Retry.Backoff < 0:
  543. return ConfigurationError("Consumer.Group.Rebalance.Retry.Backoff must be >= 0")
  544. }
  545. // validate misc shared values
  546. switch {
  547. case c.ChannelBufferSize < 0:
  548. return ConfigurationError("ChannelBufferSize must be >= 0")
  549. case !validID.MatchString(c.ClientID):
  550. return ConfigurationError("ClientID is invalid")
  551. }
  552. return nil
  553. }