Golang学习篇—kafka压缩使用生产和消费

最新推荐文章于 2025-04-23 02:15:51 发布

原创

最新推荐文章于 2025-04-23 02:15:51 发布 · 2k 阅读

0 ·

CC 4.0 BY-SA版权

1.kafka使用依赖包

"github.com/Shopify/sarama"

"github.com/bsm/sarama-cluster"

2.kafka生产和消费端二次封装

Common.go

package XLKafka

import "github.com/Shopify/sarama"

type KafkaCfg struct {
	Producer struct {
		Topic   string
		Brokers string
	}
	Consumer struct {
		Topics     string `xml:"Topics"`
		Brokers    string `xml:"Brokers"`
		GroupID    string `xml:"GroupID"`
		InitOffset string `xml:"InitOffset"` //初始化offset起始位置 Newest Oldest
	}
}

type SASLCfg struct {
	UserName string
	PassWord string
}

type FuncDataCallback func(msg *sarama.ConsumerMessage)    //消费数据回调
type FuncSuccessCallback func(msg *sarama.ProducerMessage) //生产成功数据回调
type FuncErrorCallback func(msg *sarama.ProducerError)     //生产失败数据回调
type FuncEventCallback func(info string, eventType string) //事件回调

生产端：XLProducer.go

package XLKafka

import (
	"fmt"
	"github.com/Shopify/sarama"
	"github.com/kataras/iris/core/errors"
	"strings"
	"sync"
	"time"
)

type KafkaProducer struct {
	producerObj sarama.AsyncProducer
	conf        *KafkaCfg
	successcb   FuncSuccessCallback
	errorcb     FuncErrorCallback
	bExit       bool
	wg          *sync.WaitGroup
}

func (this *KafkaProducer) Init(conf *KafkaCfg, successCB FuncSuccessCallback, errorCB FuncErrorCallback, compressionType sarama.CompressionCodec) (err error) {
	this.conf = conf
	this.successcb = successCB
	this.errorcb = errorCB

	this.bExit = false

	config := sarama.NewConfig()
	config.Producer.Flush.Messages = 10000
	config.Producer.Flush.Frequency = time.Second * 1
	config.Producer.Flush.MaxMessages = 50000
	config.Producer.RequiredAcks = sarama.WaitForLocal
	config.Producer.Partitioner = sarama.NewRandomPartitioner
	config.Version = sarama.V0_10_0_0
	config.Producer.Compression = compressionType //算法类型

	if successCB != nil && errorCB != nil {
		config.Producer.Return.Successes = true
		config.Producer.Return.Errors = true
		this.wg = &sync.WaitGroup{}
		this.wg.Add(1)
		go this.dataSuccessCB()
	}

	this.producerObj, err = sarama.NewAsyncProducer(strings.Split(conf.Producer.Brokers, ","), config)
	if err != nil {
		errString := fmt.Sprintf("生产者创建失败！err=%s", err.Error())
		return errors.New(errString)
	}
	return nil
}

func (this *KafkaProducer) Fini() {
	this.bExit = true
	this.producerObj.AsyncClose()
	this.wg.Wait()
}

func (this *KafkaProducer) dataSuccessCB() {
	defer this.wg.Done()
	last := time.Now()
	for {
		if this.bExit == true && time.Since(last) > time.Second*1 {
			//fmt.Println("数据生产回调线程退出！")
			return
		}

		if this.producerObj == nil {
			time.Sleep(time.Millisecond * 1)
			continue
		}

		select {
		case msg := <-this.producerObj.Successes():
			if msg != nil {
				if this.successcb != nil {
					this.successcb(msg)
				}
				last = time.Now()
			}
		case msg := <-this.producerObj.Errors():
			if msg != nil {
				//fmt.Println("数据生产失败！err=", msg.Err)
				if this.errorcb != nil {
					this.errorcb(msg)
				}
				last = time.Now()
			}
		case <-time.After(1 * time.Second):
			time.Sleep(time.Millisecond * 1)
		}
	}
}

func (this *KafkaProducer) ProducerAsync_Byte(data []byte, pUser interface{}) (err error) {
	return this.producerAsync(&sarama.ProducerMessage{Topic: this.conf.Producer.Topic, Key: nil, Value: sarama.ByteEncoder(data), Metadata: pUser})
}

func (this *KafkaProducer) ProducerAsync_String(data string, pUser interface{}) (err error) {
	return this.producerAsync(&sarama.ProducerMessage{Topic: this.conf.Producer.Topic, Key: nil, Value: sarama.StringEncoder(data), Metadata: pUser})
}

func (this *KafkaProducer) producerAsync(msg *sarama.ProducerMessage) (err error) {
	defer func() {
		errPanic := recover()
		if errPanic != nil {
			errString := fmt.Sprintf("生产错误：err=%s", errPanic)
			err = errors.New(errString)
		}
	}()

	msg.Timestamp = time.Now()
	this.producerObj.Input() <- msg

	return nil
}

消费端：XLConsumer.go

package XLKafka

import (
	"fmt"
	"github.com/Shopify/sarama"
	"github.com/bsm/sarama-cluster"
	"strings"
	"sync"
	"time"
)

type XLConsumer struct {
	offsetCatch *cluster.OffsetStash
	consumerObj *cluster.Consumer
	dataCB      FuncDataCallback
	eventCB     FuncEventCallback
	bExit       bool
	wg          *sync.WaitGroup
	bPause      bool
}

func (this *XLConsumer) Init(conf *KafkaCfg, dataCB FuncDataCallback) (err error) {
	this.dataCB = dataCB
	this.eventCB = nil
	this.offsetCatch = cluster.NewOffsetStash()
	config := cluster.NewConfig()
	config.Group.Return.Notifications = true
	config.Consumer.Offsets.CommitInterval = 1 * time.Second
	config.Version = sarama.V0_10_2_1

	// 初始从最新的offset开始,正式上线后改为OffsetNewest
	//config.Consumer.Offsets.Initial = sarama.OffsetNewest
	switch conf.Consumer.InitOffset {
	case "Newest":
		config.Consumer.Offsets.Initial = sarama.OffsetNewest
	case "Oldest":
		config.Consumer.Offsets.Initial = sarama.OffsetOldest
	default:
		config.Consumer.Offsets.Initial = sarama.OffsetNewest
	}

	fmt.Println("---------- 主题：", conf.Consumer.Topics, "kafkaBroker:", conf.Consumer.Brokers)

	//	连接kafka
	this.consumerObj, err = cluster.NewConsumer(
		strings.Split(conf.Consumer.Brokers, ","),
		conf.Consumer.GroupID,
		strings.Split(conf.Consumer.Topics, ","),
		config)

	if err != nil {
		return err
	}

	this.bExit = false
	this.bPause = false

	this.wg = &sync.WaitGroup{}
	this.wg.Add(3)
	go this.dealDataThread()
	go this.dealEventThread()
	go this.commitOffsetThread()

	return nil
}

func (this *XLConsumer) InitWithSASL(conf *KafkaCfg, sasl *SASLCfg, dataCB FuncDataCallback, eventCB FuncEventCallback) (err error) {

	this.dataCB = dataCB
	this.eventCB = eventCB
	this.offsetCatch = cluster.NewOffsetStash()

	config := cluster.NewConfig()
	config.Group.Return.Notifications = true
	config.Consumer.Offsets.CommitInterval = 1 * time.Second
	switch conf.Consumer.InitOffset {
	case "Newest":
		config.Consumer.Offsets.Initial = sarama.OffsetNewest
	case "Oldest":
		config.Consumer.Offsets.Initial = sarama.OffsetOldest
	default:
		config.Consumer.Offsets.Initial = sarama.OffsetNewest
	}

	// sasl相关配置
	if sasl != nil {
		config.Net.SASL.Enable = true
		config.Net.SASL.Handshake = true
		config.Net.SASL.User = sasl.UserName
		config.Net.SASL.Password = sasl.PassWord
	}

	fmt.Println("---------- 主题：", conf.Consumer.Topics, "kafkaBroker:", conf.Consumer.Brokers)

	//	连接kafka
	this.consumerObj, err = cluster.NewConsumer(
		strings.Split(conf.Consumer.Brokers, ","),
		conf.Consumer.GroupID,
		strings.Split(conf.Consumer.Topics, ","),
		config)

	if err != nil {
		return err
	}

	this.bExit = false
	this.bPause = false

	this.wg = &sync.WaitGroup{}
	this.wg.Add(3)
	go this.dealDataThread()
	go this.dealEventThread()
	go this.commitOffsetThread()

	return nil
}

func (this *XLConsumer) Fini() error {
	this.bExit = true
	this.wg.Wait()
	return this.consumerObj.Close()
}

func (this *XLConsumer) SetEventCB(pFunc FuncEventCallback) {
	this.eventCB = pFunc
}

func (this *XLConsumer) MarkOffset(msg *sarama.ConsumerMessage) {
	this.offsetCatch.MarkOffset(msg, "")
}

func (this *XLConsumer) dealDataThread() {
	defer this.wg.Done()
	for {
		if this.bExit == true {
			fmt.Println("[dealDataThread] 数据处理线程退出！")
			break
		}

		select {
		case msg := <-this.consumerObj.Messages():
			if this.dataCB != nil {
				this.dataCB(msg)
			}
		case <-time.After(1 * time.Second):
			time.Sleep(time.Millisecond * 1)
		}
	}
}

func (this *XLConsumer) dealEventThread() {
	defer this.wg.Done()
	for {
		if this.bExit == true {
			fmt.Println("[dealDataThread] 事件处理线程退出！")
			break
		}

		if this.bPause == true {
			time.Sleep(time.Second)
			continue
		}

		select {
		case err := <-this.consumerObj.Errors():
			if err != nil {
				errStr := fmt.Sprintf("kafka消费者异常，错误：%v", err.Error())
				if this.eventCB != nil {
					this.eventCB(errStr, "Error")
				} else {
					fmt.Println(errStr)
				}
			}
		case msg := <-this.consumerObj.Notifications():
			infoStr := fmt.Sprintf("\nRebalance Type:%v\nClaimed:%v\nReleased:%v\nCurrent:%v", msg.Type, msg.Claimed, msg.Released, msg.Current)
			if this.eventCB != nil {
				this.eventCB(infoStr, "Info")
			} else {
				fmt.Println(infoStr)
			}
		case <-time.After(1 * time.Second):
			time.Sleep(time.Millisecond * 1)
		}
		time.Sleep(time.Millisecond * 10)
	}
}

func (this *XLConsumer) commitOffsetThread() {
	defer this.wg.Done()
	commitTime := time.Now()
	for {
		if this.bExit == true {
			//退出前提交Offset
			err := this.consumerObj.CommitOffsets()
			if err != nil {
				fmt.Println("[commitOffsetThread] 提交offset错误!err=", err)
			}
			fmt.Println("[commitOffsetThread] offset提交线程退出！")
			break
		}

		if time.Since(commitTime) > time.Second {
			if this.offsetCatch != nil {
				offsets := this.offsetCatch.Offsets()
				if len(offsets) != 0 {
					fmt.Println("[commitOffsetThread] 提交offset!", offsets)
					this.consumerObj.MarkOffsets(this.offsetCatch)
					err := this.consumerObj.CommitOffsets()
					if err != nil {
						fmt.Println("[commitOffsetThread] 提交offset错误!err=", err)
					}
				}
			}
			commitTime = time.Now()
		}

		time.Sleep(time.Millisecond * 100)
	}
}

func (this *XLConsumer) CommitOffset() error {
	offsets := this.offsetCatch.Offsets()
	if len(offsets) != 0 {
		fmt.Println("[commitOffsetThread] 提交offset!", offsets)
		this.consumerObj.MarkOffsets(this.offsetCatch)
		err := this.consumerObj.CommitOffsets()
		if err != nil {
			fmt.Println("[commitOffsetThread] 提交offset错误!err=", err)
			return err
		}
	}

	return nil
}

func (this *XLConsumer) Pause() {
	this.bPause = true
}

func (this *XLConsumer) Resume() {
	this.bPause = false
}

3.kafka压缩生产和消费样例

生产端:ProducerTest.go

package main

import (
	. "Alang/common/XLKafka"
	"flag"
	"fmt"
	"github.com/Shopify/sarama"
	"time"
)

var tNow = time.Now()
var count = 0
var topicCount = make(map[string]int)
var c = XLConsumer{}

var tagData = `{"commonInfo":{"operationId":"10012019111317041700000001","operationType":1,"operationTime":"2019-11-13 17:04:17","dataSource":null},"metaData":{"type":1,"name":"常口","libraryId":null,"libraryMark":"1","data":[{"id":"218906b5-f483-4beb-b8e7-b5b589cf4a40","credentialNumber":"7787ce86c379a6d0f604569b7e3ea1ea","cityCode":null,"presentPlace":"650103","communityCode":null,"createDate":"2019-11-13 17:04:17","name":"房","featureValue":"-20,126,50,62,98,-83,-91,-68,-79,-99,23,62,-46,121,34,-68,-50,-88,13,62,98,88,-44,60,-105,-2,-104,61,15,-93,125,-68,-113,-72,-66,60,-80,91,53,61,36,82,-127,-67,29,69,111,61,17,127,-49,61,-54,23,-63,61,101,72,95,-68,53,-122,32,-66,43,127,62,-67,-106,-20,22,61,59,127,-119,60,-92,-105,39,61,-124,88,-118,-67,93,-50,-14,61,-22,32