Config target MBbloom module version

internal/config/config.go

@@ -40,7 +40,7 @@ type AdvancedOptions struct {
 }
 
 type ModuleOptions struct {
-	MBbloomVersion int `mapstructure:"mbbloom_version" default:"10000"` // v1.0.0
+	TargetMBbloomVersion int `mapstructure:"target_mbbloom_version" default:"0"` // v1.0.0 <=> 10000
 }
 
 func (opt *AdvancedOptions) GetPSyncCommand(address string) string {

internal/rdb/types/mbbloom.go

@@ -1,13 +1,14 @@
 package types
 
 import (
+	"RedisShake/internal/config"
 	"RedisShake/internal/rdb/structure"
 	"io"
 	"strconv"
 	"unsafe"
 )
 
-// BloomObject for MBbloom--
+// BloomObject for MBbloom-- at https://github.com/RedisBloom/RedisBloom
 type BloomObject struct {
 	encver int
 	key    string
@@ -56,23 +57,6 @@ type dumpedChainLink struct {
 	n2      uint8
 }
 
-type dumpedChainHeaderV3 struct {
-	size     uint64
-	nfilters uint32
-	options  uint32
-}
-
-type dumpedChainLinkV3 struct {
-	bytes   uint64
-	bits    uint64
-	size    uint64
-	err     float64
-	bpe     float64
-	hashes  uint32
-	entries uint32
-	n2      uint8
-}
-
 const (
 	BF_MIN_OPTIONS_ENC = 2
 	BF_MIN_GROWTH_ENC  = 4
@@ -143,10 +127,16 @@ func readDouble(rd io.Reader) float64 {
 func (o *BloomObject) Rewrite() []RedisCmd {
 	var cs []RedisCmd
 	var h string
-	if o.encver < BF_MIN_GROWTH_ENC {
-		h = getEncodedHeaderV3(&o.sb)
+	if ver := config.Opt.Module.TargetMBbloomVersion; ver > 20200 {
+		h = getEncodedHeader(&o.sb, true, true)
+	} else if ver == 20200 {
+		h = getEncodedHeader(&o.sb, true, false)
+	} else if ver >= 10000 {
+		h = getEncodedHeader(&o.sb, false, false)
+	} else if o.encver < BF_MIN_GROWTH_ENC {
+		h = getEncodedHeader(&o.sb, false, false)
 	} else {
-		h = getEncodedHeader(&o.sb)
+		h = getEncodedHeader(&o.sb, true, true)
 	}
 	cmd := RedisCmd{"BF.LOADCHUNK", o.key, "1", h}
 	cs = append(cs, cmd)
@@ -162,15 +152,25 @@ func (o *BloomObject) Rewrite() []RedisCmd {
 	return cs
 }
 
-func getEncodedHeader(sb *chain) string {
-	h := make([]byte, DUMPED_CHAIN_LINK_SIZE*sb.nfilters+DUMPED_CHAIN_HEADER_SIZE)
+func getEncodedHeader(sb *chain, withGrowth, bigEntries bool) string {
+	var hs uint64 = DUMPED_CHAIN_HEADER_SIZE_V3
+	if withGrowth {
+		hs = DUMPED_CHAIN_HEADER_SIZE
+	}
+	var ls uint64 = DUMPED_CHAIN_LINK_SIZE_V3
+	if bigEntries {
+		ls = DUMPED_CHAIN_LINK_SIZE
+	}
+	h := make([]byte, hs+ls*sb.nfilters)
 	ph := (*dumpedChainHeader)(unsafe.Pointer(&h[0]))
 	ph.size = sb.size
 	ph.nfilters = uint32(sb.nfilters)
 	ph.options = uint32(sb.options)
+	if withGrowth {
 		ph.growth = uint32(sb.growth)
+	}
 	for i := uint64(0); i < sb.nfilters; i++ {
-		pl := (*dumpedChainLink)(unsafe.Add(unsafe.Pointer(&h[0]), DUMPED_CHAIN_HEADER_SIZE+DUMPED_CHAIN_LINK_SIZE*i))
+		pl := (*dumpedChainLink)(unsafe.Add(unsafe.Pointer(&h[0]), hs+ls*i))
 		sl := sb.filters[i]
 		pl.bytes = uint64(len(sl.inner.bf))
 		pl.bits = sl.inner.bits
@@ -178,29 +178,13 @@ func getEncodedHeader(sb *chain) string {
 		pl.err = sl.inner.err
 		pl.hashes = uint32(sl.inner.hashes)
 		pl.bpe = sl.inner.bpe
+		if bigEntries {
 			*(*uint64)(unsafe.Pointer(&pl.entries)) = sl.inner.entries
 			pl.n2 = uint8(sl.inner.n2)
-	}
-	return *(*string)(unsafe.Pointer(&h))
-}
-
-func getEncodedHeaderV3(sb *chain) string {
-	h := make([]byte, DUMPED_CHAIN_LINK_SIZE_V3*sb.nfilters+DUMPED_CHAIN_HEADER_SIZE_V3)
-	ph := (*dumpedChainHeaderV3)(unsafe.Pointer(&h[0]))
-	ph.size = sb.size
-	ph.nfilters = uint32(sb.nfilters)
-	ph.options = uint32(sb.options)
-	for i := uint64(0); i < sb.nfilters; i++ {
-		pl := (*dumpedChainLinkV3)(unsafe.Add(unsafe.Pointer(&h[0]), DUMPED_CHAIN_HEADER_SIZE_V3+DUMPED_CHAIN_LINK_SIZE_V3*i))
-		sl := sb.filters[i]
-		pl.bytes = uint64(len(sl.inner.bf))
-		pl.bits = sl.inner.bits
-		pl.size = sl.size
-		pl.err = sl.inner.err
-		pl.hashes = uint32(sl.inner.hashes)
-		pl.bpe = sl.inner.bpe
+		} else {
 			pl.entries = uint32(sl.inner.entries)
-		pl.n2 = uint8(sl.inner.n2)
+			*(*uint8)(unsafe.Pointer(&pl.enthigh)) = uint8(sl.inner.n2)
+		}
 	}
 	return *(*string)(unsafe.Pointer(&h))
 }

shake.toml

@@ -63,3 +63,7 @@ target_redis_proto_max_bulk_len = 512_000_000
 
 # If the source is Elasticache or MemoryDB, you can set this item.
 aws_psync = "" # example: aws_psync = "10.0.0.1:6379@nmfu2sl5osync,10.0.0.1:6379@xhma21xfkssync"
+
+[module]
+# The data format for BF.LOADCHUNK is not compatible in different versions. v2.6.3 <=> 20603
+target_mbbloom_version = 20603