diff --git a/internal/config/config.go b/internal/config/config.go index 5da7f7e..f55643d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -40,7 +40,7 @@ type AdvancedOptions struct { } type ModuleOptions struct { - MBbloomVersion int `mapstructure:"mbbloom_version" default:"10000"` // v1.0.0 + TargetMBbloomVersion int `mapstructure:"target_mbbloom_version" default:"0"` // v1.0.0 <=> 10000 } func (opt *AdvancedOptions) GetPSyncCommand(address string) string { diff --git a/internal/rdb/types/mbbloom.go b/internal/rdb/types/mbbloom.go index d73879c..5704c2a 100644 --- a/internal/rdb/types/mbbloom.go +++ b/internal/rdb/types/mbbloom.go @@ -1,13 +1,14 @@ package types import ( + "RedisShake/internal/config" "RedisShake/internal/rdb/structure" "io" "strconv" "unsafe" ) -// BloomObject for MBbloom-- +// BloomObject for MBbloom-- at https://github.com/RedisBloom/RedisBloom type BloomObject struct { encver int key string @@ -56,23 +57,6 @@ type dumpedChainLink struct { n2 uint8 } -type dumpedChainHeaderV3 struct { - size uint64 - nfilters uint32 - options uint32 -} - -type dumpedChainLinkV3 struct { - bytes uint64 - bits uint64 - size uint64 - err float64 - bpe float64 - hashes uint32 - entries uint32 - n2 uint8 -} - const ( BF_MIN_OPTIONS_ENC = 2 BF_MIN_GROWTH_ENC = 4 @@ -143,10 +127,16 @@ func readDouble(rd io.Reader) float64 { func (o *BloomObject) Rewrite() []RedisCmd { var cs []RedisCmd var h string - if o.encver < BF_MIN_GROWTH_ENC { - h = getEncodedHeaderV3(&o.sb) + if ver := config.Opt.Module.TargetMBbloomVersion; ver > 20200 { + h = getEncodedHeader(&o.sb, true, true) + } else if ver == 20200 { + h = getEncodedHeader(&o.sb, true, false) + } else if ver >= 10000 { + h = getEncodedHeader(&o.sb, false, false) + } else if o.encver < BF_MIN_GROWTH_ENC { + h = getEncodedHeader(&o.sb, false, false) } else { - h = getEncodedHeader(&o.sb) + h = getEncodedHeader(&o.sb, true, true) } cmd := RedisCmd{"BF.LOADCHUNK", o.key, "1", h} cs = append(cs, cmd) @@ -162,36 +152,25 @@ func (o *BloomObject) Rewrite() []RedisCmd { return cs } -func getEncodedHeader(sb *chain) string { - h := make([]byte, DUMPED_CHAIN_LINK_SIZE*sb.nfilters+DUMPED_CHAIN_HEADER_SIZE) +func getEncodedHeader(sb *chain, withGrowth, bigEntries bool) string { + var hs uint64 = DUMPED_CHAIN_HEADER_SIZE_V3 + if withGrowth { + hs = DUMPED_CHAIN_HEADER_SIZE + } + var ls uint64 = DUMPED_CHAIN_LINK_SIZE_V3 + if bigEntries { + ls = DUMPED_CHAIN_LINK_SIZE + } + h := make([]byte, hs+ls*sb.nfilters) ph := (*dumpedChainHeader)(unsafe.Pointer(&h[0])) ph.size = sb.size ph.nfilters = uint32(sb.nfilters) ph.options = uint32(sb.options) - ph.growth = uint32(sb.growth) - for i := uint64(0); i < sb.nfilters; i++ { - pl := (*dumpedChainLink)(unsafe.Add(unsafe.Pointer(&h[0]), DUMPED_CHAIN_HEADER_SIZE+DUMPED_CHAIN_LINK_SIZE*i)) - sl := sb.filters[i] - pl.bytes = uint64(len(sl.inner.bf)) - pl.bits = sl.inner.bits - pl.size = sl.size - pl.err = sl.inner.err - pl.hashes = uint32(sl.inner.hashes) - pl.bpe = sl.inner.bpe - *(*uint64)(unsafe.Pointer(&pl.entries)) = sl.inner.entries - pl.n2 = uint8(sl.inner.n2) + if withGrowth { + ph.growth = uint32(sb.growth) } - return *(*string)(unsafe.Pointer(&h)) -} - -func getEncodedHeaderV3(sb *chain) string { - h := make([]byte, DUMPED_CHAIN_LINK_SIZE_V3*sb.nfilters+DUMPED_CHAIN_HEADER_SIZE_V3) - ph := (*dumpedChainHeaderV3)(unsafe.Pointer(&h[0])) - ph.size = sb.size - ph.nfilters = uint32(sb.nfilters) - ph.options = uint32(sb.options) for i := uint64(0); i < sb.nfilters; i++ { - pl := (*dumpedChainLinkV3)(unsafe.Add(unsafe.Pointer(&h[0]), DUMPED_CHAIN_HEADER_SIZE_V3+DUMPED_CHAIN_LINK_SIZE_V3*i)) + pl := (*dumpedChainLink)(unsafe.Add(unsafe.Pointer(&h[0]), hs+ls*i)) sl := sb.filters[i] pl.bytes = uint64(len(sl.inner.bf)) pl.bits = sl.inner.bits @@ -199,8 +178,13 @@ func getEncodedHeaderV3(sb *chain) string { pl.err = sl.inner.err pl.hashes = uint32(sl.inner.hashes) pl.bpe = sl.inner.bpe - pl.entries = uint32(sl.inner.entries) - pl.n2 = uint8(sl.inner.n2) + if bigEntries { + *(*uint64)(unsafe.Pointer(&pl.entries)) = sl.inner.entries + pl.n2 = uint8(sl.inner.n2) + } else { + pl.entries = uint32(sl.inner.entries) + *(*uint8)(unsafe.Pointer(&pl.enthigh)) = uint8(sl.inner.n2) + } } return *(*string)(unsafe.Pointer(&h)) } diff --git a/shake.toml b/shake.toml index 64f40c0..233596d 100644 --- a/shake.toml +++ b/shake.toml @@ -63,3 +63,7 @@ target_redis_proto_max_bulk_len = 512_000_000 # If the source is Elasticache or MemoryDB, you can set this item. aws_psync = "" # example: aws_psync = "10.0.0.1:6379@nmfu2sl5osync,10.0.0.1:6379@xhma21xfkssync" + +[module] +# The data format for BF.LOADCHUNK is not compatible in different versions. v2.6.3 <=> 20603 +target_mbbloom_version = 20603