需求:
有一些连续的id,几十万,要检查其是否存在,且把状态缓存在内存里面
如何存储可以使占用的内存最少,如何快速判断id对应的状态
可用于校验优惠码类似的业务
推荐将排好序的id依次通过Add方法放入bitmap,这样可以达到最优效果(结果所用内存空间最少,存储速度最快)
一、对于>=0的数字,每个数对于一个二进制位,1000是否存在转化为二进制的第1000位是否为1
每64位二进制数可以转化为一个uint64数值,因此这个长的二进制串可以用[]uint64来表示
bitmap.go
package util
import (
"strconv"
)
type Bitmap struct {
//按64*i取模后得到的值(i为数组元素下标)
modValues []uint64
length int
}
func (bitmap *Bitmap) IsExist(num int) bool {
floor, bit := num/64, uint(num%64)
return floor < len(bitmap.modValues) && (bitmap.modValues[floor]&(1<<bit)) != 0
}
//加入一个数,返回值:是否操作成功,true:本身不存在,加入成功;false:本身已存在
func (bitmap *Bitmap) Add(num int) bool {
floor, bit := num/64, uint(num%64)
isExist := false
for floor >= len(bitmap.modValues) {
bitmap.modValues = append(bitmap.modValues, 0)
isExist = true
}
//余数转成对应bit位的值
remainder := uint64(1<<bit)
// 判断num是否已经存在bitmap中
if !isExist && bitmap.modValues[floor] & remainder != 0 {
return false
}
isExist = true
//余数对应的bit位值置为1
bitmap.modValues[floor] |= remainder
bitmap.length++
return isExist
}
//去掉一个数,返回值:是否操作成功,true:本身存在,被删除;false:本身就不存在
func (bitmap *Bitmap) Sub(num int) bool {
floor, bit := num/64, uint(num%64)
for floor >= len(bitmap.modValues) {
return false
}
// 判断num是否已经存在bitmap中,不存在直接返回false
if bitmap.modValues[floor] & (1<<bit) == 0 {
return false
}
//余数对应的bit位值置为0
bitmap.modValues[floor] -= 1 << bit
bitmap.length--
return true
}
//元素个数
func (bitmap *Bitmap) Len() int {
return bitmap.length
}
//输出插入的数据数组json
func (bitmap *Bitmap) String() string {
json,_ := json.MarshalToString(bitmap.Value())
return json
}
//输出二进制字符串,从低位到高位输出
func (bitmap *Bitmap) BitString() string {
s := ""
for _, m := range bitmap.modValues {
item := strconv.FormatUint(m,2)
for i:=len(item)-1; i>=0; i-- {
s += item[i:i+1]
}
}
return s
}
//输出插入的数据,按从小到大顺序排
func (bitmap *Bitmap) Value() []uint64 {
arr := make([]uint64, bitmap.length)
index := 0
for i, v := range bitmap.modValues {
if v == 0 {
continue
}
uinti := uint(i)
for j := uint(0); j < 64; j++ {
if v & (1<<j) != 0 {
arr[index] = uint64(64 * uinti + j)
index++
}
}
}
return arr
}
单元测试(bitmap_test.go):
package util
import (
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func Test_Bitmap(t *testing.T) {
Convey("test Bitmap", t, func() {
bitmap := new(Bitmap)
So(bitmap.Add(0), ShouldBeTrue)
So(len(bitmap.modValues) == 1, ShouldBeTrue)
So(bitmap.String() == "[0]", ShouldBeTrue)
So(bitmap.BitString() == "1", ShouldBeTrue)
So(bitmap.Add(3), ShouldBeTrue)
So(bitmap.length == 2, ShouldBeTrue)
So(bitmap.BitString() == "1001", ShouldBeTrue)
So(len(bitmap.modValues) == 1, ShouldBeTrue)
So(bitmap.Add(63), ShouldBeTrue)
So(len(bitmap.modValues) == 1, ShouldBeTrue)
So(bitmap.length == 3, ShouldBeTrue)
So(bitmap.BitString() == "1001000000000000000000000000000000000000000000000000000000000001", ShouldBeTrue)
So(bitmap.Add(64), ShouldBeTrue)
So(len(bitmap.modValues) == 2, ShouldBeTrue)
So(bitmap.length == 4, ShouldBeTrue)
So(bitmap.BitString() == "10010000000000000000000000000000000000000000000000000000000000011", ShouldBeTrue)
So(bitmap.Add(127), ShouldBeTrue)
So(len(bitmap.modValues) == 2, ShouldBeTrue)
So(bitmap.length == 5, ShouldBeTrue)
So(bitmap.Add(128), ShouldBeTrue)
So(len(bitmap.modValues) == 3, ShouldBeTrue)
So(bitmap.Add(256), ShouldBeTrue)
So(len(bitmap.modValues) == 256/64+1, ShouldBeTrue)
So(bitmap.Add(511), ShouldBeTrue)
So(len(bitmap.modValues) == 512/64, ShouldBeTrue)
So(bitmap.length == 8, ShouldBeTrue)
So(bitmap.Add(513), ShouldBeTrue)
So(len(bitmap.modValues) == 512/64+1, ShouldBeTrue)
So(bitmap.length == 9, ShouldBeTrue)
So(bitmap.Add(63), ShouldBeFalse)
So(len(bitmap.modValues) == 512/64+1, ShouldBeTrue)
So(bitmap.length == 9, ShouldBeTrue)
So(bitmap.Sub(632), ShouldBeFalse)
So(len(bitmap.modValues) == 512/64+1, ShouldBeTrue)
So(bitmap.length == 9, ShouldBeTrue)
So(bitmap.Sub(63), ShouldBeTrue)
So(len(bitmap.modValues) == 512/64+1, ShouldBeTrue)
So(bitmap.length == 8, ShouldBeTrue)
So(bitmap.Sub(513), ShouldBeTrue)
So(len(bitmap.modValues) == 512/64+1, ShouldBeTrue)
So(bitmap.length == 7, ShouldBeTrue)
})
}
二、使用中发现,id并不连续,分布在三个区间[1000000,2000000) 、[10000000,20000000)以及[60000000,...)
因此有了下面的改进版:
根据id第值范围进行分段,如果新插入的id,比当前未扩容状态的最大值还大100 * 64以上,或者比当前未扩容状态的最小值还小100*64以上,就重新生成一个对象来存储;
bitmap_id.go
package util
import (
"fmt"
)
type IdBitmap struct {
//分了组,因为gid不连续,分了3段,所以这里也分了3组
First *BitmapItem
//间隔多大值时需要拆分不同的组,默认100,最小值10
SplitLength int
}
type BitmapItem struct {
//按64*i取模后得到的值(i为数组元素下标)
ModValues []uint64
//数据个数
Length int
//表示前面还有几个元素全是0
BeginIndex int
//Left *BitmapItem
Right *BitmapItem
}
func (bitmap *IdBitmap) IsExist(num int) bool {
if bitmap == nil || bitmap.First == nil {
return false
}
floor, bit := num/64, uint(num%64)
item := bitmap.First
for {
res := item.isExist(floor, bit)
if res {
return res
}
item = item.Right
if item == nil {
break
}
}
return false
}
func (item *BitmapItem) isExist(floor int, bit uint) bool {
if item == nil || len(item.ModValues) == 0 {
return false
}
//floor, bit := num/64, uint(num%64)
return floor < (len(item.ModValues) + item.BeginIndex) && (item.ModValues[floor - item.BeginIndex]&(1<<bit)) != 0
}
//加入一个数,返回值:是否操作成功,true:本身不存在,加入成功;false:本身已存在
func (bitmap *IdBitmap) Add(num int) bool {
floor, bit := num/64, uint(num%64)
isExist := false
if bitmap.First == nil {
bitmap.First = new(BitmapItem)
if bitmap.SplitLength <= 0 {
bitmap.SplitLength = 100
}
}
modeValLen := len(bitmap.First.ModValues)
if modeValLen == 0 {
isExist = bitmap.First.add(floor, bit)
}else if floor >= bitmap.First.BeginIndex {
if floor >= modeValLen + bitmap.First.BeginIndex + bitmap.SplitLength {
item := bitmap.First
itemModValLen := modeValLen
for {
if item == nil {
break
}
if item.Right == nil {
item.Right = new(BitmapItem)
item = item.Right
break
}
if floor >= itemModValLen + item.BeginIndex + bitmap.SplitLength {
item = item.Right
itemModValLen = len(item.ModValues)
continue
}else {
break
}
}
isExist = item.add(floor, bit)
}else{
isExist = bitmap.First.add(floor, bit)
}
}else if floor < bitmap.First.BeginIndex {
//lens := len(bitmap.ModValues)
if floor < bitmap.First.BeginIndex - bitmap.SplitLength {
item := new(BitmapItem)
item.Right = bitmap.First
bitmap.First = item
isExist = item.add(floor, bit)
}else{
isExist = bitmap.First.add(floor, bit)
}
}
return isExist
}
//加入一个数,返回值:是否操作成功,true:本身不存在,加入成功;false:本身已存在
func (item *BitmapItem) add(floor int, bit uint) bool {
isExist := false
modeValLen := len(item.ModValues)
if modeValLen == 0 {
item.ModValues = append(item.ModValues, 0)
if floor > 0 {
item.BeginIndex = floor
}
isExist = true
}else if floor >= modeValLen + item.BeginIndex {
for floor >= len(item.ModValues) + item.BeginIndex {
item.ModValues = append(item.ModValues, 0)
}
isExist = true
}else if floor < item.BeginIndex {
n := item.BeginIndex - floor
newValues := make([]uint64, n)
newValues = append(newValues, item.ModValues...)
item.ModValues = newValues
item.BeginIndex = item.BeginIndex - n
}
//余数转成对应bit位的值
remainder := uint64(1<<bit)
// 判断num是否已经存在bitmap中
if !isExist && item.ModValues[floor - item.BeginIndex] & remainder != 0 {
return false
}
isExist = true
//余数对应的bit位值置为1
item.ModValues[floor - item.BeginIndex] |= remainder
item.Length++
return isExist
}
//去掉一个数,返回值:是否操作成功,true:本身存在,被删除;false:本身就不存在
func (bitmap *IdBitmap) Sub(num int) bool {
floor, bit := num/64, uint(num%64)
item := bitmap.First
if item == nil {
return false
}
for {
res := item.sub(floor, bit)
if res {
return res
}
item = item.Right
if item == nil {
break
}
}
return false
}
//去掉一个数,返回值:是否操作成功,true:本身存在,被删除;false:本身就不存在
func (item *BitmapItem) sub(floor int, bit uint) bool {
if floor >= len(item.ModValues) + item.BeginIndex {
return false
}else if floor < item.BeginIndex {
return false
}
// 判断num是否已经存在bitmap中,不存在直接返回false
if item.ModValues[floor - item.BeginIndex] & (1<<bit) == 0 {
return false
}
//余数对应的bit位值置为0
item.ModValues[floor - item.BeginIndex] -= 1 << bit
item.Length--
return true
}
//元素个数
func (bitmap *IdBitmap) Len() int {
lens := 0
item := bitmap.First
if item == nil {
return 0
}
for {
lens += item.Length
item = item.Right
if item == nil {
break
}
}
return lens
}
//输出插入的数据数组json
func (bitmap *IdBitmap) String() string {
json,_ := json.MarshalToString(bitmap.Value())
return json
}
//输出插入的数据数组json
func (bitmap *IdBitmap) ToJson() string {
json,_ := json.MarshalToString(bitmap)
fmt.Println(json)
return json
}
//输出插入的数据,按从小到大顺序排
func (bitmap *IdBitmap) Value() []uint64 {
arr := []uint64{}
item := bitmap.First
if item == nil {
return arr
}
for {
itemArr := item.value()
if len(itemArr) > 0 {
arr = append(arr, itemArr...)
}
item = item.Right
if item == nil {
break
}
}
return arr
}
//输出插入的数据,按从小到大顺序排
func (item *BitmapItem) value() []uint64 {
arr := make([]uint64, item.Length)
index := 0
for i, v := range item.ModValues {
if v == 0 {
continue
}
uinti := uint(i + item.BeginIndex)
for j := uint(0); j < 64; j++ {
if v & (1<<j) != 0 {
arr[index] = uint64(64 * uinti + j)
index++
}
}
}
return arr
}
单元测试(bitmap_id_test.go)
package util
import (
"fmt"
. "github.com/smartystreets/goconvey/convey"
"testing"
"util"
)
func Test_IdBitmapDesc5(t *testing.T) {
Convey("test IdBitmap", t, func() {
bitmap := new(IdBitmap)
num := 100 * 64
So(bitmap.Add(num), ShouldBeTrue)
So(bitmap.String() == "[6400]", ShouldBeTrue)
So(bitmap.First.Length == 1, ShouldBeTrue)
So(bitmap.First.BeginIndex == 100, ShouldBeTrue)
So(bitmap.Add(1), ShouldBeTrue)
So(bitmap.First.Right == nil, ShouldBeTrue)
So(bitmap.First.Length == 2, ShouldBeTrue)
So(bitmap.First.BeginIndex == 0, ShouldBeTrue)
So(bitmap.String() == "[1,6400]", ShouldBeTrue)
So(bitmap.Add(63), ShouldBeTrue)
So(bitmap.String() == "[1,63,6400]", ShouldBeTrue)
So(bitmap.First.Right == nil, ShouldBeTrue)
So(bitmap.First.Length == 3, ShouldBeTrue)
So(bitmap.First.BeginIndex == 0, ShouldBeTrue)
So(bitmap.Add(64), ShouldBeTrue)
So(bitmap.String() == "[1,63,64,6400]", ShouldBeTrue)
So(bitmap.First.Right == nil, ShouldBeTrue)
So(bitmap.First.Length == 4, ShouldBeTrue)
So(bitmap.First.BeginIndex == 0, ShouldBeTrue)
So(bitmap.Add(63), ShouldBeFalse)
So(bitmap.Sub(69), ShouldBeFalse)
So(bitmap.Sub(63), ShouldBeTrue)
So(bitmap.String() == "[1,64,6400]", ShouldBeTrue)
So(bitmap.First.Right == nil, ShouldBeTrue)
So(bitmap.First.Length == 3, ShouldBeTrue)
So(bitmap.First.BeginIndex == 0, ShouldBeTrue)
jstr := util.ToJson(bitmap)
fmt.Println(jstr)
var res *IdBitmap
json.UnmarshalFromString(jstr, &res)
So(res.String() == "[1,64,6400]", ShouldBeTrue)
So(res.First.Right == nil, ShouldBeTrue)
So(res.First.Length == 3, ShouldBeTrue)
So(res.First.BeginIndex == 0, ShouldBeTrue)
})
}
func Test_IdBitmapDesc4(t *testing.T) {
Convey("test Bitmap", t, func() {
bitmap := new(IdBitmap)
num := 101 * 64 + 1
So(bitmap.Add(num), ShouldBeTrue)
So(bitmap.First.Length == 1, ShouldBeTrue)
So(bitmap.First.BeginIndex == 101, ShouldBeTrue)
So(bitmap.String() == "[6465]", ShouldBeTrue)
So(bitmap.Add(1), ShouldBeTrue)
So(bitmap.First.Length == 1, ShouldBeTrue)
So(bitmap.First.BeginIndex == 0, ShouldBeTrue)
So(bitmap.First.Right.Length == 1, ShouldBeTrue)
So(bitmap.First.Right.BeginIndex == 101, ShouldBeTrue)
So(bitmap.String() == "[1,6465]", ShouldBeTrue)
So(bitmap.Add(63), ShouldBeTrue)
So(bitmap.First.Length == 2, ShouldBeTrue)
So(bitmap.String() == "[1,63,6465]", ShouldBeTrue)
So(bitmap.First.BeginIndex == 0, ShouldBeTrue)
So(bitmap.First.Right.Length == 1, ShouldBeTrue)
So(bitmap.First.Right.BeginIndex == 101, ShouldBeTrue)
So(bitmap.Add(64), ShouldBeTrue)
So(bitmap.First.Length == 3, ShouldBeTrue)
So(bitmap.String() == "[1,63,64,6465]", ShouldBeTrue)
So(bitmap.First.BeginIndex == 0, ShouldBeTrue)
So(bitmap.First.Right.Length == 1, ShouldBeTrue)
So(bitmap.First.Right.BeginIndex == 101, ShouldBeTrue)
fmt.Println(util.ToJson(bitmap))
So(bitmap.Add(63), ShouldBeFalse)
So(bitmap.Sub(69), ShouldBeFalse)
So(bitmap.Sub(63), ShouldBeTrue)
So(bitmap.String() == "[1,64,6465]", ShouldBeTrue)
So(bitmap.First.Length == 2, ShouldBeTrue)
So(bitmap.First.BeginIndex == 0, ShouldBeTrue)
So(bitmap.First.Right.Length == 1, ShouldBeTrue)
So(bitmap.First.Right.BeginIndex == 101, ShouldBeTrue)
jstr := util.ToJson(bitmap)
fmt.Println(jstr)
var res *IdBitmap
json.UnmarshalFromString(jstr, &res)
So(res.String() == "[1,64,6465]", ShouldBeTrue)
So(res.First.Length == 2, ShouldBeTrue)
So(res.First.BeginIndex == 0, ShouldBeTrue)
So(res.First.Right.Length == 1, ShouldBeTrue)
So(res.First.Right.BeginIndex == 101, ShouldBeTrue)
})
}
func Test_IdBitmapDesc3(t *testing.T) {
Convey("test Bitmap", t, func() {
bitmap := new(IdBitmap)
So(bitmap.Add(128), ShouldBeTrue)
So(bitmap.First.Length == 1, ShouldBeTrue)
So(bitmap.String() == "[128]", ShouldBeTrue)
So(len(bitmap.First.ModValues) == 1, ShouldBeTrue)
So(bitmap.First.BeginIndex == 2, ShouldBeTrue)
So(bitmap.Add(64), ShouldBeTrue)
So(bitmap.First.Length == 2, ShouldBeTrue)
So(bitmap.String() == "[64,128]", ShouldBeTrue)
So(len(bitmap.First.ModValues) == 2, ShouldBeTrue)
So(bitmap.First.BeginIndex == 1, ShouldBeTrue)
So(bitmap.Add(63), ShouldBeTrue)
So(bitmap.First.Length == 3, ShouldBeTrue)
So(bitmap.String() == "[63,64,128]", ShouldBeTrue)
So(len(bitmap.First.ModValues) == 3, ShouldBeTrue)
So(bitmap.First.BeginIndex == 0, ShouldBeTrue)
So(bitmap.Add(63), ShouldBeFalse)
So(bitmap.Sub(69), ShouldBeFalse)
So(bitmap.Sub(63), ShouldBeTrue)
})
}
func Test_IdBitmapDesc2(t *testing.T) {
Convey("test Bitmap", t, func() {
bitmap := new(IdBitmap)
So(bitmap.Add(64), ShouldBeTrue)
So(bitmap.First.Length == 1, ShouldBeTrue)
So(bitmap.String() == "[64]", ShouldBeTrue)
So(len(bitmap.First.ModValues) == 1, ShouldBeTrue)
So(bitmap.First.BeginIndex == 1, ShouldBeTrue)
So(bitmap.Add(65), ShouldBeTrue)
So(bitmap.First.Length == 2, ShouldBeTrue)
So(bitmap.String() == "[64,65]", ShouldBeTrue)
So(len(bitmap.First.ModValues) == 1, ShouldBeTrue)
So(bitmap.First.BeginIndex == 1, ShouldBeTrue)
So(bitmap.Add(63), ShouldBeTrue)
So(bitmap.First.Length == 3, ShouldBeTrue)
So(bitmap.String() == "[63,64,65]", ShouldBeTrue)
So(len(bitmap.First.ModValues) == 2, ShouldBeTrue)
So(bitmap.First.BeginIndex == 0, ShouldBeTrue)
So(bitmap.Add(0), ShouldBeTrue)
So(bitmap.First.Length == 4, ShouldBeTrue)
So(bitmap.String() == "[0,63,64,65]", ShouldBeTrue)
So(len(bitmap.First.ModValues) == 2, ShouldBeTrue)
So(bitmap.First.BeginIndex == 0, ShouldBeTrue)
})
}
func Test_IdBitmapDesc1(t *testing.T) {
Convey("test Bitmap", t, func() {
bitmap := new(IdBitmap)
So(bitmap.Add(3), ShouldBeTrue)
So(bitmap.First.Length == 1, ShouldBeTrue)
So(bitmap.String() == "[3]", ShouldBeTrue)
So(len(bitmap.First.ModValues) == 1, ShouldBeTrue)
So(bitmap.Add(0), ShouldBeTrue)
So(bitmap.First.Length == 2, ShouldBeTrue)
So(bitmap.String() == "[0,3]", ShouldBeTrue)
So(len(bitmap.First.ModValues) == 1, ShouldBeTrue)
So(bitmap.Add(63), ShouldBeTrue)
So(len(bitmap.First.ModValues) == 1, ShouldBeTrue)
So(bitmap.First.Length == 3, ShouldBeTrue)
So(bitmap.String() == "[0,3,63]", ShouldBeTrue)
So(bitmap.Add(64), ShouldBeTrue)
So(len(bitmap.First.ModValues) == 2, ShouldBeTrue)
So(bitmap.First.Length == 4, ShouldBeTrue)
So(bitmap.String() == "[0,3,63,64]", ShouldBeTrue)
})
}
func Test_IdBitmapAsc(t *testing.T) {
Convey("test Bitmap", t, func() {
bitmap := new(IdBitmap)
So(bitmap.Add(0), ShouldBeTrue)
So(len(bitmap.First.ModValues) == 1, ShouldBeTrue)
So(bitmap.String() == "[0]", ShouldBeTrue)
So(bitmap.Add(3), ShouldBeTrue)
So(bitmap.First.Length == 2, ShouldBeTrue)
So(len(bitmap.First.ModValues) == 1, ShouldBeTrue)
So(bitmap.Add(63), ShouldBeTrue)
So(len(bitmap.First.ModValues) == 1, ShouldBeTrue)
So(bitmap.First.Length == 3, ShouldBeTrue)
So(bitmap.Add(64), ShouldBeTrue)
So(len(bitmap.First.ModValues) == 2, ShouldBeTrue)
So(bitmap.First.Length == 4, ShouldBeTrue)
So(bitmap.Add(127), ShouldBeTrue)
So(len(bitmap.First.ModValues) == 2, ShouldBeTrue)
So(bitmap.First.Length == 5, ShouldBeTrue)
So(bitmap.Add(128), ShouldBeTrue)
So(len(bitmap.First.ModValues) == 3, ShouldBeTrue)
So(bitmap.Add(256), ShouldBeTrue)
So(len(bitmap.First.ModValues) == 256/64+1, ShouldBeTrue)
So(bitmap.Add(511), ShouldBeTrue)
So(len(bitmap.First.ModValues) == 512/64, ShouldBeTrue)
So(bitmap.First.Length == 8, ShouldBeTrue)
So(bitmap.Add(513), ShouldBeTrue)
So(len(bitmap.First.ModValues) == 512/64+1, ShouldBeTrue)
So(bitmap.First.Length == 9, ShouldBeTrue)
So(bitmap.Add(63), ShouldBeFalse)
So(len(bitmap.First.ModValues) == 512/64+1, ShouldBeTrue)
So(bitmap.First.Length == 9, ShouldBeTrue)
So(bitmap.Sub(632), ShouldBeFalse)
So(len(bitmap.First.ModValues) == 512/64+1, ShouldBeTrue)
So(bitmap.First.Length == 9, ShouldBeTrue)
So(bitmap.Sub(63), ShouldBeTrue)
So(len(bitmap.First.ModValues) == 512/64+1, ShouldBeTrue)
So(bitmap.First.Length == 8, ShouldBeTrue)
So(bitmap.Sub(513), ShouldBeTrue)
So(len(bitmap.First.ModValues) == 512/64+1, ShouldBeTrue)
So(bitmap.First.Length == 7, ShouldBeTrue)
})
}
有疑问加站长微信联系(非本文作者)