背景
json格式不便描述/统一管理
在日常中json数据格式应用场景很多。比如在restful请求/返回、业务通信协议、消息（nsq/kafka），广泛使用json。但是目前存在如下问题
- json数据协议分散在代码里，没有一个统一的描述方式
- json使用往往使用map结构，然后序列化。造成上下游使用混乱，容易出错
- 如果使用struct方式定义，那么需要手动写代码，很是繁琐
比如以下示例：
json格式如下：
{
"id": 123,
"name": "fangming",
"age": 18,
"occ": {"id": 111, "industry": "professor"},
"occs": [
{"id": 111, "industry": "professor"},
{"id": 222, "industry": "engineer"}
],
}
为了描述他我们不得不定义一个结构，然后序列化
type User struct {
Id int64 `json:"id"`
Name string `json:"name"`
Age int32 `json:"age"`
Occ *Occupation `json:"occ"`
Occs []*Occupation `json:"occs"`
}
type Occupation struct {
Id int64 `json:"id"`
Industry string `json:"industry"`
}
方案
用protobuf协议描述json
考虑到golang里可以直接根据struct序列化成json，我们很自然的想到的是通过生成struct结构来实现。想想常用的描述协议的2个格式(protobuf、thrift)，也可以用来生成struct结构。这里考虑到thrift改动太大，用protobuf描述更好些。
一个描述示例如下：
message User {
required int64 id = 1;
required string name = 2;
required int32 age = 3;
optional Occupation occ = 4;
repeated Occupation occs = 5;
}
message Occupation {
required int64 id = 1;
required string industry = 2;
}
// 代表的json结构如下
{
"id": 123,
"name": "fangming",
"age": 18,
"occ": {"id": 111, "industry": "professor"},
"occs": [
{"id": 111, "industry": "professor"},
{"id": 222, "industry": "engineer"}
],
}
实现方案
直接基于protoc支持的扩展功能，写扩展插件即可。这里开发自己的插件：gogofmqjson
实现细节
后续有时间再更新
git地址
https://github.com/buptbill220/protobuf/tree/master
支持特性
- json编码完整支持required、optional字段语义检测；required字段不填序列化/反序列化报错
- 比用protobuf本身生成更简洁的代码
- 生成支持json的marshal & unmarshal方法
- 支持Set方法（主要是解决成员变量为指针类型，需要手动使用&）
- Set方法支持builder模式
protobuf其他特性
- 支持Set方法（主要是解决成员变量为指针类型，需要手动使用&）
- Set方法支持builder模式
安装使用
json协议代码生成
- go get http://github.com/buptbill220/protobuf/protoc-gen-gogofmqjson
- protoc --gogofmqjson_out=./ xxx.proto
无缝迁移gogo
- go get http://github.com/buptbill220/protobuf/protoc-gen-gogo
- go get http://github.com/buptbill220/protobuf/protoc-gen-gogofast
- go get http://github.com/buptbill220/protobuf/protoc-gen-gofast
生成代码示例
// FmqJsonCode generated by protoc-gen-gogofmqjson. Do NOT EDIT
// Author: fangming
// Email: fangming@bytedance.com
// source: test.proto
// It is generated from these files:
// test.proto
// It has these top-level messages:
//== User
//== Occupation
package test
import (
encoding_json "encoding/json"
fmt "fmt"
)
type User struct {
Id *int64 `json:"id"`
Name *string `json:"name"`
Age *int32 `json:"age"`
Occ *Occupation `json:"occ"`
Occs []*Occupation `json:"occs"`
}
func (m *User) Reset() { *m = User{} }
func (*User) ProtoMessage() {}
func (m *User) GetId() int64 {
if m != nil && m.Id != nil {
return *m.Id
}
return 0
}
func (m *User) SetId(v int64) *User {
if m != nil {
m.Id = &v
}
return m
}
func (m *User) GetName() string {
if m != nil && m.Name != nil {
return *m.Name
}
return ""
}
func (m *User) SetName(v string) *User {
if m != nil {
m.Name = &v
}
return m
}
func (m *User) GetAge() int32 {
if m != nil && m.Age != nil {
return *m.Age
}
return 0
}
func (m *User) SetAge(v int32) *User {
if m != nil {
m.Age = &v
}
return m
}
func (m *User) GetOcc() *Occupation {
if m != nil {
return m.Occ
}
return nil
}
func (m *User) SetOcc(v *Occupation) *User {
if m != nil {
m.Occ = v
}
return m
}
func (m *User) GetOccs() []*Occupation {
if m != nil {
return m.Occs
}
return nil
}
func (m *User) SetOccs(v []*Occupation) *User {
if m != nil {
m.Occs = v
}
return m
}
type Occupation struct {
Id *int64 `json:"id"`
Industry *string `json:"industry"`
}
func (m *Occupation) Reset() { *m = Occupation{} }
func (*Occupation) ProtoMessage() {}
func (m *Occupation) GetId() int64 {
if m != nil && m.Id != nil {
return *m.Id
}
return 0
}
func (m *Occupation) SetId(v int64) *Occupation {
if m != nil {
m.Id = &v
}
return m
}
func (m *Occupation) GetIndustry() string {
if m != nil && m.Industry != nil {
return *m.Industry
}
return ""
}
func (m *Occupation) SetIndustry(v string) *Occupation {
if m != nil {
m.Industry = &v
}
return m
}
func (m *User) Marshal() ([]byte, error) {
if m == nil {
return nil, fmt.Errorf("msg User is nil")
}
if err := m.Validate(); err != nil {
return nil, err
}
return encoding_json.Marshal(m)
}
func (m *User) Unmarshal(data []byte) error {
if m == nil {
return fmt.Errorf("msg User is nil")
}
if err := encoding_json.Unmarshal(data, m); err != nil {
return err
}
if err := m.Validate(); err != nil {
return err
}
return nil
}
func (m *Occupation) Marshal() ([]byte, error) {
if m == nil {
return nil, fmt.Errorf("msg Occupation is nil")
}
if err := m.Validate(); err != nil {
return nil, err
}
return encoding_json.Marshal(m)
}
func (m *Occupation) Unmarshal(data []byte) error {
if m == nil {
return fmt.Errorf("msg Occupation is nil")
}
if err := encoding_json.Unmarshal(data, m); err != nil {
return err
}
if err := m.Validate(); err != nil {
return err
}
return nil
}
func (m *User) Validate() error {
if m == nil {
return fmt.Errorf("msg User is nil")
}
if m.Id == nil {
return fmt.Errorf("required field User.Id is nil")
}
if m.Name == nil {
return fmt.Errorf("required field User.Name is nil")
}
if m.Age == nil {
return fmt.Errorf("required field User.Age is nil")
}
if m.Occ != nil {
if err := m.Occ.Validate(); err != nil {
return err
}
}
for _, p := range m.Occs {
if err := p.Validate(); err != nil {
return err
}
}
return nil
}
func (m *Occupation) Validate() error {
if m == nil {
return fmt.Errorf("msg Occupation is nil")
}
if m.Id == nil {
return fmt.Errorf("required field Occupation.Id is nil")
}
if m.Industry == nil {
return fmt.Errorf("required field Occupation.Industry is nil")
}
return nil
}
测试使用/示例
func testUser() {
user := &User{}
data, err := user.Marshal()
if err != nil {
fmt.Printf("1: user marshal err %s\n", err.Error())
}
fmt.Printf("1: user marshal data %s\n", string(data))
data, err = user.SetId(1).Marshal()
if err != nil {
fmt.Printf("2: user marshal err %s\n", err.Error())
}
fmt.Printf("2: user marshal data %s\n", string(data))
data, err = user.SetAge(28).Marshal()
if err != nil {
fmt.Printf("3: user marshal err %s\n", err.Error())
}
fmt.Printf("3: user marshal data %s\n", string(data))
data, err = user.SetName("fangming").Marshal()
if err != nil {
fmt.Printf("4: user marshal err %s\n", err.Error())
}
fmt.Printf("4: user marshal data %s\n", string(data))
occ := &Occupation{}
occ.SetId(1)
data, err = user.SetOcc(occ).Marshal()
if err != nil {
fmt.Printf("5: user marshal err %s\n", err.Error())
}
fmt.Printf("5: user marshal data %s\n", string(data))
occ.SetIndustry("computer")
data, err = user.SetOcc(occ).Marshal()
if err != nil {
fmt.Printf("6: user marshal err %s\n", err.Error())
}
fmt.Printf("6: user marshal data %s\n", string(data))
occs := []*Occupation{}
occ = &Occupation{}
occ.SetId(2)
occs = append(occs, occ)
data, err = user.SetOccs(occs).Marshal()
if err != nil {
fmt.Printf("7: user marshal err %s\n", err.Error())
}
fmt.Printf("7: user marshal data %s\n", string(data))
occ.SetIndustry("teacher")
occs = append(occs, occ)
data, err = user.SetOccs(occs).Marshal()
if err != nil {
fmt.Printf("8: user marshal err %s\n", err.Error())
}
fmt.Printf("8: user marshal data %s\n", string(data))
}
/* 输出如下
1: user marshal err required field User.Id is nil
1: user marshal data
2: user marshal err required field User.Name is nil
2: user marshal data
3: user marshal err required field User.Name is nil
3: user marshal data
4: user marshal data {"id":1,"name":"fangming","age":28,"occ":null,"occs":null}
5: user marshal err required field Occupation.Industry is nil
5: user marshal data
6: user marshal data {"id":1,"name":"fangming","age":28,"occ":{"id":1,"industry":"computer"},"occs":null}
7: user marshal err required field Occupation.Industry is nil
7: user marshal data
8: user marshal data {"id":1,"name":"fangming","age":28,"occ":{"id":1,"industry":"computer"},"occs":[{"id":2,"industry":"teacher"},{"id":2,"industry":"teacher"}]}
*/ 有疑问加站长微信联系（非本文作者）
