diff --git a/deployments/db.js b/deployments/db.js index 0c8acb22..a3304dfd 100644 --- a/deployments/db.js +++ b/deployments/db.js @@ -98,21 +98,20 @@ db.createCollection( "view", { } ); db.createCollection( "polling_detail", { + capped: true, + max: 100, validator: { $jsonSchema: { bsonType: "object", - required: [ "id","params","session_id","url_path" ], + required: [ "id","session_id","domain","url_path" ], properties: { id: { bsonType: "string", }, - session_id: { - bsonType: "string", - }, domain: { bsonType: "string", }, params: { - bsonType: "string" + bsonType: "object" }, ip: { bsonType: "string" @@ -120,9 +119,6 @@ db.createCollection( "polling_detail", { user_agent: { bsonType: "string" }, - url_path: { - bsonType: "string" - }, response_body: { bsonType: "object" }, @@ -130,7 +126,7 @@ db.createCollection( "polling_detail", { bsonType: "object" }, response_code: { - bsonType: "string" + bsonType: "number" } } } } @@ -142,7 +138,7 @@ db.kv.createIndex({key: 1, label_id: 1,domain:1,project:1},{ unique: true }); db.label.createIndex({"id": 1}, { unique: true } ); db.label.createIndex({format: 1,domain:1,project:1},{ unique: true }); db.polling_detail.createIndex({"id": 1}, { unique: true } ); -db.polling_detail.createIndex({session:1,domain:1}, { unique: true } ); +db.polling_detail.createIndex({session_id:1,domain:1}, { unique: true } ); db.view.createIndex({"id": 1}, { unique: true } ); db.view.createIndex({display:1,domain:1,project:1},{ unique: true }); //db config diff --git a/pkg/common/common.go b/pkg/common/common.go index 191d6272..06f11186 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -31,6 +31,11 @@ const ( QueryParamStatus = "status" QueryParamOffset = "offset" QueryParamLimit = "limit" + //polling data + QueryParamSessionID = "sessionId" + QueryParamIP = "ip" + QueryParamURLPath = "urlPath" + QueryParamUserAgent = "userAgent" ) //http headers diff --git a/pkg/model/db_schema.go b/pkg/model/db_schema.go index 950e0cd4..e786a85f 100644 --- a/pkg/model/db_schema.go +++ b/pkg/model/db_schema.go @@ -55,3 +55,17 @@ type ViewDoc struct { Domain string `json:"domain,omitempty" yaml:"domain,omitempty"` Criteria string `json:"criteria,omitempty" yaml:"criteria,omitempty"` } + +//PollingDetail is db struct, it record operation history +type PollingDetail struct { + ID string `json:"id,omitempty" yaml:"id,omitempty"` + SessionID string `json:"session_id,omitempty" bson:"session_id," yaml:"session_id,omitempty"` + Domain string `json:"domain,omitempty" yaml:"domain,omitempty"` + PollingData map[string]interface{} `json:"params,omitempty" yaml:"params,omitempty"` + IP string `json:"ip,omitempty" yaml:"ip,omitempty"` + UserAgent string `json:"user_agent,omitempty" bson:"user_agent," yaml:"user_agent,omitempty"` + URLPath string `json:"url_path,omitempty" bson:"url_path," yaml:"url_path,omitempty"` + ResponseBody interface{} `json:"response_body,omitempty" bson:"response_body," yaml:"response_body,omitempty"` + ResponseHeader map[string][]string `json:"response_header,omitempty" bson:"response_header," yaml:"response_header,omitempty"` + ResponseCode int `json:"response_code,omitempty" bson:"response_code," yaml:"response_code,omitempty"` +} diff --git a/pkg/model/kv.go b/pkg/model/kv.go index b0020379..71d20cd2 100644 --- a/pkg/model/kv.go +++ b/pkg/model/kv.go @@ -53,20 +53,6 @@ type ViewResponse struct { Data []*ViewDoc `json:"data,omitempty"` } -//PollingDetail record operation history -type PollingDetail struct { - ID string `json:"id,omitempty" yaml:"id,omitempty"` - SessionID string `json:"session_id,omitempty" yaml:"session_id,omitempty"` - Domain string `json:"domain,omitempty" yaml:"domain,omitempty"` - PollingData map[string]interface{} `json:"params,omitempty" yaml:"params,omitempty"` - IP string `json:"ip,omitempty" yaml:"ip,omitempty"` - UserAgent string `json:"user_agent,omitempty" yaml:"user_agent,omitempty"` - URLPath string `json:"url_path,omitempty" yaml:"url_path,omitempty"` - ResponseBody interface{} `json:"response_body,omitempty" yaml:"response_body,omitempty"` - ResponseHeader map[string][]string `json:"response_header,omitempty" yaml:"response_header,omitempty"` - ResponseCode int `json:"response_code,omitempty" yaml:"response_code,omitempty"` -} - //DocResponseSingleKey is response doc type DocResponseSingleKey struct { CreateRevision int64 `json:"create_revision"` @@ -87,6 +73,12 @@ type DocResponseGetKey struct { Total int64 `json:"total"` } +//PollingDataResponse is response doc +type PollingDataResponse struct { + Data []*PollingDetail `json:"data"` + Total int `json:"total"` +} + //DocHealthCheck is response doc type DocHealthCheck struct { Version string `json:"version"` diff --git a/server/handler/track_handler.go b/server/handler/track_handler.go index f85df937..3d6979db 100644 --- a/server/handler/track_handler.go +++ b/server/handler/track_handler.go @@ -22,7 +22,7 @@ import ( "github.com/apache/servicecomb-kie/pkg/iputil" "github.com/apache/servicecomb-kie/pkg/model" "github.com/apache/servicecomb-kie/server/resource/v1" - "github.com/apache/servicecomb-kie/server/service/mongo/record" + "github.com/apache/servicecomb-kie/server/service/mongo/track" "github.com/emicklei/go-restful" "github.com/go-chassis/go-chassis/core/handler" "github.com/go-chassis/go-chassis/core/invocation" @@ -60,14 +60,7 @@ func (h *TrackHandler) Handle(chain *handler.Chain, inv *invocation.Invocation, return } chain.Next(inv, func(ir *invocation.Response) error { - resp, ok := ir.Result.(*restful.Response) - if !ok { - err := cb(ir) - if err != nil { - return err - } - return nil - } + resp, _ := ir.Result.(*restful.Response) revStr := req.QueryParameter(common.QueryParamRev) wait := req.QueryParameter(common.QueryParamWait) data := &model.PollingDetail{} @@ -78,14 +71,16 @@ func (h *TrackHandler) Handle(chain *handler.Chain, inv *invocation.Invocation, data.IP = iputil.ClientIP(req.Request) data.ResponseBody = inv.Ctx.Value(common.RespBodyContextKey) data.ResponseCode = ir.Status - data.ResponseHeader = resp.Header() + if resp != nil { + data.ResponseHeader = resp.Header() + } data.PollingData = map[string]interface{}{ "revStr": revStr, "wait": wait, "project": req.HeaderParameter(v1.PathParameterProject), "labels": req.QueryParameter("label"), } - _, err := record.CreateOrUpdate(inv.Ctx, data) + _, err := track.CreateOrUpdate(inv.Ctx, data) if err != nil { openlogging.Warn("record polling detail failed" + err.Error()) err := cb(ir) diff --git a/server/resource/v1/common.go b/server/resource/v1/common.go index 957308f9..2d48b791 100644 --- a/server/resource/v1/common.go +++ b/server/resource/v1/common.go @@ -40,7 +40,7 @@ import ( //const of server const ( HeaderUserAgent = "User-Agent" - HeaderSessionID = "sessionID" + HeaderSessionID = "X-Session-Id" PathParameterProject = "project" PathParameterKey = "key" AttributeDomainKey = "domain" diff --git a/server/resource/v1/doc_struct.go b/server/resource/v1/doc_struct.go index 5c3e35c5..1a5aebbf 100644 --- a/server/resource/v1/doc_struct.go +++ b/server/resource/v1/doc_struct.go @@ -101,6 +101,31 @@ var ( ParamType: goRestful.QueryParameterKind, Desc: "pagination", } + //polling data + DocQuerySessionIDParameters = &restful.Parameters{ + DataType: "string", + Name: common.QueryParamSessionID, + ParamType: goRestful.QueryParameterKind, + Desc: "sessionId is the Unique identification of the client", + } + DocQueryIPParameters = &restful.Parameters{ + DataType: "string", + Name: common.QueryParamIP, + ParamType: goRestful.QueryParameterKind, + Desc: "client ip", + } + DocQueryURLPathParameters = &restful.Parameters{ + DataType: "string", + Name: common.QueryParamURLPath, + ParamType: goRestful.QueryParameterKind, + Desc: "address of the call", + } + DocQueryUserAgentParameters = &restful.Parameters{ + DataType: "string", + Name: common.QueryParamUserAgent, + ParamType: goRestful.QueryParameterKind, + Desc: "user agent of the call", + } ) //swagger doc path params diff --git a/server/resource/v1/history_resource.go b/server/resource/v1/history_resource.go index 32376c14..9b6269a7 100644 --- a/server/resource/v1/history_resource.go +++ b/server/resource/v1/history_resource.go @@ -19,6 +19,7 @@ package v1 import ( "github.com/apache/servicecomb-kie/pkg/model" + "github.com/apache/servicecomb-kie/server/service/mongo/track" "github.com/go-chassis/go-chassis/pkg/runtime" "net/http" "strconv" @@ -75,6 +76,49 @@ func (r *HistoryResource) GetRevisions(context *restful.Context) { } } +//GetPollingData get the record of the get or list history +func (r *HistoryResource) GetPollingData(context *restful.Context) { + query := &model.PollingDetail{} + sessionID := context.ReadQueryParameter(common.QueryParamSessionID) + if sessionID != "" { + query.SessionID = sessionID + } + ip := context.ReadQueryParameter(common.QueryParamIP) + if ip != "" { + query.IP = ip + } + urlPath := context.ReadQueryParameter(common.QueryParamURLPath) + if urlPath != "" { + query.URLPath = urlPath + } + userAgent := context.ReadQueryParameter(common.QueryParamUserAgent) + if userAgent != "" { + query.UserAgent = userAgent + } + domain := ReadDomain(context) + if domain == nil { + WriteErrResponse(context, http.StatusInternalServerError, common.MsgDomainMustNotBeEmpty, common.ContentTypeText) + return + } + query.Domain = domain.(string) + records, err := track.Get(context.Ctx, query) + if err != nil { + if err == service.ErrRecordNotExists { + WriteErrResponse(context, http.StatusNotFound, err.Error(), common.ContentTypeText) + return + } + WriteErrResponse(context, http.StatusInternalServerError, err.Error(), common.ContentTypeText) + return + } + resp := &model.PollingDataResponse{} + resp.Data = records + resp.Total = len(records) + err = writeResponse(context, resp) + if err != nil { + openlogging.Error(err.Error()) + } +} + //HealthCheck provider version info and time info func (r *HistoryResource) HealthCheck(context *restful.Context) { domain := ReadDomain(context) @@ -128,5 +172,23 @@ func (r *HistoryResource) URLPatterns() []restful.Route { Consumes: []string{goRestful.MIME_JSON, common.ContentTypeYaml}, Produces: []string{goRestful.MIME_JSON, common.ContentTypeYaml}, }, + { + Method: http.MethodGet, + Path: "/v1/{project}/kie/track", + ResourceFunc: r.GetPollingData, + FuncDesc: "get polling tracks of clients of kie server", + Parameters: []*restful.Parameters{ + DocPathProject, DocQuerySessionIDParameters, DocQueryIPParameters, DocQueryURLPathParameters, DocQueryUserAgentParameters, + }, + Returns: []*restful.Returns{ + { + Code: http.StatusOK, + Message: "true", + Model: []model.PollingDataResponse{}, + }, + }, + Consumes: []string{goRestful.MIME_JSON, common.ContentTypeYaml}, + Produces: []string{goRestful.MIME_JSON, common.ContentTypeYaml}, + }, } } diff --git a/server/resource/v1/history_resource_test.go b/server/resource/v1/history_resource_test.go index 2a05a564..e7f9ed84 100644 --- a/server/resource/v1/history_resource_test.go +++ b/server/resource/v1/history_resource_test.go @@ -85,6 +85,45 @@ func TestHistoryResource_GetRevisions(t *testing.T) { } +func TestHistoryResource_GetPollingData(t *testing.T) { + t.Run("list kv by service label, to create a polling data", func(t *testing.T) { + r, _ := http.NewRequest("GET", "/v1/test/kie/kv", nil) + noopH := &handler2.NoopAuthHandler{} + noopH2 := &handler2.TrackHandler{} + chain, _ := handler.CreateChain(common.Provider, "testchain3", noopH.Name(), noopH2.Name()) + r.Header.Set("Content-Type", "application/json") + r.Header.Set("X-Session-Id", "test") + kvr := &v1.KVResource{} + c, err := restfultest.New(kvr, chain) + assert.NoError(t, err) + resp := httptest.NewRecorder() + c.ServeHTTP(resp, r) + body, err := ioutil.ReadAll(resp.Body) + assert.NoError(t, err) + result := &model.KVResponse{} + err = json.Unmarshal(body, result) + assert.NoError(t, err) + }) + t.Run("get polling data", func(t *testing.T) { + r, _ := http.NewRequest("GET", "/v1/test/kie/track?sessionId=test", nil) + noopH := &handler2.NoopAuthHandler{} + chain, _ := handler.CreateChain(common.Provider, "testchain3", noopH.Name()) + r.Header.Set("Content-Type", "application/json") + revision := &v1.HistoryResource{} + c, err := restfultest.New(revision, chain) + assert.NoError(t, err) + resp := httptest.NewRecorder() + c.ServeHTTP(resp, r) + body, err := ioutil.ReadAll(resp.Body) + assert.NoError(t, err) + result := &model.PollingDataResponse{} + err = json.Unmarshal(body, result) + assert.NoError(t, err) + assert.NotEmpty(t, result.Data) + }) + +} + func Test_HeathCheck(t *testing.T) { path := fmt.Sprintf("/v1/health") r, _ := http.NewRequest("GET", path, nil) diff --git a/server/resource/v1/kv_resource_test.go b/server/resource/v1/kv_resource_test.go index 35aa46f4..e0450d7d 100644 --- a/server/resource/v1/kv_resource_test.go +++ b/server/resource/v1/kv_resource_test.go @@ -76,7 +76,6 @@ func TestKVResource_Put(t *testing.T) { noopH := &handler2.NoopAuthHandler{} chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name()) r.Header.Set("Content-Type", "application/json") - r.Header.Set("sessionID", "test") kvr := &v1.KVResource{} c, _ := restfultest.New(kvr, chain) resp := httptest.NewRecorder() @@ -101,7 +100,6 @@ func TestKVResource_Put(t *testing.T) { noopH := &handler2.NoopAuthHandler{} chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name()) r.Header.Set("Content-Type", "application/json") - r.Header.Set("sessionID", "test") kvr := &v1.KVResource{} c, _ := restfultest.New(kvr, chain) resp := httptest.NewRecorder() @@ -127,7 +125,6 @@ func TestKVResource_Put(t *testing.T) { noopH := &handler2.NoopAuthHandler{} chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name()) r.Header.Set("Content-Type", "application/json") - r.Header.Set("sessionID", "test") kvr := &v1.KVResource{} c, _ := restfultest.New(kvr, chain) resp := httptest.NewRecorder() diff --git a/server/service/mongo/record/polling_detail_dao.go b/server/service/mongo/track/polling_detail_dao.go similarity index 57% rename from server/service/mongo/record/polling_detail_dao.go rename to server/service/mongo/track/polling_detail_dao.go index 6cefe5cf..998931d3 100644 --- a/server/service/mongo/record/polling_detail_dao.go +++ b/server/service/mongo/track/polling_detail_dao.go @@ -15,12 +15,14 @@ * limitations under the License. */ -package record +package track import ( "context" "github.com/apache/servicecomb-kie/pkg/model" + "github.com/apache/servicecomb-kie/server/service" "github.com/apache/servicecomb-kie/server/service/mongo/session" + "github.com/go-mesh/openlogging" uuid "github.com/satori/go.uuid" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -42,9 +44,49 @@ func CreateOrUpdate(ctx context.Context, detail *model.PollingDetail) (*model.Po } return nil, res.Err() } - _, err := collection.UpdateOne(ctx, queryFilter, detail) + _, err := collection.UpdateOne(ctx, queryFilter, bson.D{{"$set", detail}}) if err != nil { return nil, err } return detail, nil } + +//Get is to get a +func Get(ctx context.Context, detail *model.PollingDetail) ([]*model.PollingDetail, error) { + collection := session.GetDB().Collection(session.CollectionPollingDetail) + queryFilter := bson.M{"domain": detail.Domain} + if detail.SessionID != "" { + queryFilter["session_id"] = detail.SessionID + } + if detail.IP != "" { + queryFilter["ip"] = detail.IP + } + if detail.UserAgent != "" { + queryFilter["user_agent"] = detail.UserAgent + } + if detail.URLPath != "" { + queryFilter["url_path"] = detail.URLPath + } + cur, err := collection.Find(ctx, queryFilter) + if err != nil { + return nil, err + } + defer cur.Close(ctx) + if cur.Err() != nil { + return nil, err + } + records := make([]*model.PollingDetail, 0) + for cur.Next(ctx) { + curRecord := &model.PollingDetail{} + if err := cur.Decode(curRecord); err != nil { + openlogging.Error("decode to KVs error: " + err.Error()) + return nil, err + } + curRecord.Domain = "" + records = append(records, curRecord) + } + if len(records) == 0 { + return nil, service.ErrRecordNotExists + } + return records, nil +} diff --git a/server/service/service.go b/server/service/service.go index c1451d5c..5720a6ca 100644 --- a/server/service/service.go +++ b/server/service/service.go @@ -35,6 +35,7 @@ var ( //db errors var ( ErrKeyNotExists = errors.New("can not find any key value") + ErrRecordNotExists = errors.New("can not find any polling data") ErrRevisionNotExist = errors.New("revision does not exist") ErrAliasNotGiven = errors.New("label alias not given") )