plugin: introduce plugin framework (#8788)

cmd/pluginpkg/pluginpkg.go

@@ -0,0 +1,159 @@
+// Copyright 2018 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"text/template"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	pkgDir string
+	outDir string
+)
+
+const codeTemplate = `
+package main
+
+import (
+	"github.com/pingcap/tidb/plugin"
+	"github.com/pingcap/tidb/sessionctx/variable"
+)
+
+func PluginManifest() *plugin.Manifest {
+	return plugin.ExportManifest(&plugin.{{.kind}}Manifest{
+		Manifest: plugin.Manifest{
+			Kind:           plugin.{{.kind}},
+			Name:           "{{.name}}",
+			Description:    "{{.description}}",
+			Version:        {{.version}},
+			RequireVersion: map[string]uint16{},
+			License:        "{{.license}}",
+			BuildTime:      "{{.buildTime}}",
+			SysVars: map[string]*variable.SysVar{
+			    {{range .sysVars}}
+				"{{.name}}": {
+					Scope: variable.Scope{{.scope}},
+					Name:  "{{.name}}",
+					Value: "{{.value}}",
+				},
+				{{end}}
+			},
+			Validate:   {{.validate}},
+			OnInit:     {{.onInit}},
+			OnShutdown: {{.onShutdown}},
+		},
+		{{range .export}}
+		{{.extPoint}}: {{.impl}},
+		{{end}}
+	})
+}
+`
+
+func init() {
+	flag.StringVar(&pkgDir, "pkg-dir", "", "plugin package folder path")
+	flag.StringVar(&outDir, "out-dir", "", "plugin packaged folder path")
+	flag.Usage = usage
+}
+
+func usage() {
+	log.Printf("Usage: %s --pkg-dir [plugin source pkg folder] --outDir-dir [outDir-dir]\n", path.Base(os.Args[0]))
+	flag.PrintDefaults()
+	os.Exit(1)
+}
+
+func main() {
+	flag.Parse()
+	if pkgDir == "" || outDir == "" {
+		flag.Usage()
+	}
+	var manifest map[string]interface{}
+	_, err := toml.DecodeFile(filepath.Join(pkgDir, "manifest.toml"), &manifest)
+	if err != nil {
+		log.Printf("read pkg %s's manifest failure, %+v\n", pkgDir, err)
+		os.Exit(1)
+	}
+	manifest["buildTime"] = time.Now().String()
+
+	pluginName := manifest["name"].(string)
+	if strings.Contains(pluginName, "-") {
+		log.Printf("plugin name should not contain '-'\n")
+		os.Exit(1)
+	}
+	if pluginName != filepath.Base(pkgDir) {
+		log.Printf("plugin package must be same with plugin name in manifest file\n")
+		os.Exit(1)
+	}
+
+	version := manifest["version"].(string)
+	tmpl, err := template.New("gen-plugin").Parse(codeTemplate)
+	if err != nil {
+		log.Printf("generate code failure during parse template, %+v\n", err)
+		os.Exit(1)
+	}
+
+	genFileName := filepath.Join(pkgDir, filepath.Base(pkgDir)+".gen.go")
+	genFile, err := os.OpenFile(genFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
+	if err != nil {
+		log.Printf("generate code failure during prepare output file, %+v\n", err)
+		os.Exit(1)
+	}
+	defer func() {
+		err1 := os.Remove(genFileName)
+		if err1 != nil {
+			log.Printf("remove tmp file %s failure, please clean up manually at %v", genFileName, err1)
+		}
+	}()
+
+	err = tmpl.Execute(genFile, manifest)
+	if err != nil {
+		log.Printf("generate code failure during generating code, %+v\n", err)
+		os.Exit(1)
+	}
+
+	outputFile := filepath.Join(outDir, pluginName+"-"+version+".so")
+	pluginPath := `-pluginpath=` + pluginName + "-" + version
+	ctx := context.Background()
+	buildCmd := exec.CommandContext(ctx, "go", "build",
+		"-ldflags", pluginPath,
+		"-buildmode=plugin",
+		"-o", outputFile, pkgDir)
+	buildCmd.Stderr = os.Stderr
+	buildCmd.Stdout = os.Stdout
+	buildCmd.Env = append(os.Environ(), "GO111MODULE=on")
+	err = buildCmd.Run()
+	if err != nil {
+		log.Printf("compile plugin source code failure, %+v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf(`Package "%s" as plugin "%s" success.`+"\nManifest:\n", pkgDir, outputFile)
+	encoder := json.NewEncoder(os.Stdout)
+	encoder.SetIndent(" ", "\t")
+	err = encoder.Encode(manifest)
+	if err != nil {
+		log.Printf("print manifest detail failure, err: %v", err)
+	}
+}

go.mod

@@ -81,3 +81,5 @@ require (
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.2.1 // indirect
 )
+
+replace github.com/pingcap/parser => github.com/lysu/parser v0.0.0-20190325074808-d880cf39390b

go.sum

@@ -66,6 +66,8 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/lysu/parser v0.0.0-20190325074808-d880cf39390b h1:jWTIO8rWK9pg3RO6eu5/5M386uY07wE+lYq0WKVkaVA=
+github.com/lysu/parser v0.0.0-20190325074808-d880cf39390b/go.mod h1:CJk6LPzPxAcwHIcTugQaKxzvTR10NDJ5ln8XR7uYTJk=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb h1:bsjNADsjHq0gjU7KO7zwoX5k3HtFdf6TDzB3ncl5iUs=
@@ -92,8 +94,6 @@ github.com/pingcap/goleveldb v0.0.0-20171020084629-8d44bfdf1030 h1:XJLuW0lsP7vAt
 github.com/pingcap/goleveldb v0.0.0-20171020084629-8d44bfdf1030/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
 github.com/pingcap/kvproto v0.0.0-20190226063853-f6c0b7ffff11 h1:iGNfAHgK0VHJobW4bPTlFmdnt3YWsEHdSTIcjut6ffk=
 github.com/pingcap/kvproto v0.0.0-20190226063853-f6c0b7ffff11/go.mod h1:0gwbe1F2iBIjuQ9AH0DbQhL+Dpr5GofU8fgYyXk+ykk=
-github.com/pingcap/parser v0.0.0-20190305073013-4f60445a0550 h1:zs2q6jzN04S63J0bvBBp+zK5cXHtiWscGDEjhTkmMbE=
-github.com/pingcap/parser v0.0.0-20190305073013-4f60445a0550/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
 github.com/pingcap/pd v2.1.0-rc.4+incompatible h1:/buwGk04aHO5odk/+O8ZOXGs4qkUjYTJ2UpCJXna8NE=
 github.com/pingcap/pd v2.1.0-rc.4+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E=
 github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible h1:e9Gi/LP9181HT3gBfSOeSBA+5JfemuE4aEAhqNgoE4k=

plugin/conn_ip_example/conn_ip_example.go

@@ -0,0 +1,49 @@
+// Copyright 2019 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pingcap/tidb/plugin"
+	"github.com/pingcap/tidb/sessionctx/variable"
+)
+
+// Validate implements TiDB plugin's Validate SPI.
+func Validate(ctx context.Context, m *plugin.Manifest) error {
+	fmt.Println("conn_ip_example validate called")
+	return nil
+}
+
+// OnInit implements TiDB plugin's OnInit SPI.
+func OnInit(ctx context.Context, manifest *plugin.Manifest) error {
+	fmt.Println("conn_ip_example init called")
+	fmt.Println("read cfg in init", manifest.SysVars["conn_ip_example_test_variable"].Value)
+	return nil
+}
+
+// OnShutdown implements TiDB plugin's OnShutdown SPI.
+func OnShutdown(ctx context.Context, manifest *plugin.Manifest) error {
+	fmt.Println("conn_ip_examples hutdown called")
+	return nil
+}
+
+// NotifyEvent implements TiDB Audit plugin's NotifyEvent SPI.
+func NotifyEvent(ctx context.Context) error {
+	fmt.Println("conn_ip_example notifiy called")
+	fmt.Println("variable test: ", variable.GetSysVar("conn_ip_example_test_variable").Value)
+	fmt.Printf("new connection by %s\n", ctx.Value("ip"))
+	return nil
+}

plugin/conn_ip_example/conn_ip_example_test.go

@@ -0,0 +1,61 @@
+// Copyright 2019 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main_test
+
+import (
+	"context"
+
+	"github.com/pingcap/tidb/plugin"
+	"github.com/pingcap/tidb/sessionctx/variable"
+)
+
+func Example_LoadRunShutdownPlugin() {
+	ctx := context.Background()
+	var pluginVarNames []string
+	cfg := plugin.Config{
+		Plugins:        []string{"conn_ip_example-1"},
+		PluginDir:      "/home/robi/Code/go/src/github.com/pingcap/tidb/plugin/conn_ip_example",
+		GlobalSysVar:   &variable.SysVars,
+		PluginVarNames: &pluginVarNames,
+	}
+
+	err := plugin.Init(ctx, cfg)
+	if err != nil {
+		panic(err)
+	}
+
+	ps := plugin.GetByKind(plugin.Audit)
+	for _, auditPlugin := range ps {
+		if auditPlugin.State != plugin.Ready {
+			continue
+		}
+		plugin.DeclareAuditManifest(auditPlugin.Manifest).NotifyEvent(context.Background(), nil)
+	}
+
+	err = plugin.Reload(ctx, cfg, plugin.ID("conn_ip_example-2"))
+	if err != nil {
+		panic(err)
+	}
+
+	for _, auditPlugin := range plugin.GetByKind(plugin.Audit) {
+		if auditPlugin.State != plugin.Ready {
+			continue
+		}
+		plugin.DeclareAuditManifest(auditPlugin.Manifest).NotifyEvent(
+			context.WithValue(context.Background(), "ip", "1.1.1.2"), nil,
+		)
+	}
+
+	plugin.Shutdown(context.Background())
+}

plugin/conn_ip_example/manifest.toml

@@ -0,0 +1,15 @@
+name = "conn_ip_example"
+kind = "Audit"
+description = "just a test"
+version = "1"
+license = ""
+sysVars = [
+    {name="conn_ip_example_test_variable", scope="Global", value="2"},
+    {name="conn_ip_example_test_variable2", scope="Session", value="2"},
+]
+validate = "Validate"
+onInit = "OnInit"
+onShutdown = "OnShutdown"
+export = [
+    {extPoint="NotifyEvent", impl="NotifyEvent"}
+]

plugin/const.go

@@ -0,0 +1,70 @@
+// Copyright 2019 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package plugin
+
+// Kind presents the kind of plugin.
+type Kind uint8
+
+const (
+	// Audit indicates it is a Audit plugin.
+	Audit Kind = 1 + iota
+	// Authentication indicate it is a Authentication plugin.
+	Authentication
+	// Schema indicate a plugin that can change TiDB schema.
+	Schema
+	// Daemon indicate a plugin that can run as daemon task.
+	Daemon
+)
+
+func (k Kind) String() (str string) {
+	switch k {
+	case Audit:
+		str = "Audit"
+	case Authentication:
+		str = "Authentication"
+	case Schema:
+		str = "Schema"
+	case Daemon:
+		str = "Daemon"
+	}
+	return
+}
+
+// State present the state of plugin.
+type State uint8
+
+const (
+	// Uninitialized indicates plugin is uninitialized.
+	Uninitialized State = iota
+	// Ready indicates plugin is ready to work.
+	Ready
+	// Dying indicates plugin will be close soon.
+	Dying
+	// Disable indicate plugin is disabled.
+	Disable
+)
+
+func (s State) String() (str string) {
+	switch s {
+	case Uninitialized:
+		str = "Uninitialized"
+	case Ready:
+		str = "Ready"
+	case Dying:
+		str = "Dying"
+	case Disable:
+		str = "Disable"
+	}
+	return
+}