Merge remote-tracking branch 'grafana/master' into datalink-on-field

* grafana/master:
  QueryEditor: check if optional func toggleEditorMode is provided (grafana#18705)
  Emails: remove the yarn.lock (grafana#18724)
  OAuth: Support JMES path lookup when retrieving user email (grafana#14683)
  Emails: resurrect template notification (grafana#18686)
  Email: add reply-to and direct attachment (grafana#18715)

.gitignore

@@ -10,6 +10,11 @@ awsconfig
 /public/views/index.html
 /public/views/error.html
 /emails/dist
+
+# Enterprise emails
+/emails/templates/enterprise_*
+/public/emails/enterprise_*
+
 /public_gen
 /public/vendor/npm
 /tmp

conf/defaults.ini

@@ -366,6 +366,7 @@ client_id = some_id
 client_secret = some_secret
 scopes = user:email
 email_attribute_name = email:primary
+email_attribute_path =
 auth_url =
 token_url =
 api_url =

conf/sample.ini

@@ -319,6 +319,8 @@
 ;client_id = some_id
 ;client_secret = some_secret
 ;scopes = user:email,read:org
+;email_attribute_name = email:primary
+;email_attribute_path =
 ;auth_url = https://foo.bar/login/oauth/authorize
 ;token_url = https://foo.bar/login/oauth/access_token
 ;api_url = https://foo.bar/user

docs/sources/auth/generic-oauth.md

@@ -40,9 +40,11 @@ Set `api_url` to the resource that returns [OpenID UserInfo](https://connect2id.
 Grafana will attempt to determine the user's e-mail address by querying the OAuth provider as described below in the following order until an e-mail address is found:
 
 1. Check for the presence of an e-mail address via the `email` field encoded in the OAuth `id_token` parameter.
-2. Check for the presence of an e-mail address in the `attributes` map encoded in the OAuth `id_token` parameter. By default Grafana will perform a lookup into the attributes map using the `email:primary` key, however, this is configurable and can be adjusted by using the `email_attribute_name` configuration option.
-3. Query the `/emails` endpoint of the OAuth provider's API (configured with `api_url`) and check for the presence of an e-mail address marked as a primary address.
-4. If no e-mail address is found in steps (1-3), then the e-mail address of the user is set to the empty string.
+2. Check for the presence of an e-mail address using the [JMES path](http://jmespath.org/examples.html) specified via the `email_attribute_path` configuration option. The JSON used for the path lookup is the HTTP response obtained from querying the UserInfo endpoint specified via the `api_url` configuration option.
+**Note**: Only available in Grafana v6.4+.
+3. Check for the presence of an e-mail address in the `attributes` map encoded in the OAuth `id_token` parameter. By default Grafana will perform a lookup into the attributes map using the `email:primary` key, however, this is configurable and can be adjusted by using the `email_attribute_name` configuration option.
+4. Query the `/emails` endpoint of the OAuth provider's API (configured with `api_url`) and check for the presence of an e-mail address marked as a primary address.
+5. If no e-mail address is found in steps (1-4), then the e-mail address of the user is set to the empty string.
 
 ## Set up OAuth2 with Okta
 

emails/README.md

@@ -1,9 +1,15 @@
+## Prerequisites
 
 - npm install
 - gem install premailer
-- grunt   (default task will build new inlines email templates)
-- grunt watch  (will build on source html or css change)
 
-assembled email templates will be in dist/ and final
-inlined templates will be in ../public/emails/
+## Tasks
+
+- npm run build (default task will build new inlines email templates)
+- npm start (will build on source html or css change)
+
+## Result
+
+Assembled email templates will be in `dist/` and final
+inlined templates will be in `../public/emails/`
 

emails/package.json

@@ -8,10 +8,15 @@
     "email": "delder@riester.com",
     "url": "https://github.com/dnnsldr"
   },
+  "scripts": {
+    "build": "grunt",
+    "start": "grunt watch"
+  },
+
   "devDependencies": {
     "grunt": "^0.4.5",
-    "grunt-premailer": "^0.2.10",
-    "grunt-processhtml": "^0.3.3",
+    "grunt-premailer": "^1.1.10",
+    "grunt-processhtml": "^0.4.2",
     "grunt-uncss": "^0.3.7",
     "load-grunt-config": "^0.14.0",
     "grunt-contrib-watch": "^0.6.1",

pkg/login/social/generic_oauth.go

@@ -10,7 +10,7 @@ import (
 	"regexp"
 
 	"github.com/grafana/grafana/pkg/models"
-
+	"github.com/jmespath/go-jmespath"
 	"golang.org/x/oauth2"
 )
 
@@ -21,6 +21,7 @@ type SocialGenericOAuth struct {
 	apiUrl               string
 	allowSignup          bool
 	emailAttributeName   string
+	emailAttributePath   string
 	teamIds              []int
 }
 
@@ -78,6 +79,37 @@ func (s *SocialGenericOAuth) IsOrganizationMember(client *http.Client) bool {
 	return false
 }
 
+// searchJSONForEmail searches the provided JSON response for an e-mail address
+// using the configured e-mail attribute path associated with the generic OAuth
+// provider.
+// Returns an empty string if an e-mail address is not found.
+func (s *SocialGenericOAuth) searchJSONForEmail(data []byte) string {
+	if s.emailAttributePath == "" {
+		s.log.Error("No e-mail attribute path specified")
+		return ""
+	}
+	if len(data) == 0 {
+		s.log.Error("Empty user info JSON response provided")
+		return ""
+	}
+	var buf interface{}
+	if err := json.Unmarshal(data, &buf); err != nil {
+		s.log.Error("Failed to unmarshal user info JSON response", "err", err.Error())
+		return ""
+	}
+	val, err := jmespath.Search(s.emailAttributePath, buf)
+	if err != nil {
+		s.log.Error("Failed to search user info JSON response with provided path", "emailAttributePath", s.emailAttributePath, "err", err.Error())
+		return ""
+	}
+	strVal, ok := val.(string)
+	if ok {
+		return strVal
+	}
+	s.log.Error("E-mail not found when searching JSON with provided path", "emailAttributePath", s.emailAttributePath)
+	return ""
+}
+
 func (s *SocialGenericOAuth) FetchPrivateEmail(client *http.Client) (string, error) {
 	type Record struct {
 		Email       string `json:"email"`
@@ -181,23 +213,24 @@ type UserInfoJson struct {
 
 func (s *SocialGenericOAuth) UserInfo(client *http.Client, token *oauth2.Token) (*BasicUserInfo, error) {
 	var data UserInfoJson
+	var rawUserInfoResponse HttpGetResponse
 	var err error
 
 	if !s.extractToken(&data, token) {
-		response, err := HttpGet(client, s.apiUrl)
+		rawUserInfoResponse, err = HttpGet(client, s.apiUrl)
 		if err != nil {
 			return nil, fmt.Errorf("Error getting user info: %s", err)
 		}
 
-		err = json.Unmarshal(response.Body, &data)
+		err = json.Unmarshal(rawUserInfoResponse.Body, &data)
 		if err != nil {
 			return nil, fmt.Errorf("Error decoding user info JSON: %s", err)
 		}
 	}
 
 	name := s.extractName(&data)
 
-	email := s.extractEmail(&data)
+	email := s.extractEmail(&data, rawUserInfoResponse.Body)
 	if email == "" {
 		email, err = s.FetchPrivateEmail(client)
 		if err != nil {
@@ -250,8 +283,7 @@ func (s *SocialGenericOAuth) extractToken(data *UserInfoJson, token *oauth2.Toke
 		return false
 	}
 
-	email := s.extractEmail(data)
-	if email == "" {
+	if email := s.extractEmail(data, payload); email == "" {
 		s.log.Debug("No email found in id_token", "json", string(payload), "data", data)
 		return false
 	}
@@ -260,11 +292,18 @@ func (s *SocialGenericOAuth) extractToken(data *UserInfoJson, token *oauth2.Toke
 	return true
 }
 
-func (s *SocialGenericOAuth) extractEmail(data *UserInfoJson) string {
+func (s *SocialGenericOAuth) extractEmail(data *UserInfoJson, userInfoResp []byte) string {
 	if data.Email != "" {
 		return data.Email
 	}
 
+	if s.emailAttributePath != "" {
+		email := s.searchJSONForEmail(userInfoResp)
+		if email != "" {
+			return email
+		}
+	}
+
 	emails, ok := data.Attributes[s.emailAttributeName]
 	if ok && len(emails) != 0 {
 		return emails[0]
@@ -275,6 +314,7 @@ func (s *SocialGenericOAuth) extractEmail(data *UserInfoJson) string {
 		if emailErr == nil {
 			return emailAddr.Address
 		}
+		s.log.Debug("Failed to parse e-mail address", "err", emailErr.Error())
 	}
 
 	return ""

pkg/login/social/generic_oauth_test.go

@@ -0,0 +1,86 @@
+package social
+
+import (
+	"github.com/grafana/grafana/pkg/infra/log"
+	. "github.com/smartystreets/goconvey/convey"
+	"testing"
+)
+
+func TestSearchJSONForEmail(t *testing.T) {
+	Convey("Given a generic OAuth provider", t, func() {
+		provider := SocialGenericOAuth{
+			SocialBase: &SocialBase{
+				log: log.New("generic_oauth_test"),
+			},
+		}
+
+		tests := []struct {
+			Name                 string
+			UserInfoJSONResponse []byte
+			EmailAttributePath   string
+			ExpectedResult       string
+		}{
+			{
+				Name:                 "Given an invalid user info JSON response",
+				UserInfoJSONResponse: []byte("{"),
+				EmailAttributePath:   "attributes.email",
+				ExpectedResult:       "",
+			},
+			{
+				Name:                 "Given an empty user info JSON response and empty JMES path",
+				UserInfoJSONResponse: []byte{},
+				EmailAttributePath:   "",
+				ExpectedResult:       "",
+			},
+			{
+				Name:                 "Given an empty user info JSON response and valid JMES path",
+				UserInfoJSONResponse: []byte{},
+				EmailAttributePath:   "attributes.email",
+				ExpectedResult:       "",
+			},
+			{
+				Name: "Given a simple user info JSON response and valid JMES path",
+				UserInfoJSONResponse: []byte(`{
+	"attributes": {
+		"email": "grafana@localhost"
+	}
+}`),
+				EmailAttributePath: "attributes.email",
+				ExpectedResult:     "grafana@localhost",
+			},
+			{
+				Name: "Given a user info JSON response with e-mails array and valid JMES path",
+				UserInfoJSONResponse: []byte(`{
+	"attributes": {
+		"emails": ["grafana@localhost", "admin@localhost"]
+	}
+}`),
+				EmailAttributePath: "attributes.emails[0]",
+				ExpectedResult:     "grafana@localhost",
+			},
+			{
+				Name: "Given a nested user info JSON response and valid JMES path",
+				UserInfoJSONResponse: []byte(`{
+	"identities": [
+		{
+			"userId": "grafana@localhost"
+		},
+		{
+			"userId": "admin@localhost"
+		}
+	]
+}`),
+				EmailAttributePath: "identities[0].userId",
+				ExpectedResult:     "grafana@localhost",
+			},
+		}
+
+		for _, test := range tests {
+			provider.emailAttributePath = test.EmailAttributePath
+			Convey(test.Name, func() {
+				actualResult := provider.searchJSONForEmail(test.UserInfoJSONResponse)
+				So(actualResult, ShouldEqual, test.ExpectedResult)
+			})
+		}
+	})
+}

pkg/login/social/social.go

@@ -73,6 +73,7 @@ func NewOAuthService() {
 			ApiUrl:                       sec.Key("api_url").String(),
 			Enabled:                      sec.Key("enabled").MustBool(),
 			EmailAttributeName:           sec.Key("email_attribute_name").String(),
+			EmailAttributePath:           sec.Key("email_attribute_path").String(),
 			AllowedDomains:               util.SplitString(sec.Key("allowed_domains").String()),
 			HostedDomain:                 sec.Key("hosted_domain").String(),
 			AllowSignup:                  sec.Key("allow_sign_up").MustBool(),
@@ -167,6 +168,7 @@ func NewOAuthService() {
 				apiUrl:               info.ApiUrl,
 				allowSignup:          info.AllowSignup,
 				emailAttributeName:   info.EmailAttributeName,
+				emailAttributePath:   info.EmailAttributePath,
 				teamIds:              sec.Key("team_ids").Ints(","),
 				allowedOrganizations: util.SplitString(sec.Key("allowed_organizations").String()),
 			}

pkg/models/notifications.go

@@ -5,15 +5,25 @@ import "errors"
 var ErrInvalidEmailCode = errors.New("Invalid or expired email code")
 var ErrSmtpNotEnabled = errors.New("SMTP not configured, check your grafana.ini config file's [smtp] section")
 
+// SendEmailAttachFile is a definition of the attached files without path
+type SendEmailAttachFile struct {
+	Name    string
+	Content []byte
+}
+
+// SendEmailCommand is command for sending emails
 type SendEmailCommand struct {
-	To           []string
-	Template     string
-	Subject      string
-	Data         map[string]interface{}
-	Info         string
-	EmbededFiles []string
+	To            []string
+	Template      string
+	Subject       string
+	Data          map[string]interface{}
+	Info          string
+	ReplyTo       []string
+	EmbededFiles  []string
+	AttachedFiles []*SendEmailAttachFile
 }
 
+// SendEmailCommandSync is command for sending emails in sync
 type SendEmailCommandSync struct {
 	SendEmailCommand
 }

pkg/services/alerting/notifiers/email.go

@@ -2,11 +2,11 @@ package notifiers
 
 import (
 	"os"
-	"strings"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/util"
 
 	"github.com/grafana/grafana/pkg/services/alerting"
 	"github.com/grafana/grafana/pkg/setting"
@@ -48,13 +48,7 @@ func NewEmailNotifier(model *models.AlertNotification) (alerting.Notifier, error
 	}
 
 	// split addresses with a few different ways
-	addresses := strings.FieldsFunc(addressesString, func(r rune) bool {
-		switch r {
-		case ',', ';', '\n':
-			return true
-		}
-		return false
-	})
+	addresses := util.SplitEmails(addressesString)
 
 	return &EmailNotifier{
 		NotifierBase: NewNotifierBase(model),

pkg/services/notifications/email.go

@@ -5,13 +5,22 @@ import (
 	"github.com/grafana/grafana/pkg/setting"
 )
 
+// AttachedFile is struct representating email attached files
+type AttachedFile struct {
+	Name    string
+	Content []byte
+}
+
+// Message is representation of the email message
 type Message struct {
-	To           []string
-	From         string
-	Subject      string
-	Body         string
-	Info         string
-	EmbededFiles []string
+	To            []string
+	From          string
+	Subject       string
+	Body          string
+	Info          string
+	ReplyTo       []string
+	EmbededFiles  []string
+	AttachedFiles []*AttachedFile
 }
 
 func setDefaultTemplateData(data map[string]interface{}, u *m.User) {