Decouple reporting/tagging and publishing

[2opremio]

Following up on #881 (comment):

we should have each one [tagger/publisher] on its own goroutine, trigger a report on the groutine, and then set a timeout for receiving results from them.

This will allow us to be more deadlock-resilient and do some analysis/detection of reporters/taggers taking longer than expected.

[2opremio]

Relevant code:

scope/probe/probe.go

Lines 131 to 164 in c16f0d7

	func (p *Probe) report() report.Report {
	reports := make(chan report.Report, len(p.reporters))
	for _, rep := range p.reporters {
	go func(rep Reporter) {
	t := time.Now()
	newReport, err := rep.Report()
	metrics.MeasureSince([]string{rep.Name(), "reporter"}, t)
	if err != nil {
	log.Printf("error generating report: %v", err)
	newReport = report.MakeReport() // empty is OK to merge
	}
	reports <- newReport
	}(rep)
	}
	result := report.MakeReport()
	for i := 0; i < cap(reports); i++ {
	result = result.Merge(<-reports)
	}
	return result
	}
	func (p *Probe) tag(r report.Report) report.Report {
	var err error
	for _, tagger := range p.taggers {
	t := time.Now()
	r, err = tagger.Tag(r)
	metrics.MeasureSince([]string{tagger.Name(), "tagger"}, t)
	if err != nil {
	log.Printf("error applying tagger: %v", err)
	}
	}
	return r
	}