server

go.mod

@@ -8,11 +8,15 @@ require (
 	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
 	github.com/bvinc/go-sqlite-lite v0.6.1
 	github.com/develar/errors v0.9.0
+	github.com/didip/tollbooth v4.0.2+incompatible
 	github.com/json-iterator/go v1.1.7
 	github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pkg/errors v0.8.1 // indirect
+	github.com/rs/cors v1.7.0
 	github.com/tdewolff/minify/v2 v2.5.2
 	go.uber.org/atomic v1.4.0 // indirect
 	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.10.0
+	golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 // indirect
 )

go.sum

@@ -10,6 +10,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/develar/errors v0.9.0/go.mod h1:zNbO3fZHcBjapJKbvUnvyaNrKGKkxgaL6C8Z7uNzQMc=
+github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7zjmGO20HCZtF4M=
+github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -22,8 +24,11 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OH
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
+github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -40,6 +45,8 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
+golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

main.go

@@ -1,21 +1,13 @@
 package main
 
 import (
-	"bufio"
-	"bytes"
-	"context"
 	"fmt"
 	"github.com/alecthomas/kingpin"
-	"github.com/develar/errors"
 	"go.uber.org/zap"
 	"log"
 	"os"
-	"os/signal"
-	"path/filepath"
-	"report-aggregator/pkg/analyzer"
-	"report-aggregator/pkg/util"
-	"strings"
-	"time"
+	"report-aggregator/pkg/ideaLog"
+	"report-aggregator/pkg/server"
 )
 
 func createLogger() *zap.Logger {
@@ -37,133 +29,11 @@ func main() {
 
 	var app = kingpin.New("report-aggregator", "report-aggregator").Version("0.0.1")
 
-	configureCollectFromDirCommand(app, logger)
+	ideaLog.ConfigureCollectFromDirCommand(app, logger)
+	server.ConfigureServeCommand(app, logger)
 
 	_, err := app.Parse(os.Args[1:])
 	if err != nil {
 		log.Fatal(fmt.Sprintf("%+v", err))
 	}
 }
-
-func configureCollectFromDirCommand(app *kingpin.Application, log *zap.Logger) {
-	command := app.Command("collect", "Collect reports from idea.log files")
-	dir := command.Flag("dir", "The input directory").Short('i').Required().String()
-	dbPath := command.Flag("db", "The output SQLite database file.").Short('o').Required().String()
-	command.Action(func(context *kingpin.ParseContext) error {
-		err := collectFromDir(*dir, *dbPath, log)
-		if err != nil {
-			return err
-		}
-
-		return nil
-	})
-}
-
-func collectFromDir(dir string, dbPath string, logger *zap.Logger) error {
-	taskContext, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	signals := make(chan os.Signal, 1)
-	signal.Notify(signals, os.Interrupt)
-	go func() {
-		<-signals
-		cancel()
-	}()
-
-	files, err := filepath.Glob(dir + "/idea*.log*")
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	reportAnalyzer, err := analyzer.CreateReportAnalyzer(dbPath, taskContext, logger)
-	if err != nil {
-		return err
-	}
-
-	go func() {
-		err = <-reportAnalyzer.ErrorChannel
-		cancel()
-
-		if err != nil {
-			logger.Error("cannot analyze", zap.Error(err))
-		}
-	}()
-
-	defer util.Close(reportAnalyzer, logger)
-
-	for _, file := range files {
-		if taskContext.Err() != nil {
-			return nil
-		}
-
-		err := collectFromLogFile(file, logger, reportAnalyzer, taskContext)
-		if err != nil {
-			return err
-		}
-	}
-
-	select {
-	case analyzeError := <-reportAnalyzer.ErrorChannel:
-		cancel()
-		return analyzeError
-
-	case <-reportAnalyzer.Done():
-		cancel()
-		return nil
-
-	case <-taskContext.Done():
-		return nil
-	}
-}
-
-func collectFromLogFile(filePath string, log *zap.Logger, reportAnalyzer *analyzer.ReportAnalyzer, taskContext context.Context) error {
-	file, err := os.Open(filePath)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	defer util.Close(file, log)
-
-	scanner := bufio.NewScanner(file)
-	var jsonData bytes.Buffer
-	state := 0
-
-	startSuffix := []byte("=== Start: StartUp Measurement ===")
-	endSuffix := []byte("=== Stop: StartUp Measurement ===")
-
-	lastGeneratedTime := int64(-1)
-	for scanner.Scan() {
-		line := scanner.Bytes()
-		if state == 1 {
-			// idea start-up performance writer bug - end suffix has extra trailing space, so, HasSuffix cannot be used
-			if bytes.HasPrefix(line, endSuffix) {
-				if taskContext.Err() != nil {
-					return nil
-				}
-
-				err = reportAnalyzer.Analyze(jsonData.Bytes(), lastGeneratedTime)
-				if err != nil {
-					return err
-				}
-
-				state = 0
-				lastGeneratedTime = -1
-				jsonData.Reset()
-			} else {
-				jsonData.Write(line)
-			}
-		} else if bytes.HasSuffix(line, startSuffix) {
-			lineString := scanner.Text()
-			// UTC, but it is ok, modern reports contain correct generated time
-			parsedTime, err := time.Parse("2006-01-02 15:04:05", lineString[0:strings.IndexRune(lineString, ',')])
-			if err != nil {
-				return errors.WithStack(err)
-			}
-
-			state = 1
-			lastGeneratedTime = parsedTime.Unix()
-		}
-	}
-
-	return nil
-}

pkg/analyzer/analyzer.go

@@ -7,7 +7,6 @@ import (
 	"github.com/bvinc/go-sqlite-lite/sqlite3"
 	"github.com/develar/errors"
 	"github.com/json-iterator/go"
-	"github.com/mcuadros/go-version"
 	"github.com/tdewolff/minify/v2"
 	"github.com/tdewolff/minify/v2/json"
 	"go.uber.org/zap"
@@ -33,10 +32,12 @@ type ReportAnalyzer struct {
 	insertStatement *sqlite3.Stmt
 	hash            hash.Hash
 
+	machine string
+
 	logger *zap.Logger
 }
 
-func CreateReportAnalyzer(dbPath string, analyzeContext context.Context, logger *zap.Logger) (*ReportAnalyzer, error) {
+func CreateReportAnalyzer(dbPath string, machine string, analyzeContext context.Context, logger *zap.Logger) (*ReportAnalyzer, error) {
 	err := prepareDatabaseFile(dbPath, logger)
 	if err != nil {
 		return nil, errors.WithStack(err)
@@ -61,9 +62,11 @@ func CreateReportAnalyzer(dbPath string, analyzeContext context.Context, logger
 		hash:     sha1.New(),
 
 		logger: logger,
+
+		machine: machine,
 	}
 
-	analyzer.insertStatement, err = db.Prepare(`INSERT INTO report (id, generated_time, metrics_version, metrics, raw_report) VALUES (?, ?, ?, ?, ?)`)
+	analyzer.insertStatement, err = db.Prepare(`INSERT INTO report (id, machine, generated_time, metrics_version, metrics, raw_report) VALUES (?, ?, ?, ?, ?, ?)`)
 	if err != nil {
 		util.Close(db, logger)
 		return nil, err
@@ -222,7 +225,7 @@ func (t *ReportAnalyzer) doAnalyze(report *model.Report) error {
 		return errors.WithStack(err)
 	}
 
-	err = t.insertStatement.Exec(id, report.GeneratedTime, metricsVersion, serializedMetrics, report.RawData)
+	err = t.insertStatement.Exec(id, t.machine, report.GeneratedTime, metricsVersion, serializedMetrics, report.RawData)
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -231,88 +234,6 @@ func (t *ReportAnalyzer) doAnalyze(report *model.Report) error {
 	return nil
 }
 
-func (t *ReportAnalyzer) computeMetrics(report *model.Report, logger *zap.Logger) *model.Metrics {
-	metrics := &model.Metrics{
-		Bootstrap: -1,
-		Splash:    -1,
-
-		AppInitPreparation:       -1,
-		AppInit:                  -1,
-		PluginDescriptorsLoading: -1,
-
-		AppComponentCreation:     -1,
-		ProjectComponentCreation: -1,
-		ModuleLoading:            -1,
-	}
-
-	if version.Compare(report.Version, "12", ">=") && len(report.TraceEvents) == 0 {
-		logger.Warn("invalid report (due to opening second project?), report will be skipped")
-		return nil
-	}
-
-	// v < 12: PluginDescriptorsLoading can be or in MainActivities, or in PrepareAppInitActivities
-
-	for _, activity := range report.MainActivities {
-		switch activity.Name {
-		case "bootstrap":
-			metrics.Bootstrap = activity.Duration
-
-		case "app initialization preparation":
-			metrics.AppInitPreparation = activity.Duration
-		case "app initialization":
-			metrics.AppInit = activity.Duration
-		case "plugin descriptors loading":
-			metrics.PluginDescriptorsLoading = activity.Duration
-
-		case "app component creation":
-			metrics.AppComponentCreation = activity.Duration
-		case "project component creation":
-			metrics.ProjectComponentCreation = activity.Duration
-		case "module loading":
-			metrics.ModuleLoading = activity.Duration
-		}
-	}
-
-	if version.Compare(report.Version, "11", "<") {
-		for _, activity := range report.PrepareAppInitActivities {
-			switch activity.Name {
-			case "plugin descriptors loading":
-				metrics.PluginDescriptorsLoading = activity.Start
-			case "splash initialization":
-				metrics.Splash = activity.Start
-			}
-		}
-	} else {
-		for _, activity := range report.TraceEvents {
-			if activity.Phase == "i" && (activity.Name == "splash" || activity.Name == "splash shown") {
-				metrics.Splash = activity.Timestamp / 1000
-			}
-		}
-	}
-
-	if metrics.Bootstrap == -1 {
-		logRequiredMetricNotFound(logger, "bootstrap")
-		return nil
-	}
-	if metrics.PluginDescriptorsLoading == -1 {
-		logRequiredMetricNotFound(logger, "pluginDescriptorsLoading")
-		return nil
-	}
-	if metrics.AppComponentCreation == -1 {
-		logRequiredMetricNotFound(logger, "AppComponentCreation")
-		return nil
-	}
-	if metrics.ModuleLoading == -1 {
-		logRequiredMetricNotFound(logger, "ModuleLoading")
-		return nil
-	}
-	return metrics
-}
-
-func logRequiredMetricNotFound(logger *zap.Logger, metricName string) {
-	logger.Error("metric is required, but not found, report will be skipped", zap.String("metric", metricName))
-}
-
 func (t *ReportAnalyzer) isReportAlreadyProcessed(id string) (bool, error) {
 	stmt, err := t.db.Prepare(`SELECT metrics_version FROM report WHERE id = ?`, id)
 	if err != nil {