support more time formats for generated, ignore missing "bootstrap"…

… for report <= v5

Makefile

@@ -2,6 +2,11 @@
 
 # https://github.com/bvinc/go-sqlite-lite/issues/10#issuecomment-498539630
 
+# go get -u github.com/go-bindata/go-bindata/...
+
+assets:
+	go-bindata -o ./pkg/analyzer/sqlScript.go -pkg analyzer -prefix ./pkg/analyzer/sql ./pkg/analyzer/sql
+
 build: lint
 	go mod tidy
 	make build-mac
@@ -13,6 +18,7 @@ build-mac:
 
 build-linux:
 	env GOOS=linux GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-linux-musl-gcc CXX=x86_64-linux-musl-g++ go build -ldflags='-s -w' -o dist/linux/report-aggregator ./
+	XZ_OPT=-9 tar -cJf dist/linux-report-aggregator.tar.xz dist/linux/report-aggregator
 
 build-windows:
 	env GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=/usr/local/bin/x86_64-w64-mingw32-gcc CXX=/usr/local/bin/x86_64-w64-mingw32-g++ go build -ldflags='-s -w' -o dist/windows/report-aggregator.exe ./

go.mod

@@ -11,12 +11,14 @@ require (
 	github.com/didip/tollbooth v4.0.2+incompatible
 	github.com/json-iterator/go v1.1.7
 	github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pkg/errors v0.8.1 // indirect
 	github.com/rs/cors v1.7.0
 	github.com/tdewolff/minify/v2 v2.5.2
 	go.uber.org/atomic v1.4.0 // indirect
-	go.uber.org/multierr v1.1.0 // indirect
+	go.uber.org/multierr v1.2.0 // indirect
 	go.uber.org/zap v1.10.0
 	golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 // indirect
 )

go.sum

@@ -1,3 +1,4 @@
+github.com/alecthomas/kingpin v2.2.6+incompatible h1:5svnBTFgJjZvGKyYBtMB0+m5wvrbUHiqye8wRJMlnYI=
 github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -8,7 +9,9 @@ github.com/bvinc/go-sqlite-lite v0.6.1/go.mod h1:2GiE60NUdb0aNhDdY+LXgrqAVDpi2Ij
 github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/develar/errors v0.9.0 h1:ftXOTwkajtgkUwLTw1iKG+mJwrUTvCp9Zr/Z6Y+rvMY=
 github.com/develar/errors v0.9.0/go.mod h1:zNbO3fZHcBjapJKbvUnvyaNrKGKkxgaL6C8Z7uNzQMc=
 github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7zjmGO20HCZtF4M=
 github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
@@ -22,10 +25,17 @@ github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2 h1:YocNLcTBdEd
 github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2/go.mod h1:76rfSfYPWj01Z85hUf/ituArm797mNKcvINh1OlsZKo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
@@ -41,12 +51,18 @@ github.com/tdewolff/parse/v2 v2.3.9/go.mod h1:HansaqmN4I/U7L6/tUp0NcwT2tFO0F4EAW
 github.com/tdewolff/test v1.0.0/go.mod h1:DiQUlutnqlEvdvhSn2LPGy4TFwRauAaYDsL+683RNX4=
 github.com/tdewolff/test v1.0.4 h1:ih38SXuQJ32Hng5EtSW32xqEsVeMnPp6nNNRPhBBDE8=
 github.com/tdewolff/test v1.0.4/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
+go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.2.0 h1:6I+W7f5VwC5SV9dNrZ3qXrDB9mD0dyGOi/ZJmYw03T4=
+go.uber.org/multierr v1.2.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

pkg/analyzer/analyzer.go

@@ -27,10 +27,10 @@ type ReportAnalyzer struct {
 	waitGroup sync.WaitGroup
 	closeOnce sync.Once
 
-	minifier        *minify.M
-	db              *sqlite3.Conn
-	insertStatement *sqlite3.Stmt
-	hash            hash.Hash
+	minifier     *minify.M
+	db           *sqlite3.Conn
+	putStatement *sqlite3.Stmt
+	hash         hash.Hash
 
 	machine string
 
@@ -66,12 +66,6 @@ func CreateReportAnalyzer(dbPath string, machine string, analyzeContext context.
 		machine: machine,
 	}
 
-	analyzer.insertStatement, err = db.Prepare(`INSERT INTO report (id, machine, generated_time, metrics_version, metrics, raw_report) VALUES (?, ?, ?, ?, ?, ?)`)
-	if err != nil {
-		util.Close(db, logger)
-		return nil, err
-	}
-
 	go func() {
 		for {
 			select {
@@ -156,12 +150,18 @@ func parseTime(report *model.Report) (*time.Time, error) {
 	parsedTime, err := time.Parse(time.RFC1123Z, report.Generated)
 	if err != nil {
 		parsedTime, err = time.Parse(time.RFC1123, report.Generated)
-		if err != nil {
-			parsedTime, err = time.Parse("Jan 2, 2006, 3:04:05 PM MST", report.Generated)
-			if err != nil {
-				return nil, errors.WithStack(err)
-			}
-		}
+	}
+
+	if err != nil {
+		parsedTime, err = time.Parse("Jan 2, 2006, 3:04:05 PM MST", report.Generated)
+	}
+
+	if err != nil {
+		parsedTime, err = time.Parse("Mon, 2 Jan 2006 15:04:05 -0700", report.Generated)
+	}
+
+	if err != nil {
+		return nil, errors.WithStack(err)
 	}
 	return &parsedTime, nil
 }
@@ -171,10 +171,10 @@ func (t *ReportAnalyzer) Close() error {
 		close(t.input)
 	})
 
-	insertStatement := t.insertStatement
-	if insertStatement != nil {
-		util.Close(insertStatement, t.logger)
-		t.insertStatement = nil
+	putStatement := t.putStatement
+	if putStatement != nil {
+		util.Close(putStatement, t.logger)
+		t.putStatement = nil
 	}
 
 	db := t.db
@@ -193,7 +193,7 @@ func (t *ReportAnalyzer) Done() <-chan struct{} {
 	return t.waitChannel
 }
 
-const metricsVersion = 1
+const metricsVersion = 2
 
 func (t *ReportAnalyzer) doAnalyze(report *model.Report) error {
 	t.waitGroup.Add(1)
@@ -207,13 +207,13 @@ func (t *ReportAnalyzer) doAnalyze(report *model.Report) error {
 
 	id := base64.RawURLEncoding.EncodeToString(t.hash.Sum(nil))
 
-	isAlreadyProcessed, err := t.isReportAlreadyProcessed(id)
+	currentMetricsVersion, err := t.getMetricsVersion(id)
 	if err != nil {
 		return errors.WithStack(err)
 	}
 
 	logger := t.logger.With(zap.String("id", id), zap.String("generatedTime", time.Unix(report.GeneratedTime, 0).Format(time.RFC1123)))
-	if isAlreadyProcessed {
+	if currentMetricsVersion == metricsVersion {
 		logger.Info("report already processed")
 		return nil
 	}
@@ -228,26 +228,46 @@ func (t *ReportAnalyzer) doAnalyze(report *model.Report) error {
 		return errors.WithStack(err)
 	}
 
-	err = t.insertStatement.Exec(id, t.machine, report.GeneratedTime, metricsVersion, serializedMetrics, report.RawData)
+	statement := t.putStatement
+	if statement == nil {
+		statement, err = t.db.Prepare(`REPLACE INTO report (id, machine, generated_time, metrics_version, metrics, raw_report) VALUES (?, ?, ?, ?, ?, ?)`)
+		if err != nil {
+			return err
+		}
+
+		t.putStatement = statement
+	}
+
+	err = statement.Exec(id, t.machine, report.GeneratedTime, metricsVersion, serializedMetrics, report.RawData)
 	if err != nil {
 		return errors.WithStack(err)
 	}
 
-	logger.Info("new report added")
+	if currentMetricsVersion >= 0 && currentMetricsVersion == metricsVersion {
+		logger.Info("report metrics updated", zap.Int("oldMetricsVersion", currentMetricsVersion), zap.Int("newMetricsVersion", metricsVersion))
+	} else {
+		logger.Info("new report added")
+	}
 	return nil
 }
 
-func (t *ReportAnalyzer) isReportAlreadyProcessed(id string) (bool, error) {
+func (t *ReportAnalyzer) getMetricsVersion(id string) (int, error) {
 	stmt, err := t.db.Prepare(`SELECT metrics_version FROM report WHERE id = ?`, id)
 	if err != nil {
-		return false, errors.WithStack(err)
+		return 1, errors.WithStack(err)
 	}
 
 	defer util.Close(stmt, t.logger)
 
 	hasRow, err := stmt.Step()
 	if err != nil {
-		return false, errors.WithStack(err)
+		return -1, errors.WithStack(err)
 	}
-	return hasRow, nil
+
+	if hasRow {
+		result, _, err := stmt.ColumnInt(0)
+		return result, errors.WithStack(err)
+	}
+
+	return -1, nil
 }