Go: Use toolchain directives for version selection if available, and add tests

go/extractor/autobuilder/build-environment.go

@@ -8,6 +8,7 @@ import (
 	"github.com/github/codeql-go/extractor/diagnostics"
 	"github.com/github/codeql-go/extractor/project"
 	"github.com/github/codeql-go/extractor/toolchain"
+	"github.com/github/codeql-go/extractor/util"
 	"golang.org/x/mod/semver"
 )
 
@@ -30,13 +31,13 @@ func (v versionInfo) String() string {
 // Check if `version` is lower than `minGoVersion`. Note that for this comparison we ignore the
 // patch part of the version, so 1.20.1 and 1.20 are considered equal.
 func belowSupportedRange(version string) bool {
-	return semver.Compare(semver.MajorMinor("v"+version), "v"+minGoVersion) < 0
+	return semver.Compare(semver.MajorMinor(util.FormatSemVer(version)), util.FormatSemVer(minGoVersion)) < 0
 }
 
 // Check if `version` is higher than `maxGoVersion`. Note that for this comparison we ignore the
 // patch part of the version, so 1.20.1 and 1.20 are considered equal.
 func aboveSupportedRange(version string) bool {
-	return semver.Compare(semver.MajorMinor("v"+version), "v"+maxGoVersion) > 0
+	return semver.Compare(semver.MajorMinor(util.FormatSemVer(version)), util.FormatSemVer(maxGoVersion)) > 0
 }
 
 // Check if `version` is lower than `minGoVersion` or higher than `maxGoVersion`. Note that for
@@ -113,7 +114,7 @@ func getVersionWhenGoModVersionTooHigh(v versionInfo) (msg, version string) {
 			"). Requesting the maximum supported version of Go (" + maxGoVersion + ")."
 		version = maxGoVersion
 		diagnostics.EmitGoModVersionTooHighAndEnvVersionTooLow(msg)
-	} else if semver.Compare("v"+maxGoVersion, "v"+v.goEnvVersion) > 0 {
+	} else if semver.Compare(util.FormatSemVer(maxGoVersion), util.FormatSemVer(v.goEnvVersion)) > 0 {
 		// The version in the `go.mod` file is above the supported range. The version of Go that
 		// is installed is supported and below the maximum supported version. We install the
 		// maximum supported version as a best effort.
@@ -195,7 +196,7 @@ func getVersionWhenGoModVersionSupported(v versionInfo) (msg, version string) {
 			v.goModVersion + ")."
 		version = v.goModVersion
 		diagnostics.EmitGoModVersionSupportedAndGoEnvUnsupported(msg)
-	} else if semver.Compare("v"+v.goModVersion, "v"+v.goEnvVersion) > 0 {
+	} else if semver.Compare(util.FormatSemVer(v.goModVersion), util.FormatSemVer(v.goEnvVersion)) > 0 {
 		// The version of Go that is installed is supported. The version in the `go.mod` file is
 		// supported and is higher than the version that is installed. We install the version from
 		// the `go.mod` file.
@@ -233,18 +234,18 @@ func getVersionWhenGoModVersionSupported(v versionInfo) (msg, version string) {
 // +-----------------------+-----------------------+-----------------------+-----------------------------------------------------+------------------------------------------------+
 func getVersionToInstall(v versionInfo) (msg, version string) {
 	if !v.goModVersionFound {
-		return getVersionWhenGoModVersionNotFound(v)
-	}
-
-	if aboveSupportedRange(v.goModVersion) {
-		return getVersionWhenGoModVersionTooHigh(v)
-	}
-
-	if belowSupportedRange(v.goModVersion) {
-		return getVersionWhenGoModVersionTooLow(v)
+		msg, version = getVersionWhenGoModVersionNotFound(v)
+	} else if aboveSupportedRange(v.goModVersion) {
+		msg, version = getVersionWhenGoModVersionTooHigh(v)
+	} else if belowSupportedRange(v.goModVersion) {
+		msg, version = getVersionWhenGoModVersionTooLow(v)
+	} else {
+		msg, version = getVersionWhenGoModVersionSupported(v)
 	}
 
-	return getVersionWhenGoModVersionSupported(v)
+	// Make sure that we return a normal version string, not one starting with "v"
+	version = util.UnformatSemVer(version)
+	return
 }
 
 // Output some JSON to stdout specifying the version of Go to install, unless `version` is the

go/extractor/autobuilder/build-environment_test.go

@@ -1,6 +1,10 @@
 package autobuilder
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/github/codeql-go/extractor/util"
+)
 
 func TestGetVersionToInstall(t *testing.T) {
 	tests := map[versionInfo]string{
@@ -44,5 +48,17 @@ func TestGetVersionToInstall(t *testing.T) {
 		if actual != expected {
 			t.Errorf("Expected getVersionToInstall(\"%s\") to be \"%s\", but got \"%s\".", input, expected, actual)
 		}
+
+		if input.goEnvVersionFound {
+			input.goEnvVersion = util.FormatSemVer(input.goEnvVersion)
+		}
+		if input.goEnvVersionFound {
+			input.goModVersion = util.FormatSemVer(input.goModVersion)
+		}
+
+		_, actual = getVersionToInstall(input)
+		if actual != expected {
+			t.Errorf("Expected getVersionToInstall(\"%s\") to be \"%s\", but got \"%s\".", input, expected, actual)
+		}
 	}
 }

go/extractor/cli/go-autobuilder/go-autobuilder.go

@@ -537,12 +537,12 @@ func installDependenciesAndBuild() {
 
 	// This diagnostic is not required if the system Go version is 1.21 or greater, since the
 	// Go tooling should install required Go versions as needed.
-	if semver.Compare(toolchain.GetEnvGoSemVer(), "v1.21.0") < 0 && greatestGoVersion.Found && semver.Compare("v"+greatestGoVersion.Version, toolchain.GetEnvGoSemVer()) > 0 {
-		diagnostics.EmitNewerGoVersionNeeded(toolchain.GetEnvGoSemVer(), "v"+greatestGoVersion.Version)
+	if semver.Compare(toolchain.GetEnvGoSemVer(), "v1.21.0") < 0 && greatestGoVersion.Found && semver.Compare(greatestGoVersion.Version, toolchain.GetEnvGoSemVer()) > 0 {
+		diagnostics.EmitNewerGoVersionNeeded(toolchain.GetEnvGoSemVer(), greatestGoVersion.Version)
 		if val, _ := os.LookupEnv("GITHUB_ACTIONS"); val == "true" {
 			log.Printf(
 				"A go.mod file requires version %s of Go, but version %s is installed. Consider adding an actions/setup-go step to your workflow.\n",
-				"v"+greatestGoVersion.Version,
+				greatestGoVersion.Version,
 				toolchain.GetEnvGoSemVer())
 		}
 	}

go/extractor/project/project.go

@@ -37,6 +37,18 @@ type GoModule struct {
 	Module *modfile.File // The parsed contents of the `go.mod` file
 }
 
+// Tries to find the Go toolchain version required for this module.
+func (module *GoModule) RequiredGoVersion() GoVersionInfo {
+	if module.Module != nil && module.Module.Toolchain != nil {
+		return VersionFound(toolchain.ToolchainVersionToSemVer(module.Module.Toolchain.Name))
+	}
+	if module.Module != nil && module.Module.Go != nil {
+		return VersionFound(toolchain.GoVersionToSemVer(module.Module.Go.Version))
+	} else {
+		return tryReadGoDirective(module.Path)
+	}
+}
+
 // Represents information about a Go project workspace: this may either be a folder containing
 // a `go.work` file or a collection of `go.mod` files.
 type GoWorkspace struct {
@@ -48,54 +60,69 @@ type GoWorkspace struct {
 	Extracted     bool                    // A value indicating whether this workspace was extracted successfully
 }
 
-// Represents a nullable version string.
+// Represents a nullable version string. Use `VersionNotFound` and `VersionFound`
+// instead of constructing values of this type directly.
 type GoVersionInfo struct {
-	// The version string, if any
+	// The semantic version string, such as "v1.20.0-rc1", if any.
+	// This is a valid semantic version if `Found` is `true` or the empty string if not.
 	Version string
-	// A value indicating whether a version string was found
+	// A value indicating whether a version string was found.
+	// If this value is `true`, then `Version` is a valid semantic version.
+	// IF this value is `false`, then `Version` is the empty string.
 	Found bool
 }
 
+// Represents a `GoVersionInfo` indicating that no version was found.
+var VersionNotFound GoVersionInfo = GoVersionInfo{"", false}
+
+// Constructs a `GoVersionInfo` for a version we found.
+func VersionFound(version string) GoVersionInfo {
+	return GoVersionInfo{
+		Version: util.FormatSemVer(version),
+		Found:   true,
+	}
+}
+
 // Determines the version of Go that is required by this workspace. This is, in order of preference:
 // 1. The Go version specified in the `go.work` file, if any.
 // 2. The greatest Go version specified in any `go.mod` file, if any.
 func (workspace *GoWorkspace) RequiredGoVersion() GoVersionInfo {
-	if workspace.WorkspaceFile != nil && workspace.WorkspaceFile.Go != nil {
-		// If we have parsed a `go.work` file, return the version number from it.
-		return GoVersionInfo{Version: workspace.WorkspaceFile.Go.Version, Found: true}
+	// If we have parsed a `go.work` file, we prioritise versions from it over those in individual `go.mod`
+	// files. We are interested in toolchain versions, so if there is an explicit toolchain declaration in
+	// a `go.work` file, we use that. Otherwise, we fall back to the language version in the `go.work` file
+	// and use that as toolchain version. If we didn't parse a `go.work` file, then we try to find the
+	// greatest version contained in `go.mod` files.
+	if workspace.WorkspaceFile != nil && workspace.WorkspaceFile.Toolchain != nil {
+		return VersionFound(toolchain.ToolchainVersionToSemVer(workspace.WorkspaceFile.Toolchain.Name))
+	} else if workspace.WorkspaceFile != nil && workspace.WorkspaceFile.Go != nil {
+		return VersionFound(toolchain.GoVersionToSemVer(workspace.WorkspaceFile.Go.Version))
 	} else if workspace.Modules != nil && len(workspace.Modules) > 0 {
 		// Otherwise, if we have `go.work` files, find the greatest Go version in those.
 		var greatestVersion string = ""
 		for _, module := range workspace.Modules {
-			if module.Module != nil && module.Module.Go != nil {
-				// If we have parsed the file, retrieve the version number we have already obtained.
-				if greatestVersion == "" || semver.Compare("v"+module.Module.Go.Version, "v"+greatestVersion) > 0 {
-					greatestVersion = module.Module.Go.Version
-				}
-			} else {
-				modVersion := tryReadGoDirective(module.Path)
-				if modVersion.Found && (greatestVersion == "" || semver.Compare("v"+modVersion.Version, "v"+greatestVersion) > 0) {
-					greatestVersion = modVersion.Version
-				}
+			moduleVersionInfo := module.RequiredGoVersion()
+
+			if greatestVersion == "" || semver.Compare(moduleVersionInfo.Version, greatestVersion) > 0 {
+				greatestVersion = moduleVersionInfo.Version
 			}
 		}
 
 		// If we have found some version, return it.
 		if greatestVersion != "" {
-			return GoVersionInfo{Version: greatestVersion, Found: true}
+			return VersionFound(greatestVersion)
 		}
 	}
 
-	return GoVersionInfo{Version: "", Found: false}
+	return VersionNotFound
 }
 
 // Finds the greatest Go version required by any of the given `workspaces`.
 // Returns a `GoVersionInfo` value with `Found: false` if no version information is available.
 func RequiredGoVersion(workspaces *[]GoWorkspace) GoVersionInfo {
-	greatestGoVersion := GoVersionInfo{Version: "", Found: false}
+	greatestGoVersion := VersionNotFound
 	for _, workspace := range *workspaces {
 		goVersionInfo := workspace.RequiredGoVersion()
-		if goVersionInfo.Found && (!greatestGoVersion.Found || semver.Compare("v"+goVersionInfo.Version, "v"+greatestGoVersion.Version) > 0) {
+		if goVersionInfo.Found && (!greatestGoVersion.Found || semver.Compare(util.FormatSemVer(goVersionInfo.Version), util.FormatSemVer(greatestGoVersion.Version)) > 0) {
 			greatestGoVersion = goVersionInfo
 		}
 	}
@@ -583,9 +610,9 @@ func tryReadGoDirective(path string) GoVersionInfo {
 		matches := versionRe.FindSubmatch(goMod)
 		if matches != nil {
 			if len(matches) > 1 {
-				return GoVersionInfo{string(matches[1]), true}
+				return VersionFound(toolchain.GoVersionToSemVer(string(matches[1])))
 			}
 		}
 	}
-	return GoVersionInfo{"", false}
+	return VersionNotFound
 }

go/extractor/project/project_test.go

@@ -28,14 +28,18 @@ func TestStartsWithAnyOf(t *testing.T) {
 	testStartsWithAnyOf(t, filepath.Join("foo", "bar"), filepath.Join("foo", "baz"), false)
 }
 
-func testHasInvalidToolchainVersion(t *testing.T, contents string) bool {
-	modFile, err := modfile.Parse("test.go", []byte(contents), nil)
+func parseModFile(t *testing.T, contents string) *modfile.File {
+	modFile, err := modfile.Parse("go.mod", []byte(contents), nil)
 
 	if err != nil {
 		t.Errorf("Unable to parse %s: %s.\n", contents, err.Error())
 	}
 
-	return hasInvalidToolchainVersion(modFile)
+	return modFile
+}
+
+func testHasInvalidToolchainVersion(t *testing.T, contents string) bool {
+	return hasInvalidToolchainVersion(parseModFile(t, contents))
 }
 
 func TestHasInvalidToolchainVersion(t *testing.T) {
@@ -62,3 +66,113 @@ func TestHasInvalidToolchainVersion(t *testing.T) {
 		}
 	}
 }
+
+func parseWorkFile(t *testing.T, contents string) *modfile.WorkFile {
+	workFile, err := modfile.ParseWork("go.work", []byte(contents), nil)
+
+	if err != nil {
+		t.Errorf("Unable to parse %s: %s.\n", contents, err.Error())
+	}
+
+	return workFile
+}
+
+type FileVersionPair struct {
+	FileContents    string
+	ExpectedVersion string
+}
+
+func checkRequiredGoVersionResult(t *testing.T, fun string, file string, testData FileVersionPair, result GoVersionInfo) {
+	if !result.Found {
+		t.Errorf(
+			"Expected %s to return %s for the below `%s` file, but got nothing:\n%s",
+			fun,
+			testData.ExpectedVersion,
+			file,
+			testData.FileContents,
+		)
+	} else if result.Version != testData.ExpectedVersion {
+		t.Errorf(
+			"Expected %s to return %s for the below `%s` file, but got %s:\n%s",
+			fun,
+			testData.ExpectedVersion,
+			file,
+			result.Version,
+			testData.FileContents,
+		)
+	}
+}
+
+func TestRequiredGoVersion(t *testing.T) {
+	testFiles := []FileVersionPair{
+		{"go 1.20", "v1.20.0"},
+		{"go 1.21.2", "v1.21.2"},
+		{"go 1.21rc1", "v1.21.0-rc1"},
+		{"go 1.21rc1\ntoolchain go1.22.0", "v1.22.0"},
+		{"go 1.21rc1\ntoolchain go1.22rc1", "v1.22.0-rc1"},
+	}
+
+	var modules []*GoModule = []*GoModule{}
+
+	for _, testData := range testFiles {
+		// `go.mod` and `go.work` files have mostly the same format
+		modFile := parseModFile(t, testData.FileContents)
+		workFile := parseWorkFile(t, testData.FileContents)
+		mod := &GoModule{
+			Path:   "test", // irrelevant
+			Module: modFile,
+		}
+		work := &GoWorkspace{
+			WorkspaceFile: workFile,
+		}
+
+		result := mod.RequiredGoVersion()
+		checkRequiredGoVersionResult(t, "mod.RequiredGoVersion()", "go.mod", testData, result)
+
+		result = work.RequiredGoVersion()
+		checkRequiredGoVersionResult(t, "work.RequiredGoVersion()", "go.work", testData, result)
+
+		modules = append(modules, mod)
+	}
+
+	// Create a test workspace with all the modules in one workspace.
+	workspace := GoWorkspace{
+		Modules: modules,
+	}
+	workspaceVer := "v1.22.0"
+
+	result := RequiredGoVersion(&[]GoWorkspace{workspace})
+	if !result.Found {
+		t.Errorf(
+			"Expected RequiredGoVersion to return %s, but got nothing.",
+			workspaceVer,
+		)
+	} else if result.Version != workspaceVer {
+		t.Errorf(
+			"Expected RequiredGoVersion to return %s, but got %s.",
+			workspaceVer,
+			result.Version,
+		)
+	}
+
+	// Create test workspaces for each module.
+	workspaces := []GoWorkspace{}
+
+	for _, mod := range modules {
+		workspaces = append(workspaces, GoWorkspace{Modules: []*GoModule{mod}})
+	}
+
+	result = RequiredGoVersion(&workspaces)
+	if !result.Found {
+		t.Errorf(
+			"Expected RequiredGoVersion to return %s, but got nothing.",
+			workspaceVer,
+		)
+	} else if result.Version != workspaceVer {
+		t.Errorf(
+			"Expected RequiredGoVersion to return %s, but got %s.",
+			workspaceVer,
+			result.Version,
+		)
+	}
+}