Add the possibility to run tests with an Agent 6 pipeline artefacts

common/config/environment.go

@@ -52,6 +52,7 @@ const (
 	DDAgentAPPKeyParamName               = "appKey"
 	DDAgentFakeintake                    = "fakeintake"
 	DDAgentSite                          = "site"
+	DDAgentMajorVersion                  = "majorVersion"
 	DDAgentExtraEnvVars                  = "extraEnvVars" // extraEnvVars is expected in the format: <key1>=<value1>,<key2>=<value2>,...
 
 	// Updater Namespace
@@ -117,6 +118,7 @@ type Env interface {
 	DogstatsdDeploy() bool
 	DogstatsdFullImagePath() string
 	UpdaterDeploy() bool
+	MajorVersion() string
 
 	GetBoolWithDefault(config *sdkconfig.Config, paramName string, defaultValue bool) bool
 	GetStringListWithDefault(config *sdkconfig.Config, paramName string, defaultValue []string) []string
@@ -306,6 +308,10 @@ func (e *CommonEnvironment) Site() string {
 	return e.AgentConfig.Get(DDAgentSite)
 }
 
+func (e *CommonEnvironment) MajorVersion() string {
+	return e.GetStringWithDefault(e.AgentConfig, DDAgentMajorVersion, "7")
+}
+
 func (e *CommonEnvironment) AgentExtraEnvVars() map[string]string {
 	result := make(map[string]string)
 	envVars, err := e.AgentConfig.Try(DDAgentExtraEnvVars)

components/datadog/agent/host_linuxos.go

@@ -2,17 +2,18 @@ package agent
 
 import (
 	"fmt"
-	"strings"
-
 	"github.com/DataDog/test-infra-definitions/components/command"
 	"github.com/DataDog/test-infra-definitions/components/datadog/agentparams"
 	"github.com/DataDog/test-infra-definitions/components/os"
 	remoteComp "github.com/DataDog/test-infra-definitions/components/remote"
+	"strings"
 
 	"github.com/pulumi/pulumi-command/sdk/go/command/remote"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 )
 
+const DefaultMajorVersion = "7"
+
 type agentLinuxManager struct {
 	targetOS os.OS
 }
@@ -22,33 +23,32 @@ func newLinuxManager(host *remoteComp.Host) agentOSManager {
 }
 
 func (am *agentLinuxManager) getInstallCommand(version agentparams.PackageVersion, _ []string) (string, error) {
+	var commandLine string
+	testEnvVars := []string{}
+
 	if version.PipelineID != "" {
-		testEnvVars := []string{}
 		testEnvVars = append(testEnvVars, "TESTING_APT_URL=apttesting.datad0g.com")
 		// apt testing repo
 		// TESTING_APT_REPO_VERSION="pipeline-xxxxx-a7 7"
-		testEnvVars = append(testEnvVars, fmt.Sprintf(`TESTING_APT_REPO_VERSION="pipeline-%v-a7-%s 7"`, version.PipelineID, am.targetOS.Descriptor().Architecture))
+		testEnvVars = append(testEnvVars, fmt.Sprintf(`TESTING_APT_REPO_VERSION="pipeline-%[1]v-a%[2]v-%[3]s %[2]v"`, version.PipelineID, version.Major, am.targetOS.Descriptor().Architecture))
 		testEnvVars = append(testEnvVars, "TESTING_YUM_URL=yumtesting.datad0g.com")
 		// yum testing repo
 		// TESTING_YUM_VERSION_PATH="testing/pipeline-xxxxx-a7/7"
-		testEnvVars = append(testEnvVars, fmt.Sprintf("TESTING_YUM_VERSION_PATH=testing/pipeline-%v-a7/7", version.PipelineID))
-		commandLine := strings.Join(testEnvVars, " ")
-
-		return fmt.Sprintf(
-			`for i in 1 2 3 4 5; do curl -fsSL https://s3.amazonaws.com/dd-agent/scripts/%v -o install-script.sh && break || sleep $((2**$i)); done &&  for i in 1 2 3; do DD_API_KEY=%%s %v DD_INSTALL_ONLY=true bash install-script.sh  && exit 0  || sleep $((2**$i)); done; exit 1`,
-			"install_script_agent7.sh",
-			commandLine), nil
-	}
+		testEnvVars = append(testEnvVars, fmt.Sprintf("TESTING_YUM_VERSION_PATH=testing/pipeline-%[1]v-a%[2]v/%[2]v", version.PipelineID, version.Major))
+	} else {
+		testEnvVars = append(testEnvVars, fmt.Sprintf("DD_AGENT_MAJOR_VERSION=%v", version.Major))
 
-	commandLine := fmt.Sprintf("DD_AGENT_MAJOR_VERSION=%v ", version.Major)
+		if version.Minor != "" {
+			testEnvVars = append(testEnvVars, fmt.Sprintf("DD_AGENT_MINOR_VERSION=%v", version.Minor))
+		}
 
-	if version.Minor != "" {
-		commandLine += fmt.Sprintf("DD_AGENT_MINOR_VERSION=%v ", version.Minor)
+		if version.Channel != "" && version.Channel != agentparams.StableChannel {
+			testEnvVars = append(testEnvVars, "REPO_URL=datad0g.com")
+			testEnvVars = append(testEnvVars, fmt.Sprintf("DD_AGENT_DIST_CHANNEL=%s", version.Channel))
+		}
 	}
 
-	if version.Channel != "" && version.Channel != agentparams.StableChannel {
-		commandLine += fmt.Sprintf("REPO_URL=datad0g.com DD_AGENT_DIST_CHANNEL=%s ", version.Channel)
-	}
+	commandLine = strings.Join(testEnvVars, " ")
 
 	return fmt.Sprintf(
 		`for i in 1 2 3 4 5; do curl -fsSL https://s3.amazonaws.com/dd-agent/scripts/%v -o install-script.sh && break || sleep $((2**$i)); done &&  for i in 1 2 3; do DD_API_KEY=%%s %v DD_INSTALL_ONLY=true bash install-script.sh  && exit 0 || sleep $((2**$i)); done; exit 1`,

components/datadog/agent/host_windowsos.go

@@ -115,7 +115,7 @@ func getAgentURL(version agentparams.PackageVersion) (string, error) {
 	fullVersion := fmt.Sprintf("%v.%v", version.Major, minor)
 
 	if version.PipelineID != "" {
-		return getAgentURLFromPipelineID(version.PipelineID)
+		return getAgentURLFromPipelineID(version.PipelineID, version.Major)
 	}
 
 	if version.Channel == agentparams.BetaChannel {
@@ -150,7 +150,7 @@ func getAgentURL(version agentparams.PackageVersion) (string, error) {
 	return finder.findVersion(fullVersion)
 }
 
-func getAgentURLFromPipelineID(pipelineID string) (string, error) {
+func getAgentURLFromPipelineID(pipelineID string, majorVersion string) (string, error) {
 	// TODO: Replace context.Background() with a Pulumi context.Context.
 	// dd-agent-mstesting is a public bucket so we can use anonymous credentials
 	config, err := awsConfig.LoadDefaultConfig(context.Background(), awsConfig.WithCredentialsProvider(aws.AnonymousCredentials{}))
@@ -162,7 +162,7 @@ func getAgentURLFromPipelineID(pipelineID string) (string, error) {
 
 	result, err := s3Client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{
 		Bucket: aws.String("dd-agent-mstesting"),
-		Prefix: aws.String(fmt.Sprintf("pipelines/A7/%v", pipelineID)),
+		Prefix: aws.String(fmt.Sprintf("pipelines/A%v/%v", majorVersion, pipelineID)),
 	})
 	if err != nil {
 		return "", err

components/datadog/agentparams/params.go

@@ -60,6 +60,8 @@ type Params struct {
 
 type Option = func(*Params) error
 
+const DefaultMajorVersion = "7"
+
 func NewParams(env config.Env, options ...Option) (*Params, error) {
 	p := &Params{
 		Integrations: make(map[string]*FileDefinition),
@@ -72,6 +74,11 @@ func NewParams(env config.Env, options ...Option) (*Params, error) {
 	if env.AgentVersion() != "" {
 		defaultVersion = WithVersion(env.AgentVersion())
 	}
+
+	if env.MajorVersion() != "" {
+		options = append([]Option{WithMajorVersion(env.MajorVersion())}, options...)
+	}
+
 	options = append([]Option{defaultVersion}, options...)
 	return common.ApplyOption(p, options)
 }
@@ -109,9 +116,16 @@ func WithVersion(version string) func(*Params) error {
 // WithPipeline use a specific version of the Agent by pipeline id
 func WithPipeline(pipelineID string) func(*Params) error {
 	return func(p *Params) error {
-		p.Version = PackageVersion{
-			PipelineID: pipelineID,
-		}
+		p.Version.PipelineID = pipelineID
+		p.Version.Major = DefaultMajorVersion
+		return nil
+	}
+}
+
+// WithMajorVersion specify the major version of the Agent
+func WithMajorVersion(majorVersion string) func(*Params) error {
+	return func(p *Params) error {
+		p.Version.Major = majorVersion
 		return nil
 	}
 }

components/datadog/agentparams/params_test.go

@@ -33,6 +33,7 @@ func TestParams(t *testing.T) {
 		assert.NoError(t, err)
 		assert.Equal(t, result.Version, PackageVersion{
 			PipelineID: "16362517",
+			Major:      "7",
 		})
 	})
 	t.Run("WithIntegration should correctly add conf.d/integration/conf.yaml to the path", func(t *testing.T) {