Skip to content

Commit

Permalink
Log improvement for Targets (#452)
Browse files Browse the repository at this point in the history 
* target logs

* fix iot test

* logging

* Add trace value in unwinding parent catalogs (#454)

* add traceValue when read catalog property in parent catalog

* update trace value

* Turn off validation in standalone mode for cross type check (#446)

* Turn off standalone validation for cross type check

* Handle state.Spec is nil

---------

Co-authored-by: Jiawei Du <59427055+msftcoderdjw@users.noreply.github.com>

* add parent circular dependency check in catalog validation (#453)

* add parent circular dependency validation

* update manifest name

* update helper function

---------

Co-authored-by: Jiawei Du <59427055+msftcoderdjw@users.noreply.github.com>

* Block campaign update and delete with running activation (#458)

* update actions/upload-artifact version from v2 to v4 (#461)

* Support parsing nested JSON properties and schema check for nested properties (#398)

* support nested json properties

* update go mod for coa

* update scenario 04

* update go.sum

* go mod tidy

* update ut and syntax

* update schema doc

* fix test case

* update samples

* add doc for gojq

* revert file

Signed-off-by: linyguo <lingyunguo@microsoft.com>

* revert file

Signed-off-by: linyguo <lingyunguo@microsoft.com>

* revert symphony-ghcr-values.yaml

* merge update unit test

* resolve go mod

* update gojq format string

* revert sample changes

* update

* update config expression check and docs

---------

Signed-off-by: linyguo <lingyunguo@microsoft.com>
Co-authored-by: Jiawei Du <59427055+msftcoderdjw@users.noreply.github.com>

* update canary sample for minikube (#462)

* Log lines improvement for workflow and solution deployment and small refactors (#447)

* Log lines improvement and small refactors

* return status when nextStage is not empty

* Check pause status when selfDriving is false. Fix typo

* Check pauseRequested regardless of selfDriving or not

---------

Co-authored-by: Jiawei Du <59427055+msftcoderdjw@users.noreply.github.com>

* fixed broken markdown links (#471)

* use () to convert expression output to string (#470)

* add context in evaluation context (#459)

* add context in evaluation context

* update context in stage manager

---------

Co-authored-by: Jiawei Du <59427055+msftcoderdjw@users.noreply.github.com>

* checking if the activation still exist for waitcount = 0 or waitcount very large. (#460)

Co-authored-by: Jiawei Du <59427055+msftcoderdjw@users.noreply.github.com>

* fix helm test

---------

Signed-off-by: linyguo <lingyunguo@microsoft.com>
Co-authored-by: Jiawei Du <59427055+msftcoderdjw@users.noreply.github.com>
Co-authored-by: linyguo <lingyunguo@microsoft.com>
Co-authored-by: Xingdong Li <xdlisjtu@gmail.com>
Co-authored-by: Renjie Li <renjieli@microsoft.com>
Co-authored-by: Haishi2016 <hbai@microsoft.com>
  • Loading branch information
@LeiQL @msftcoderdjw
@linyguo @RemindD @lirenjie95 @Haishi2016
6 people authored Sep 19, 2024
1 parent f1ae181 commit f053e55
Show file tree
Hide file tree
Showing 17 changed files with 212 additions and 122 deletions.
22 changes: 13 additions & 9 deletions api/pkg/apis/v1alpha1/providers/target/adb/adb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ func AdbProviderConfigFromMap(properties map[string]string) (AdbProviderConfig,
func (i *AdbProvider) InitWithMap(properties map[string]string) error {
config, err := AdbProviderConfigFromMap(properties)
if err != nil {
aLog.Errorf(" P (Android ADB Target): expected AdbProviderConfig: %+v", err)
return err
}
return i.Init(config)
Expand All @@ -63,11 +64,11 @@ func (i *AdbProvider) Init(config providers.IProviderConfig) error {
defer observ_utils.CloseSpanWithError(span, &err)
defer observ_utils.EmitUserDiagnosticsLogs(ctx, &err)

aLog.InfoCtx(ctx, " P (Android ADB): Init()")
aLog.InfoCtx(ctx, " P (Android ADB Target): Init()")

updateConfig, err := toAdbProviderConfig(config)
if err != nil {
aLog.Errorf(" P (Android ADB): expected AdbProviderConfig: %+v", err)
aLog.ErrorfCtx(ctx, " P (Android ADB Target): expected AdbProviderConfig: %+v", err)
return errors.New("expected AdbProviderConfig")
}
i.Config = updateConfig
Expand All @@ -94,10 +95,10 @@ func (i *AdbProvider) Get(ctx context.Context, deployment model.DeploymentSpec,

if deployment.Instance.Spec == nil {
err = errors.New("deployment instance spec is nil")
aLog.ErrorfCtx(ctx, " P (Android ADB): failed to get deployment, error: %+v", err)
aLog.ErrorfCtx(ctx, " P (Android ADB Target): failed to get deployment, error: %+v", err)
return nil, err
}
aLog.InfofCtx(ctx, " P (Android ADB): getting artifacts: %s - %s", deployment.Instance.Spec.Scope, deployment.Instance.ObjectMeta.Name)
aLog.InfofCtx(ctx, " P (Android ADB Target): getting artifacts: %s - %s", deployment.Instance.Spec.Scope, deployment.Instance.ObjectMeta.Name)

ret := make([]model.ComponentSpec, 0)

Expand All @@ -115,7 +116,7 @@ func (i *AdbProvider) Get(ctx context.Context, deployment model.DeploymentSpec,
out, err = exec.Command("adb", params...).Output()

if err != nil {
aLog.ErrorfCtx(ctx, " P (Android ADB): failed to get application %+v, error: %+v", p, err)
aLog.ErrorfCtx(ctx, " P (Android ADB Target): failed to get application %+v, error: %+v", p, err)
return nil, err
}
str := string(out)
Expand All @@ -141,22 +142,24 @@ func (i *AdbProvider) Apply(ctx context.Context, deployment model.DeploymentSpec
defer observ_utils.CloseSpanWithError(span, &err)
defer observ_utils.EmitUserDiagnosticsLogs(ctx, &err)

aLog.InfofCtx(ctx, " P (Android ADB Provider): applying artifacts: %s - %s", deployment.Instance.Spec.Scope, deployment.Instance.ObjectMeta.Name)
aLog.InfofCtx(ctx, " P (Android ADB Target): applying artifacts: %s - %s", deployment.Instance.Spec.Scope, deployment.Instance.ObjectMeta.Name)

components := step.GetComponents()

err = i.GetValidationRule(ctx).Validate(components)
if err != nil {
aLog.ErrorfCtx(ctx, " P (Android ADB Provider): failed to validate components, error: %v", err)
aLog.ErrorfCtx(ctx, " P (Android ADB Target): failed to validate components, error: %v", err)
return nil, err
}
if isDryRun {
aLog.DebugCtx(ctx, " P (Android ADB Target): dryRun is enabled, skipping apply")
err = nil
return nil, nil
}
ret := step.PrepareResultMap()
components = step.GetUpdatedComponents()
if len(components) > 0 {
aLog.InfofCtx(ctx, " P (Android ADB Target): get updated components: count - %d", len(components))
for _, component := range components {
if component.Name != "" {
if p, ok := component.Properties[model.AppImage]; ok && p != "" {
Expand All @@ -167,7 +170,7 @@ func (i *AdbProvider) Apply(ctx context.Context, deployment model.DeploymentSpec
cmd := exec.Command("adb", params...)
err = cmd.Run()
if err != nil {
aLog.ErrorfCtx(ctx, " P (Android ADB): failed to install application %+v, error: %+v", p, err)
aLog.ErrorfCtx(ctx, " P (Android ADB Target): failed to install application %+v, error: %+v", p, err)
ret[component.Name] = model.ComponentResultSpec{
Status: v1alpha2.UpdateFailed,
Message: err.Error(),
Expand All @@ -181,6 +184,7 @@ func (i *AdbProvider) Apply(ctx context.Context, deployment model.DeploymentSpec
}
components = step.GetDeletedComponents()
if len(components) > 0 {
aLog.InfofCtx(ctx, " P (Android ADB Target): get deleted components: count - %d", len(components))
for _, component := range components {
if component.Name != "" {
if p, ok := component.Properties[model.AppPackage]; ok && p != "" {
Expand All @@ -191,7 +195,7 @@ func (i *AdbProvider) Apply(ctx context.Context, deployment model.DeploymentSpec
cmd := exec.Command("adb", params...)
err = cmd.Run()
if err != nil {
aLog.ErrorfCtx(ctx, " P (Android ADB): failed to uninstall application %+v, error: %+v", p, err)
aLog.ErrorfCtx(ctx, " P (Android ADB Target): failed to uninstall application %+v, error: %+v", p, err)
ret[component.Name] = model.ComponentResultSpec{
Status: v1alpha2.DeleteFailed,
Message: err.Error(),
Expand Down
5 changes: 3 additions & 2 deletions api/pkg/apis/v1alpha1/providers/target/azure/adu/adu.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ func ADUTargetProviderConfigFromMap(properties map[string]string) (ADUTargetProv
func (i *ADUTargetProvider) InitWithMap(properties map[string]string) error {
config, err := ADUTargetProviderConfigFromMap(properties)
if err != nil {
sLog.Errorf(" P (ADU Target): expected ADUTargetProviderConfig %+v", err)
sLog.Errorf(" P (ADU Target): expected ADUTargetProviderConfig: %+v", err)
return err
}
return i.Init(config)
Expand All @@ -103,7 +103,7 @@ func (i *ADUTargetProvider) Init(config providers.IProviderConfig) error {

updateConfig, err := toADUTargetProviderConfig(config)
if err != nil {
sLog.Errorf(" P (ADU Target): expected ADUTargetProviderConfig: %+v", err)
sLog.ErrorfCtx(ctx, " P (ADU Target): expected ADUTargetProviderConfig: %+v", err)
return err
}
i.Config = updateConfig
Expand Down Expand Up @@ -205,6 +205,7 @@ func (i *ADUTargetProvider) Apply(ctx context.Context, deployment model.Deployme
return nil, err
}
if isDryRun {
sLog.DebugCtx(ctx, " P (ADU Target): dryRun is enabled, skipping apply")
err = nil
return nil, nil
}
Expand Down
46 changes: 28 additions & 18 deletions api/pkg/apis/v1alpha1/providers/target/azure/iotedge/iotedge.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ func IoTEdgeTargetProviderConfigFromMap(properties map[string]string) (IoTEdgeTa
func (i *IoTEdgeTargetProvider) InitWithMap(properties map[string]string) error {
config, err := IoTEdgeTargetProviderConfigFromMap(properties)
if err != nil {
sLog.Errorf(" P (IoT Edge Target): expected IoTEdgeTargetProviderConfig: %+v", err)
return err
}
return i.Init(config)
Expand Down Expand Up @@ -235,9 +236,11 @@ func (i *IoTEdgeTargetProvider) Apply(ctx context.Context, deployment model.Depl
components := step.GetComponents()
err = i.GetValidationRule(ctx).Validate(components)
if err != nil {
sLog.ErrorfCtx(ctx, " P (IoT Edge Target): failed to validate components: %+v", err)
return nil, err
}
if isDryRun {
sLog.DebugfCtx(ctx, " P (IoT Edge Target): dry run, no action taken")
err = nil
return nil, nil
}
Expand Down Expand Up @@ -272,6 +275,7 @@ func (i *IoTEdgeTargetProvider) Apply(ctx context.Context, deployment model.Depl
modules[a.Name] = module
}
if len(modules) > 0 {
sLog.InfofCtx(ctx, " P (IoT Edge Target): deploy to IoT edge with modules count %d", len(modules))
err = i.deployToIoTEdge(ctx, deployment.Instance.ObjectMeta.Name, deployment.Instance.Spec.Metadata, modules, edgeAgent, edgeHub)
if err != nil {
sLog.ErrorfCtx(ctx, " P (IoT Edge Target): failed to deploy to IoT edge: %+v", err)
Expand All @@ -294,6 +298,7 @@ func (i *IoTEdgeTargetProvider) Apply(ctx context.Context, deployment model.Depl
modules[a.Name] = module
}
if len(modules) > 0 {
sLog.InfofCtx(ctx, " P (IoT Edge Target): remove from IoT edge with modules count %d", len(modules))
err = i.remvoefromIoTEdge(ctx, deployment.Instance.ObjectMeta.Name, deployment.Instance.Spec.Metadata, modules, edgeAgent, edgeHub)
if err != nil {
sLog.ErrorfCtx(ctx, " P (IoT Edge Target): failed to remove from IoT edge: %+v", err)
Expand Down Expand Up @@ -545,34 +550,36 @@ func (i *IoTEdgeTargetProvider) getIoTEdgeModuleTwin(ctx context.Context, id str
defer observ_utils.CloseSpanWithError(span, &err)
defer observ_utils.EmitUserDiagnosticsLogs(ctx, &err)

sLog.InfofCtx(ctx, " P (IoT Edge Target): start to get IoT Edge modules using url %v", url)

module := ModuleTwin{}
sasToken := azureutils.CreateSASToken(fmt.Sprintf("%s/devices/%s", i.Config.IoTHub, i.Config.DeviceName), i.Config.KeyName, i.Config.Key)
client := &http.Client{}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
sLog.ErrorfCtx(ctx, "failed to get IoT Edge modules: %v", err)
sLog.ErrorfCtx(ctx, " P (IoT Edge Target): failed to get IoT Edge modules: %v", err)
return module, v1alpha2.NewCOAError(err, "failed to get IoT Edge modules", v1alpha2.InternalError)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", sasToken)
resp, err := client.Do(req)
if err != nil {
sLog.ErrorfCtx(ctx, "failed to get IoT Edge modules: %v", err)
sLog.ErrorfCtx(ctx, " P (IoT Edge Target): failed to get IoT Edge modules: %v", err)
return module, v1alpha2.NewCOAError(err, "failed to get IoT Edge modules", v1alpha2.InternalError)
}
if resp.StatusCode != http.StatusOK {
sLog.ErrorfCtx(ctx, "failed to get IoT Edge modules: %v", resp)
sLog.ErrorfCtx(ctx, " P (IoT Edge Target): failed to get IoT Edge modules: %v", resp)
//return module, v1alpha1.NewCOAError(nil, "failed to get IoT Edge modules", v1alpha1.InternalError) //TODO: carry over HTTP status code
}
defer resp.Body.Close()
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
sLog.ErrorfCtx(ctx, "failed to get IoT Edge modules: %v", err)
sLog.ErrorfCtx(ctx, " P (IoT Edge Target): failed to get IoT Edge modules: %v", err)
return module, v1alpha2.NewCOAError(err, "failed to get IoT Edge modules", v1alpha2.InternalError)
}
err = json.Unmarshal(bodyBytes, &module)
if err != nil {
sLog.ErrorfCtx(ctx, "failed to get IoT Edge modules: %v", err)
sLog.ErrorfCtx(ctx, " P (IoT Edge Target): failed to get IoT Edge modules: %v", err)
return module, v1alpha2.NewCOAError(err, "failed to get IoT Edge modules", v1alpha2.InternalError)
}
return module, nil
Expand All @@ -596,19 +603,18 @@ func (i *IoTEdgeTargetProvider) getIoTEdgeModules(ctx context.Context) (map[stri
}

func (i *IoTEdgeTargetProvider) remvoefromIoTEdge(ctx context.Context, name string, metadata map[string]string, modules map[string]Module, agentRef ModuleTwin, hubRef ModuleTwin) error {
deployment := makeDefaultDeployment(metadata, i.Config.EdgeAgentVersion, i.Config.EdgeHubVersion)
err := reduceDeployment(&deployment, name, modules, agentRef, hubRef)
deployment := makeDefaultDeployment(ctx, metadata, i.Config.EdgeAgentVersion, i.Config.EdgeHubVersion)
err := reduceDeployment(ctx, &deployment, name, modules, agentRef, hubRef)
if err != nil {
return err
}
return i.applyIoTEdgeDeployment(ctx, deployment)
}

func (i *IoTEdgeTargetProvider) deployToIoTEdge(ctx context.Context, name string, metadata map[string]string, modules map[string]Module, agentRef ModuleTwin, hubRef ModuleTwin) error {
deployment := makeDefaultDeployment(ctx, metadata, i.Config.EdgeAgentVersion, i.Config.EdgeHubVersion)

deployment := makeDefaultDeployment(metadata, i.Config.EdgeAgentVersion, i.Config.EdgeHubVersion)

err := updateDeployment(&deployment, name, modules, agentRef, hubRef)
err := updateDeployment(ctx, &deployment, name, modules, agentRef, hubRef)
if err != nil {
return err
}
Expand All @@ -630,23 +636,24 @@ func (i *IoTEdgeTargetProvider) applyIoTEdgeDeployment(ctx context.Context, depl
client := &http.Client{}
payload, err := json.Marshal(deployment)
if err != nil {
sLog.ErrorfCtx(ctx, " P (IoT Edge Target): failed to marshall deployment: %v", err)
return v1alpha2.NewCOAError(err, "failed to serialize IoT Edge deployemnt", v1alpha2.SerializationError)
}

req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(payload))
if err != nil {
sLog.ErrorfCtx(ctx, "failed to post IoT Edge deployment: %v", err)
sLog.ErrorfCtx(ctx, " P (IoT Edge Target): failed to post IoT Edge deployment: %v", err)
return v1alpha2.NewCOAError(err, "failed to post IoT Edge deployment", v1alpha2.InternalError)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", sasToken)
resp, err := client.Do(req)
if err != nil {
sLog.ErrorfCtx(ctx, "failed to post IoT Edge deployment: %v", err)
sLog.ErrorfCtx(ctx, " P (IoT Edge Target): failed to post IoT Edge deployment: %v", err)
return v1alpha2.NewCOAError(err, "failed to post IoT Edge deployment", v1alpha2.InternalError)
}
if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
sLog.ErrorfCtx(ctx, "failed to post IoT Edge deployment: %v", resp)
sLog.ErrorfCtx(ctx, " P (IoT Edge Target): failed to post IoT Edge deployment: %v", resp)
return v1alpha2.NewCOAError(nil, "failed to post IoT Edge deployment", v1alpha2.InternalError) //TODO: carry over HTTP status code
}
return nil
Expand Down Expand Up @@ -685,7 +692,8 @@ func carryOverRoutes(deployment *IoTEdgeDeployment, ref ModuleTwin) {
}
}

func updateDeployment(deployment *IoTEdgeDeployment, name string, modules map[string]Module, agentRef ModuleTwin, hubRef ModuleTwin) error {
func updateDeployment(ctx context.Context, deployment *IoTEdgeDeployment, name string, modules map[string]Module, agentRef ModuleTwin, hubRef ModuleTwin) error {
sLog.InfoCtx(ctx, " P (IoT Edge Target): reduceDeployment()")

// add all other modules that are not in the current module list so that we can write them back
otherModules := map[string]bool{}
Expand Down Expand Up @@ -750,6 +758,7 @@ func updateDeployment(deployment *IoTEdgeDeployment, name string, modules map[st
}
return nil
}

func modifyRoutes(route string, name string, modules map[string]Module, otherModules map[string]bool) string {
for k, _ := range modules {
if _, ok := otherModules[k]; !ok {
Expand All @@ -759,8 +768,8 @@ func modifyRoutes(route string, name string, modules map[string]Module, otherMod
return route
}

func reduceDeployment(deployment *IoTEdgeDeployment, name string, modules map[string]Module, ref ModuleTwin, hubRef ModuleTwin) error {

func reduceDeployment(ctx context.Context, deployment *IoTEdgeDeployment, name string, modules map[string]Module, ref ModuleTwin, hubRef ModuleTwin) error {
sLog.InfoCtx(ctx, " P (IoT Edge Target): reduceDeployment()")
otherModules := map[string]bool{}

rd := deployment.ModulesContent["$edgeHub"].DesiredProperties["routes"].(map[string]string)
Expand Down Expand Up @@ -820,8 +829,7 @@ func reduceDeployment(deployment *IoTEdgeDeployment, name string, modules map[st
return nil
}

func makeDefaultDeployment(metadata map[string]string, edgeAgentVersion string, edgeHubVersion string) IoTEdgeDeployment {

func makeDefaultDeployment(ctx context.Context, metadata map[string]string, edgeAgentVersion string, edgeHubVersion string) IoTEdgeDeployment {
deployment := IoTEdgeDeployment{
ModulesContent: map[string]ModuleState{
"$edgeAgent": {
Expand Down Expand Up @@ -871,6 +879,8 @@ func makeDefaultDeployment(metadata map[string]string, edgeAgentVersion string,
err := json.Unmarshal(data, &credentials)
if err == nil {
(deployment.ModulesContent["$edgeAgent"].DesiredProperties["runtime"].(Runtime)).Settings["registryCredentials"] = credentials
} else {
sLog.ErrorfCtx(ctx, " P (IoT Edge Target): failed to unmarshall credentials: %v", err)
}
}
return deployment
Expand Down
Loading

0 comments on commit f053e55

Please sign in to comment.