diff --git a/agreementbot/governance.go b/agreementbot/governance.go index 565fe916c..69d067fba 100644 --- a/agreementbot/governance.go +++ b/agreementbot/governance.go @@ -46,6 +46,7 @@ func (w *AgreementBotWorker) GovernAgreements() int { // Grab the next set of secret updates to process. secretUpdates := w.secretUpdateManager.GetNextUpdateEvent() + secretExistsMap := make(map[string]bool) // Look at all agreements across all protocols for _, agp := range policy.AllAgreementProtocols() { @@ -202,13 +203,13 @@ func (w *AgreementBotWorker) GovernAgreements() int { glog.Infof(logString(fmt.Sprintf("checking secret %v against %v", updatedSecretName, bs))) } // Call the secret manager plugin to get the secret details. - secretUser, _, secretName, err := compcheck.ParseVaultSecretName(exchange.GetId(updatedSecretName), nil) + secretUser, updateSecretNode, secretName, err := compcheck.ParseVaultSecretName(exchange.GetId(updatedSecretName), nil) if err != nil { glog.Errorf(logString(fmt.Sprintf("error parsing secret %s, error: %v", updatedSecretName, err))) continue } if smSecretName == exchange.GetId(updatedSecretName) || smSecretName == fmt.Sprintf("user/%s/%s", secretUser, secretName) || smSecretName == secretName { - + secretExistsMap[updatedSecretName] = true newBS := make(exchangecommon.BoundSecret) if binding.EnableNodeLevelSecrets { secretNode := exchange.GetId(ag.DeviceId) @@ -220,6 +221,9 @@ func (w *AgreementBotWorker) GovernAgreements() int { details, err := w.secretProvider.GetSecretDetails(w.GetExchangeId(), w.GetExchangeToken(), exchange.GetOrg(updatedSecretName), secretUser, secretNode, secretName) if err != nil { glog.Errorf(logString(fmt.Sprintf("error retrieving secret %v for policy %v, error: %v", updatedSecretName, ag.PolicyName, err))) + if updateSecretNode != "" { + secretExistsMap[updatedSecretName] = false + } } else { detailBytes, err := json.Marshal(details) if err != nil { @@ -242,6 +246,9 @@ func (w *AgreementBotWorker) GovernAgreements() int { details, err := w.secretProvider.GetSecretDetails(w.GetExchangeId(), w.GetExchangeToken(), exchange.GetOrg(updatedSecretName), secretUser, "", secretName) if err != nil { glog.Errorf(logString(fmt.Sprintf("error retrieving secret %v for policy %v, error: %v", updatedSecretName, ag.PolicyName, err))) + if updateSecretNode == "" { + secretExistsMap[updatedSecretName] = false + } continue } detailBytes, err := json.Marshal(details) @@ -314,8 +321,11 @@ func (w *AgreementBotWorker) GovernAgreements() int { if secretUpdates != nil { for _, su := range secretUpdates.Updates { glog.V(5).Infof(logString(fmt.Sprintf("updating secret DB %s/%s with time %v", su.SecretOrg, su.SecretFullName, su.SecretUpdateTime))) - - err := w.db.SetSecretUpdate(su.SecretOrg, su.SecretFullName, su.SecretUpdateTime) + secretExists := true + if exists, ok := secretExistsMap[su.SecretFullName]; ok { + secretExists = exists + } + err := w.db.SetSecretUpdate(su.SecretOrg, su.SecretFullName, su.SecretUpdateTime, secretExists) if err != nil { glog.Errorf(logString(fmt.Sprintf("unable to save secret update time for %s/%s, error: %v", su.SecretOrg, su.SecretFullName, err))) } diff --git a/agreementbot/persistence/bolt/secrets.go b/agreementbot/persistence/bolt/secrets.go index 1d73f6d51..63cb0e21a 100644 --- a/agreementbot/persistence/bolt/secrets.go +++ b/agreementbot/persistence/bolt/secrets.go @@ -1,6 +1,6 @@ package bolt -func (db *AgbotBoltDB) AddManagedPolicySecret(secretOrg, secretName, policyOrg, policyName string, updateTime int64) error { +func (db *AgbotBoltDB) AddManagedPolicySecret(secretOrg, secretName, policyOrg, policyName string, secretExists bool, updateTime int64) error { return nil } @@ -8,11 +8,15 @@ func (db *AgbotBoltDB) GetManagedPolicySecretNames(policyOrg, policyName string) return []string{}, nil } -func (db *AgbotBoltDB) GetPoliciesWithUpdatedSecrets(secretOrg, secretName string, lastUpdate int64) ([]string, error) { +func (db *AgbotBoltDB) GetPoliciesWithUpdatedSecrets(secretOrg, secretName string, lastUpdate int64, secretExists bool) ([]string, error) { return []string{}, nil } -func (db *AgbotBoltDB) SetSecretUpdate(secretOrg, secretName string, secretUpdateTime int64) error { +func (db *AgbotBoltDB) SetSecretUpdate(secretOrg, secretName string, secretUpdateTime int64, secretExists bool) error { + return nil +} + +func (db *AgbotBoltDB) SetSecretExists(secretOrg, secretName string, secretUpdateTime int64) error { return nil } @@ -28,7 +32,7 @@ func (db *AgbotBoltDB) DeletePolicySecret(secretOrg, secretName, policyOrg, poli return nil } -func (db *AgbotBoltDB) AddManagedPatternSecret(secretOrg, secretName, policyOrg, policyName string, updateTime int64) error { +func (db *AgbotBoltDB) AddManagedPatternSecret(secretOrg, secretName, policyOrg, policyName string, secretExists bool, updateTime int64) error { return nil } @@ -36,7 +40,7 @@ func (db *AgbotBoltDB) GetManagedPatternSecretNames(policyOrg, policyName string return []string{}, nil } -func (db *AgbotBoltDB) GetPatternsWithUpdatedSecrets(secretOrg, secretName string, lastUpdate int64) ([]string, error) { +func (db *AgbotBoltDB) GetPatternsWithUpdatedSecrets(secretOrg, secretName string, lastUpdate int64, secretExists bool) ([]string, error) { return []string{}, nil } diff --git a/agreementbot/persistence/database.go b/agreementbot/persistence/database.go index eaa47a83a..2e7762ce7 100644 --- a/agreementbot/persistence/database.go +++ b/agreementbot/persistence/database.go @@ -80,13 +80,14 @@ type AgbotDatabase interface { DumpSearchSessions() error // Functions related to persistence of secrets in use by active agreements. - AddManagedPolicySecret(secretOrg, secretName, policyOrg, policyName string, updateTime int64) error - AddManagedPatternSecret(secretOrg, secretName, patternOrg, patternName string, updateTime int64) error + AddManagedPolicySecret(secretOrg, secretName, policyOrg, policyName string, secretExists bool, updateTime int64) error + AddManagedPatternSecret(secretOrg, secretName, patternOrg, patternName string, secretExists bool, updateTime int64) error GetManagedPolicySecretNames(policyOrg, policyName string) ([]string, error) GetManagedPatternSecretNames(patternOrg, patternName string) ([]string, error) - GetPoliciesWithUpdatedSecrets(secretOrg, secretName string, lastUpdate int64) ([]string, error) - GetPatternsWithUpdatedSecrets(secretOrg, secretName string, lastUpdate int64) ([]string, error) - SetSecretUpdate(secretOrg, secretName string, secretUpdateTime int64) error + GetPoliciesWithUpdatedSecrets(secretOrg, secretName string, lastUpdate int64, secretExists bool) ([]string, error) + GetPatternsWithUpdatedSecrets(secretOrg, secretName string, lastUpdate int64, secretExists bool) ([]string, error) + SetSecretUpdate(secretOrg, secretName string, secretUpdateTime int64, secretExists bool) error + SetSecretExists(secretOrg, secretName string, secretUpdateTime int64) error GetPoliciesInOrg(org string) ([]string, error) GetPatternsInOrg(org string) ([]string, error) DeleteSecretsForPolicy(polOrg, polName string) error diff --git a/agreementbot/persistence/postgresql/secrets.go b/agreementbot/persistence/postgresql/secrets.go index 51ff75f19..d373a7cbe 100644 --- a/agreementbot/persistence/postgresql/secrets.go +++ b/agreementbot/persistence/postgresql/secrets.go @@ -15,6 +15,7 @@ const SECRET_CREATE_MAIN_TABLE_POLICY = `CREATE TABLE IF NOT EXISTS secrets_poli secret_name text NOT NULL, policy_org text NOT NULL, policy_name text NOT NULL, + secret_exists boolean NOT NULL, last_update_check int NOT NULL, partition text NOT NULL, updated timestamp with time zone DEFAULT current_timestamp @@ -25,6 +26,7 @@ const SECRET_CREATE_MAIN_TABLE_PATTERN = `CREATE TABLE IF NOT EXISTS secrets_pat secret_name text NOT NULL, pattern_org text NOT NULL, pattern_name text NOT NULL, + secret_exists boolean NOT NULL, last_update_check int NOT NULL, partition text NOT NULL, updated timestamp with time zone DEFAULT current_timestamp @@ -51,12 +53,14 @@ const SECRET_TABLE_NAME_ROOT_POLICY = `secrets_policy_` const SECRET_TABLE_NAME_ROOT_PATTERN = `secrets_pattern_` const SECRET_PARTITION_FILLIN = `partition_name` -const SECRET_INSERT_POLICY = `INSERT INTO "secrets_policy_ (secret_org, secret_name, policy_org, policy_name, last_update_check, partition) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING;` -const SECRET_UPDATE_TIME_POLICY = `UPDATE "secrets_policy_ SET last_update_check = $1, updated = current_timestamp WHERE secret_org = $2 AND secret_name = $3;` +const SECRET_INSERT_POLICY = `INSERT INTO "secrets_policy_ (secret_org, secret_name, policy_org, policy_name, secret_exists, last_update_check, partition) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING;` +const SECRET_UPDATE_TIME_POLICY = `UPDATE "secrets_policy_ SET last_update_check = $1, updated = current_timestamp, secret_exists = $2 WHERE secret_org = $3 AND secret_name = $4;` +const SECRET_EXISTS_UPDATE_TIME_POLICY = `UPDATE "secrets_policy_ SET last_update_check = $1, secret_exists = false WHERE secret_org = $2 AND secret_name = $3 AND secret_exists = $4;` const SECRET_DELETE_POLICY = `DELETE FROM "secrets_policy_ WHERE secret_org = $1 AND secret_name = $2 AND policy_org = $3 AND policy_name = $4;` -const SECRET_INSERT_PATTERN = `INSERT INTO "secrets_pattern_ (secret_org, secret_name, pattern_org, pattern_name, last_update_check, partition) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING;` -const SECRET_UPDATE_TIME_PATTERN = `UPDATE "secrets_pattern_ SET last_update_check = $1, updated = current_timestamp WHERE secret_org = $2 AND secret_name = $3;` +const SECRET_INSERT_PATTERN = `INSERT INTO "secrets_pattern_ (secret_org, secret_name, pattern_org, pattern_name, secret_exists, last_update_check, partition) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING;` +const SECRET_UPDATE_TIME_PATTERN = `UPDATE "secrets_pattern_ SET last_update_check = $1, updated = current_timestamp, secret_exists =$2 WHERE secret_org = $3 AND secret_name = $4;` +const SECRET_EXISTS_UPDATE_TIME_PATTERN = `UPDATE "secrets_pattern_ SET last_update_check = $1, secret_exists = false WHERE secret_org = $2 AND secret_name = $3 AND secret_exists = $4;` const SECRET_DELETE_PATTERN = `DELETE FROM "secrets_pattern_ WHERE secret_org = $1 AND secret_name = $2 AND pattern_org = $3 AND pattern_name = $4;` const SECRET_MOVE = `WITH moved_rows AS ( @@ -78,6 +82,9 @@ const SECRET_DISTINCT_NAMES_BY_PATTERN = `SELECT DISTINCT secret_org, secret_nam const SECRET_POLICIES_TO_UPDATE = `SELECT DISTINCT policy_org, policy_name FROM "secrets_policy_ WHERE secret_org = $1 AND secret_name = $2 AND last_update_check < $3;` const SECRET_PATTERNS_TO_UPDATE = `SELECT DISTINCT pattern_org, pattern_name FROM "secrets_pattern_ WHERE secret_org = $1 AND secret_name = $2 AND last_update_check < $3;` +const SECRET_POLICIES_TO_UPDATE_REMOVED_SECRET = `SELECT DISTINCT policy_org, policy_name FROM "secrets_policy_ WHERE secret_org = $1 AND secret_name = $2 AND (last_update_check < $3 OR secret_exists);` +const SECRET_PATTERNS_TO_UPDATE_REMOVED_SECRET = `SELECT DISTINCT pattern_org, pattern_name FROM "secrets_pattern_ WHERE secret_org = $1 AND secret_name = $2 AND (last_update_check < $3 OR secret_exists);` + const SECRET_DISTINCT_POLICIES = `SELECT DISTINCT policy_name FROM "secrets_policy_ WHERE policy_org = $1;` const SECRET_DISTINCT_PATTERNS = `SELECT DISTINCT pattern_name FROM "secrets_pattern_ WHERE pattern_org = $1;` @@ -156,16 +163,36 @@ func (db *AgbotPostgresqlDB) GetPatternsForUpdatedSecretQuery() string { return sql } +func (db *AgbotPostgresqlDB) GetPoliciesForRemovedSecretQuery() string { + sql := strings.Replace(SECRET_POLICIES_TO_UPDATE_REMOVED_SECRET, SECRET_TABLE_NAME_ROOT_POLICY, db.GetSecretPartitionTableNamePolicy(db.PrimaryPartition()), 1) + return sql +} + +func (db *AgbotPostgresqlDB) GetPatternsForRemovedSecretQuery() string { + sql := strings.Replace(SECRET_PATTERNS_TO_UPDATE_REMOVED_SECRET, SECRET_TABLE_NAME_ROOT_PATTERN, db.GetSecretPartitionTableNamePattern(db.PrimaryPartition()), 1) + return sql +} + func (db *AgbotPostgresqlDB) GetUpdateSecretUpdateTimeQueryPolicy() string { sql := strings.Replace(SECRET_UPDATE_TIME_POLICY, SECRET_TABLE_NAME_ROOT_POLICY, db.GetSecretPartitionTableNamePolicy(db.PrimaryPartition()), 1) return sql } +func (db *AgbotPostgresqlDB) GetUpdateSecretExistsUpdateTimeQueryPolicy() string { + sql := strings.Replace(SECRET_EXISTS_UPDATE_TIME_POLICY, SECRET_TABLE_NAME_ROOT_POLICY, db.GetSecretPartitionTableNamePolicy(db.PrimaryPartition()), 1) + return sql +} + func (db *AgbotPostgresqlDB) GetUpdateSecretUpdateTimeQueryPattern() string { sql := strings.Replace(SECRET_UPDATE_TIME_PATTERN, SECRET_TABLE_NAME_ROOT_PATTERN, db.GetSecretPartitionTableNamePattern(db.PrimaryPartition()), 1) return sql } +func (db *AgbotPostgresqlDB) GetUpdateSecretExistsUpdateTimeQueryPattern() string { + sql := strings.Replace(SECRET_EXISTS_UPDATE_TIME_PATTERN, SECRET_TABLE_NAME_ROOT_PATTERN, db.GetSecretPartitionTableNamePattern(db.PrimaryPartition()), 1) + return sql +} + func (db *AgbotPostgresqlDB) GetUniquePoliciesQuery() string { sql := strings.Replace(SECRET_DISTINCT_POLICIES, SECRET_TABLE_NAME_ROOT_POLICY, db.GetSecretPartitionTableNamePolicy(db.PrimaryPartition()), 1) return sql @@ -262,12 +289,18 @@ func (db *AgbotPostgresqlDB) getManagedSecretNames(sqlString, org, name string) } // Returns a list of unique org qualified secret names. -func (db *AgbotPostgresqlDB) GetPoliciesWithUpdatedSecrets(secretOrg, secretName string, lastUpdate int64) ([]string, error) { - return db.getUpdatedSecrets(db.GetPoliciesForUpdatedSecretQuery(), secretOrg, secretName, lastUpdate) +func (db *AgbotPostgresqlDB) GetPoliciesWithUpdatedSecrets(secretOrg, secretName string, lastUpdate int64, secretExists bool) ([]string, error) { + if secretExists { + return db.getUpdatedSecrets(db.GetPoliciesForUpdatedSecretQuery(), secretOrg, secretName, lastUpdate) + } + return db.getUpdatedSecrets(db.GetPoliciesForRemovedSecretQuery(), secretOrg, secretName, lastUpdate) } -func (db *AgbotPostgresqlDB) GetPatternsWithUpdatedSecrets(secretOrg, secretName string, lastUpdate int64) ([]string, error) { - return db.getUpdatedSecrets(db.GetPatternsForUpdatedSecretQuery(), secretOrg, secretName, lastUpdate) +func (db *AgbotPostgresqlDB) GetPatternsWithUpdatedSecrets(secretOrg, secretName string, lastUpdate int64, secretExists bool) ([]string, error) { + if secretExists { + return db.getUpdatedSecrets(db.GetPatternsForUpdatedSecretQuery(), secretOrg, secretName, lastUpdate) + } + return db.getUpdatedSecrets(db.GetPatternsForRemovedSecretQuery(), secretOrg, secretName, lastUpdate) } func (db *AgbotPostgresqlDB) getUpdatedSecrets(sql, org, name string, lastUpdate int64) ([]string, error) { @@ -299,14 +332,14 @@ func (db *AgbotPostgresqlDB) getUpdatedSecrets(sql, org, name string, lastUpdate return names, nil } -func (db *AgbotPostgresqlDB) SetSecretUpdate(secretOrg, secretName string, secretUpdateTime int64) error { +func (db *AgbotPostgresqlDB) SetSecretUpdate(secretOrg, secretName string, secretUpdateTime int64, secretExists bool) error { - err := db.setInternalSecretUpdate(db.GetUpdateSecretUpdateTimeQueryPolicy(), secretOrg, secretName, secretUpdateTime) + err := db.setInternalSecretUpdate(db.GetUpdateSecretUpdateTimeQueryPolicy(), secretOrg, secretName, secretUpdateTime, secretExists) if err != nil { return errors.New(fmt.Sprintf("error updating policy secret %s/%s: %v", secretOrg, secretName, err)) } - err = db.setInternalSecretUpdate(db.GetUpdateSecretUpdateTimeQueryPattern(), secretOrg, secretName, secretUpdateTime) + err = db.setInternalSecretUpdate(db.GetUpdateSecretUpdateTimeQueryPattern(), secretOrg, secretName, secretUpdateTime, secretExists) if err != nil { return errors.New(fmt.Sprintf("error updating pattern secret %s/%s: %v", secretOrg, secretName, err)) } @@ -314,9 +347,24 @@ func (db *AgbotPostgresqlDB) SetSecretUpdate(secretOrg, secretName string, secre return nil } -func (db *AgbotPostgresqlDB) setInternalSecretUpdate(sql, secretOrg, secretName string, secretUpdateTime int64) error { +func (db *AgbotPostgresqlDB) SetSecretExists(secretOrg, secretName string, secretUpdateTime int64) error { - updated, err := db.db.Exec(sql, secretUpdateTime, secretOrg, secretName) + err := db.setInternalSecretExistsUpdate(db.GetUpdateSecretExistsUpdateTimeQueryPolicy(), secretOrg, secretName, secretUpdateTime, true) + if err != nil { + return errors.New(fmt.Sprintf("error updating policy secret %s/%s: %v", secretOrg, secretName, err)) + } + + err = db.setInternalSecretExistsUpdate(db.GetUpdateSecretExistsUpdateTimeQueryPattern(), secretOrg, secretName, secretUpdateTime, true) + if err != nil { + return errors.New(fmt.Sprintf("error updating pattern secret %s/%s: %v", secretOrg, secretName, err)) + } + + return nil +} + +func (db *AgbotPostgresqlDB) setInternalSecretUpdate(sql, secretOrg, secretName string, secretUpdateTime int64, secretExists bool) error { + + updated, err := db.db.Exec(sql, secretUpdateTime, secretExists, secretOrg, secretName) if err != nil { return errors.New(fmt.Sprintf("error setting update time for %s/%s: %v", secretOrg, secretName, err)) } @@ -326,13 +374,32 @@ func (db *AgbotPostgresqlDB) setInternalSecretUpdate(sql, secretOrg, secretName if err == nil { glog.V(2).Infof("Succeeded setting update time in %v rows for %s/%s", rowsAffected, secretOrg, secretName) } else { - glog.V(2).Infof("Succeeded setting update time for %s/%s", secretOrg, secretName) + glog.V(2).Infof("Succeeded setting update time for %s/%s. %v", secretOrg, secretName, err) } return nil } +func (db *AgbotPostgresqlDB) setInternalSecretExistsUpdate(sql, secretOrg, secretName string, secretUpdateTime int64, secretExists bool) error { + + updated, err := db.db.Exec(sql, secretUpdateTime, secretOrg, secretName, secretExists) + if err != nil { + return errors.New(fmt.Sprintf("error setting update time for %s/%s: %v", secretOrg, secretName, err)) + } + + // Not all DB drivers support the rows affected function. + rowsAffected, err := updated.RowsAffected() + if err == nil { + glog.V(2).Infof("Succeeded setting update time in %v rows for %s/%s", rowsAffected, secretOrg, secretName) + } else { + glog.V(2).Infof("Succeeded setting update time for %s/%s", secretOrg, secretName) + } + + return nil + +} + func (db *AgbotPostgresqlDB) GetPoliciesInOrg(org string) ([]string, error) { return db.getDeploymentInOrg(db.GetUniquePoliciesQuery(), org) } @@ -411,11 +478,11 @@ func (db *AgbotPostgresqlDB) DeletePatternSecret(secretOrg, secretName, patternO return nil } -func (db *AgbotPostgresqlDB) AddManagedPolicySecret(secretOrg, secretName, policyOrg, policyName string, updateTime int64) error { +func (db *AgbotPostgresqlDB) AddManagedPolicySecret(secretOrg, secretName, policyOrg, policyName string, secretExists bool, updateTime int64) error { sql := strings.Replace(SECRET_INSERT_POLICY, SECRET_TABLE_NAME_ROOT_POLICY, db.GetSecretPartitionTableNamePolicy(db.PrimaryPartition()), 1) - if _, err := db.db.Exec(sql, secretOrg, secretName, policyOrg, policyName, updateTime, db.PrimaryPartition()); err != nil { + if _, err := db.db.Exec(sql, secretOrg, secretName, policyOrg, policyName, secretExists, updateTime, db.PrimaryPartition()); err != nil { return err } else { glog.V(2).Infof("Succeeded creating managed policy secret record") @@ -424,11 +491,11 @@ func (db *AgbotPostgresqlDB) AddManagedPolicySecret(secretOrg, secretName, polic return nil } -func (db *AgbotPostgresqlDB) AddManagedPatternSecret(secretOrg, secretName, patternOrg, patternName string, updateTime int64) error { +func (db *AgbotPostgresqlDB) AddManagedPatternSecret(secretOrg, secretName, patternOrg, patternName string, secretExists bool, updateTime int64) error { sql := strings.Replace(SECRET_INSERT_PATTERN, SECRET_TABLE_NAME_ROOT_PATTERN, db.GetSecretPartitionTableNamePattern(db.PrimaryPartition()), 1) - if _, err := db.db.Exec(sql, secretOrg, secretName, patternOrg, patternName, updateTime, db.PrimaryPartition()); err != nil { + if _, err := db.db.Exec(sql, secretOrg, secretName, patternOrg, patternName, secretExists, updateTime, db.PrimaryPartition()); err != nil { return err } else { glog.V(2).Infof("Succeeded creating managed pattern secret record") diff --git a/agreementbot/secret_updater.go b/agreementbot/secret_updater.go index 505ddad2e..170338c92 100644 --- a/agreementbot/secret_updater.go +++ b/agreementbot/secret_updater.go @@ -87,6 +87,7 @@ func (sm *SecretUpdateManager) CheckForUpdates(secretProvider secrets.AgbotSecre } glog.V(5).Infof(smlogString(fmt.Sprintf("Checking for changes to secret %s", fullSecretName))) + secretExists := true // All secrets that are referenced by a policy or pattern are in the secret update tables, but some of these secrets // might not exist yet. @@ -94,27 +95,37 @@ func (sm *SecretUpdateManager) CheckForUpdates(secretProvider secrets.AgbotSecre if err != nil { // For secrets that dont exist yet, just ignore them. glog.Warningf(smlogString(fmt.Sprintf("Error retrieving metadata for secret %s for user %s for node %s in org %s metadata, error: %v", secretName, secretUser, secretNode, secretOrg, err))) - continue + + secretExists = false } glog.V(5).Infof(smlogString(fmt.Sprintf("Secret %s metadata: %v", fullSecretName, secretMetadata))) // Get a list of policies that have a secret which has been updated. - policyNames, err := db.GetPoliciesWithUpdatedSecrets(secretOrg, exchange.GetId(fullSecretName), secretMetadata.UpdateTime) + policyNames, err := db.GetPoliciesWithUpdatedSecrets(secretOrg, exchange.GetId(fullSecretName), secretMetadata.UpdateTime, secretExists) if err != nil { glog.Errorf(smlogString(fmt.Sprintf("Error checking policies for updated secret %s", fullSecretName))) continue } + if !secretExists { + err := db.SetSecretExists(secretOrg, secretName, time.Now().Unix()) + glog.Errorf(smlogString(fmt.Sprintf("Error updating secret %s in database: %v", fullSecretName, err))) + } + // If there are policies returned, then it means that the policy references the secret and the secret has been updated. if len(policyNames) != 0 { - su := events.NewSecretUpdate(secretOrg, exchange.GetId(fullSecretName), secretMetadata.UpdateTime, policyNames, []string{}, secretNode) + updateTime := secretMetadata.UpdateTime + if updateTime == 0 { + updateTime = time.Now().Unix() + } + su := events.NewSecretUpdate(secretOrg, exchange.GetId(fullSecretName), updateTime, policyNames, []string{}, secretNode) secretUpdates.AddSecretUpdate(su) glog.V(5).Infof(smlogString(fmt.Sprintf("Policies affected by %s, %v Node: %s", fullSecretName, policyNames, secretNode))) } // Get a list of patterns that have a secret which has been updated. - patternNames, err := db.GetPatternsWithUpdatedSecrets(secretOrg, exchange.GetId(fullSecretName), secretMetadata.UpdateTime) + patternNames, err := db.GetPatternsWithUpdatedSecrets(secretOrg, exchange.GetId(fullSecretName), secretMetadata.UpdateTime, secretExists) if err != nil { glog.Errorf(smlogString(fmt.Sprintf("Error checking patterns for updated secret %s", fullSecretName))) continue @@ -122,7 +133,11 @@ func (sm *SecretUpdateManager) CheckForUpdates(secretProvider secrets.AgbotSecre // If there are patterns returned, then it means that the secret has been updated. if len(patternNames) != 0 { - su := events.NewSecretUpdate(secretOrg, exchange.GetId(fullSecretName), secretMetadata.UpdateTime, []string{}, patternNames, secretNode) + updateTime := secretMetadata.UpdateTime + if updateTime == 0 { + updateTime = time.Now().Unix() + } + su := events.NewSecretUpdate(secretOrg, exchange.GetId(fullSecretName), updateTime, []string{}, patternNames, secretNode) secretUpdates.AddSecretUpdate(su) glog.V(5).Infof(smlogString(fmt.Sprintf("Patterns affected by %s, %v Node: %v", fullSecretName, patternNames, secretNode))) } @@ -181,12 +196,15 @@ func (sm *SecretUpdateManager) UpdateNodePolicySecrets(org string, exchPolsMetad referencedSecrets[fmt.Sprintf("%s/%s", org, secretFullName)] = true } + secretExists := true + // Get the secret's metadata, if it exists sm, err := secretProvider.GetSecretMetadata(org, secretUser, secretNode, secretName) if err != nil { // The secret should be stored in the table even if it doesnt exist, so that if it is created later // changes to it will be recognized. glog.Warningf(smlogString(fmt.Sprintf("unable to retrieve metadata for %s %s, error: %v", org, secretFullName, err))) + secretExists = false //continue } else { secretLastUpdateTime = sm.UpdateTime @@ -195,7 +213,7 @@ func (sm *SecretUpdateManager) UpdateNodePolicySecrets(org string, exchPolsMetad glog.V(5).Infof(smlogString(fmt.Sprintf("storing managed secret %v %v from %v/%v", org, secretFullName, org, pName))) // Only secrets that have never been referenced before are added to the DB. DB rows that already exist will not be updated. - err = db.AddManagedPolicySecret(org, secretFullName, org, pName, secretLastUpdateTime) + err = db.AddManagedPolicySecret(org, secretFullName, org, pName, secretExists, secretLastUpdateTime) if err != nil { glog.Errorf(smlogString(fmt.Sprintf("unable to persist secret %v %v from %v/%v, error: %v", org, secretFullName, org, pName, err))) } @@ -269,13 +287,14 @@ func (sm *SecretUpdateManager) UpdateNodePatternSecrets(org string, exchPatsMeta referencedSecrets[fmt.Sprintf("%s/%s", org, secretFullName)] = true } + secretExists := true // Get the secret's metadata, if it exists sm, err := secretProvider.GetSecretMetadata(org, secretUser, secretNode, secretName) if err != nil { // The secret should be stored in the table even if it doesnt exist, so that if it is created later // changes to it will be recognized. glog.Warningf(smlogString(fmt.Sprintf("unable to retrieve metadata for %s %s, error: %v", org, secretFullName, err))) - //continue + secretExists = false } else { secretLastUpdateTime = sm.UpdateTime } @@ -283,7 +302,7 @@ func (sm *SecretUpdateManager) UpdateNodePatternSecrets(org string, exchPatsMeta glog.V(5).Infof(smlogString(fmt.Sprintf("storing managed secret %v %v from %v/%v", org, secretFullName, org, pName))) // Only secrets that have never been referenced before are added to the DB. DB rows that already exist will not be updated. - err = db.AddManagedPatternSecret(org, secretFullName, org, pName, secretLastUpdateTime) + err = db.AddManagedPatternSecret(org, secretFullName, org, pName, secretExists, secretLastUpdateTime) if err != nil { glog.Errorf(smlogString(fmt.Sprintf("unable to persist secret %v %v from %v/%v, error: %v", org, secretFullName, org, pName, err))) } @@ -366,13 +385,15 @@ func (sm *SecretUpdateManager) UpdatePolicies(org string, exchPolsMetadata map[s // Use now as the last update time for secrets that dont exist yet. secretLastUpdateTime := time.Now().Unix() + secretExists := true + // Get the secret's metadata, if it exists sm, err := secretProvider.GetSecretMetadata(org, secretUser, secretNode, secretName) if err != nil { // The secret should be stored in the table even if it doesnt exist, so that if it is created later // changes to it will be recognized. glog.Warningf(smlogString(fmt.Sprintf("unable to retrieve metadata for %s %s, error: %v", org, secretFullName, err))) - //continue + secretExists = false } else { secretLastUpdateTime = sm.UpdateTime } @@ -380,7 +401,7 @@ func (sm *SecretUpdateManager) UpdatePolicies(org string, exchPolsMetadata map[s glog.V(5).Infof(smlogString(fmt.Sprintf("storing managed secret %v %v from %v/%v", org, secretFullName, org, pName))) // Only secrets that have never been referenced before are added to the DB. DB rows that already exist will not be updated. - err = db.AddManagedPolicySecret(org, secretFullName, org, pName, secretLastUpdateTime) + err = db.AddManagedPolicySecret(org, secretFullName, org, pName, secretExists, secretLastUpdateTime) if err != nil { glog.Errorf(smlogString(fmt.Sprintf("unable to persist secret %v %v from %v/%v, error: %v", org, secretFullName, org, pName, err))) } @@ -472,13 +493,15 @@ func (sm *SecretUpdateManager) UpdatePatterns(org string, exchPatternMetadata ma for _, secretOrg := range sOrgs { referencedSecrets[fmt.Sprintf("%s/%s", secretOrg, secretFullName)] = true + secretExists := true + // Get the secret's metadata, if it exists sm, err := secretProvider.GetSecretMetadata(secretOrg, secretUser, secretNode, secretName) if err != nil { // The secret should be stored in the table even if it doesnt exist, so that if it is created later // changes to it will be recognized. glog.Warningf(smlogString(fmt.Sprintf("unable to retrieve metadata for %s %s, error: %v", org, secretFullName, err))) - //continue + secretExists = false } else { secretLastUpdateTime = sm.UpdateTime } @@ -486,7 +509,7 @@ func (sm *SecretUpdateManager) UpdatePatterns(org string, exchPatternMetadata ma glog.V(5).Infof(smlogString(fmt.Sprintf("storing managed secret %v %v from %v/%v", org, secretFullName, org, pName))) // Only secrets that have never been referenced before are added to the DB. DB rows that already exist will not be updated. - err = db.AddManagedPatternSecret(secretOrg, secretFullName, org, pName, secretLastUpdateTime) + err = db.AddManagedPatternSecret(secretOrg, secretFullName, org, pName, secretExists, secretLastUpdateTime) if err != nil { glog.Errorf(smlogString(fmt.Sprintf("unable to persist secret %v %v from %v/%v, error: %v", org, secretFullName, org, pName, err))) }