From 2b1c8e37df563edbce375b9c06e8d88669d7b301 Mon Sep 17 00:00:00 2001 From: "Ali R. Vahdati" Date: Wed, 24 Apr 2024 13:41:27 +0200 Subject: [PATCH 1/5] Add tests for getAVFromPolicy --- datasetIngestor/updateMetaData.go | 21 ++++++++++++-- datasetIngestor/updateMetaData_test.go | 39 ++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 datasetIngestor/updateMetaData_test.go diff --git a/datasetIngestor/updateMetaData.go b/datasetIngestor/updateMetaData.go index 4b79f97..7a24c52 100644 --- a/datasetIngestor/updateMetaData.go +++ b/datasetIngestor/updateMetaData.go @@ -2,7 +2,7 @@ package datasetIngestor import ( "encoding/json" - "io/ioutil" + "io" "log" "net/http" "strings" @@ -11,6 +11,23 @@ import ( "github.com/fatih/color" ) +/* +getAVFromPolicy retrieves the AV (?) from a policy. + +Parameters: +- client: An HTTP client used to send requests. +- APIServer: The URL of the API server. +- user: A map containing user information. It should contain an "accessToken" key. +- owner: The owner of the policy. + +The function constructs a URL using the APIServer, owner, and user's access token, and sends a GET request to this URL. +If the response status code is 200, it reads the response body and unmarshals it into a slice of Policy structs. +If there are no policies available for the owner, it logs a warning and sets the level to "low". +If there are policies available, it sets the level to the TapeRedundancy of the first policy. + +Returns: +- level: The TapeRedundancy level of the first policy if available, otherwise "low". +*/ func getAVFromPolicy(client *http.Client, APIServer string, user map[string]string, owner string) (level string) { var myurl = APIServer + "/Policies?filter=%7B%22where%22%3A%7B%22ownerGroup%22%3A%22" + owner + "%22%7D%7D&access_token=" + user["accessToken"] resp, _ := client.Get(myurl) @@ -18,7 +35,7 @@ func getAVFromPolicy(client *http.Client, APIServer string, user map[string]stri level = "low" if resp.StatusCode == 200 { - body, _ := ioutil.ReadAll(resp.Body) + body, _ := io.ReadAll(resp.Body) type Policy struct { TapeRedundancy string AutoArchive bool diff --git a/datasetIngestor/updateMetaData_test.go b/datasetIngestor/updateMetaData_test.go new file mode 100644 index 0000000..af2be98 --- /dev/null +++ b/datasetIngestor/updateMetaData_test.go @@ -0,0 +1,39 @@ +package datasetIngestor + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestGetAVFromPolicy(t *testing.T) { + // Test case 1: No policies available + ts1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`[]`)) // empty policy list + })) + defer ts1.Close() + + client := ts1.Client() + + level := getAVFromPolicy(client, ts1.URL, map[string]string{"accessToken": "testToken"}, "testOwner") + + if level != "low" { + t.Errorf("Expected level to be 'low', got '%s'", level) + } + + // Test case 2: Policies available + ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`[{"TapeRedundancy": "medium", "AutoArchive": false}]`)) // policy list with one policy + })) + defer ts2.Close() + + client = ts2.Client() + + level = getAVFromPolicy(client, ts2.URL, map[string]string{"accessToken": "testToken"}, "testOwner") + + if level != "medium" { + t.Errorf("Expected level to be 'medium', got '%s'", level) + } +} From aef0d04d612574a8bfd900534d7edf6b8737976b Mon Sep 17 00:00:00 2001 From: "Ali R. Vahdati" Date: Wed, 24 Apr 2024 13:52:28 +0200 Subject: [PATCH 2/5] Add docstring to UpdateMetaData and test it --- datasetIngestor/updateMetaData.go | 28 ++++++++++++- datasetIngestor/updateMetaData_test.go | 54 ++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/datasetIngestor/updateMetaData.go b/datasetIngestor/updateMetaData.go index 7a24c52..5d457c0 100644 --- a/datasetIngestor/updateMetaData.go +++ b/datasetIngestor/updateMetaData.go @@ -12,7 +12,7 @@ import ( ) /* -getAVFromPolicy retrieves the AV (?) from a policy. +getAVFromPolicy retrieves the AV (?) from a policy. Parameters: - client: An HTTP client used to send requests. @@ -53,6 +53,32 @@ func getAVFromPolicy(client *http.Client, APIServer string, user map[string]stri return level } +/* +UpdateMetaData updates the metadata of a dataset. + +Parameters: +- client: An HTTP client used to send requests. +- APIServer: The URL of the API server. +- user: A map containing user information. It should contain an "accessToken" key. +- originalMap: A map containing the original metadata. +- metaDataMap: A map containing the metadata to be updated. +- startTime: The start time of the dataset. +- endTime: The end time of the dataset. +- owner: The owner of the dataset. +- tapecopies: A pointer to an integer indicating the number of tape copies. + +The function updates the following fields in metaDataMap: +- "creationTime": If it's equal to DUMMY_TIME, it's replaced with startTime. +- "ownerGroup": If it's equal to DUMMY_OWNER, it's replaced with owner. +- "endTime": If the "type" field is "raw" and "endTime" is equal to DUMMY_TIME, it's replaced with endTime. +- "license": If it doesn't exist, it's set to "CC BY-SA 4.0". +- "isPublished": If it doesn't exist, it's set to false. +- "classification": If it doesn't exist, it's set to "IN=medium,AV=,CO=low". If tapecopies is 1 or 2, the "AV" part is replaced with "AV=low" or "AV=medium" respectively. + +The function logs a message each time it updates a field. If the "classification" field contains "AV=medium", it also logs a note that the dataset will be copied to two tape copies. + +The function does not return a value. +*/ func UpdateMetaData(client *http.Client, APIServer string, user map[string]string, originalMap map[string]string, metaDataMap map[string]interface{}, startTime time.Time, endTime time.Time, owner string, tapecopies *int) { // add real creationTime if not yet existing diff --git a/datasetIngestor/updateMetaData_test.go b/datasetIngestor/updateMetaData_test.go index af2be98..1e73da6 100644 --- a/datasetIngestor/updateMetaData_test.go +++ b/datasetIngestor/updateMetaData_test.go @@ -4,6 +4,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" ) func TestGetAVFromPolicy(t *testing.T) { @@ -37,3 +38,56 @@ func TestGetAVFromPolicy(t *testing.T) { t.Errorf("Expected level to be 'medium', got '%s'", level) } } + +// Check whether `UpdateMetaData` correctly updates the metaDataMap +func TestUpdateMetaData(t *testing.T) { + // Create a mock server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`[{"TapeRedundancy": "medium", "AutoArchive": false}]`)) // policy list with one policy + })) + defer ts.Close() + + // Create a test client + client := ts.Client() + + // Define test parameters + APIServer := ts.URL // Use the mock server's URL + + user := map[string]string{"accessToken": "testToken"} + originalMap := map[string]string{} + metaDataMap := map[string]interface{}{ + "creationTime": DUMMY_TIME, + "ownerGroup": DUMMY_OWNER, + "type": "raw", + "endTime": DUMMY_TIME, + } + startTime := time.Now() + endTime := startTime.Add(time.Hour) + owner := "testOwner" + tapecopies := new(int) + *tapecopies = 1 + + // Call the function + UpdateMetaData(client, APIServer, user, originalMap, metaDataMap, startTime, endTime, owner, tapecopies) + + // Check results + if metaDataMap["creationTime"] != startTime { + t.Errorf("Expected creationTime to be '%v', got '%v'", startTime, metaDataMap["creationTime"]) + } + if metaDataMap["ownerGroup"] != owner { + t.Errorf("Expected ownerGroup to be '%s', got '%s'", owner, metaDataMap["ownerGroup"]) + } + if metaDataMap["endTime"] != endTime { + t.Errorf("Expected endTime to be '%v', got '%v'", endTime, metaDataMap["endTime"]) + } + if metaDataMap["license"] != "CC BY-SA 4.0" { + t.Errorf("Expected license to be 'CC BY-SA 4.0', got '%s'", metaDataMap["license"]) + } + if metaDataMap["isPublished"] != false { + t.Errorf("Expected isPublished to be 'false', got '%v'", metaDataMap["isPublished"]) + } + if _, ok := metaDataMap["classification"]; !ok { + t.Errorf("Expected classification to be set, got '%v'", metaDataMap["classification"]) + } +} From 9fd0c05c043518d720e4cfef94956fa997d4364e Mon Sep 17 00:00:00 2001 From: "Ali R. Vahdati" Date: Wed, 24 Apr 2024 14:00:00 +0200 Subject: [PATCH 3/5] Refactor `UpdateMetaData` --- datasetIngestor/updateMetaData.go | 153 ++++++++++++++++++++++-------- 1 file changed, 116 insertions(+), 37 deletions(-) diff --git a/datasetIngestor/updateMetaData.go b/datasetIngestor/updateMetaData.go index 5d457c0..394e081 100644 --- a/datasetIngestor/updateMetaData.go +++ b/datasetIngestor/updateMetaData.go @@ -79,48 +79,127 @@ The function logs a message each time it updates a field. If the "classification The function does not return a value. */ -func UpdateMetaData(client *http.Client, APIServer string, user map[string]string, - originalMap map[string]string, metaDataMap map[string]interface{}, startTime time.Time, endTime time.Time, owner string, tapecopies *int) { - // add real creationTime if not yet existing - color.Set(color.FgGreen) - if metaDataMap["creationTime"] == DUMMY_TIME { - originalMap["creationTime"] = metaDataMap["creationTime"].(string) - metaDataMap["creationTime"] = startTime - log.Printf("creationTime field added: %v\n", metaDataMap["creationTime"]) - } - - if metaDataMap["ownerGroup"] == DUMMY_OWNER { - originalMap["ownerGroup"] = metaDataMap["ownerGroup"].(string) - metaDataMap["ownerGroup"] = owner - log.Printf("ownerGroup field added from file info: %v\n", metaDataMap["ownerGroup"]) - } - - if metaDataMap["type"] == "raw" { - if metaDataMap["endTime"] == DUMMY_TIME { - originalMap["endTime"] = metaDataMap["endTime"].(string) - metaDataMap["endTime"] = endTime - log.Printf("endTime field added: %v\n", metaDataMap["endTime"]) - } - } +// func UpdateMetaData(client *http.Client, APIServer string, user map[string]string, +// originalMap map[string]string, metaDataMap map[string]interface{}, startTime time.Time, endTime time.Time, owner string, tapecopies *int) { +// // add real creationTime if not yet existing +// color.Set(color.FgGreen) +// if metaDataMap["creationTime"] == DUMMY_TIME { +// originalMap["creationTime"] = metaDataMap["creationTime"].(string) +// metaDataMap["creationTime"] = startTime +// log.Printf("creationTime field added: %v\n", metaDataMap["creationTime"]) +// } + +// if metaDataMap["ownerGroup"] == DUMMY_OWNER { +// originalMap["ownerGroup"] = metaDataMap["ownerGroup"].(string) +// metaDataMap["ownerGroup"] = owner +// log.Printf("ownerGroup field added from file info: %v\n", metaDataMap["ownerGroup"]) +// } + +// if metaDataMap["type"] == "raw" { +// if metaDataMap["endTime"] == DUMMY_TIME { +// originalMap["endTime"] = metaDataMap["endTime"].(string) +// metaDataMap["endTime"] = endTime +// log.Printf("endTime field added: %v\n", metaDataMap["endTime"]) +// } +// } + +// // check other fields to be added + +// if _, ok := metaDataMap["license"]; !ok { +// metaDataMap["license"] = "CC BY-SA 4.0" +// log.Printf("license field added: %v\n", metaDataMap["license"]) +// } +// if _, ok := metaDataMap["isPublished"]; !ok { +// metaDataMap["isPublished"] = false +// log.Printf("isPublished field added: %v\n", metaDataMap["isPublished"]) +// } + +// /* update classification depending on policy settings and tapecopies parameter +// precedence order from highest to lowest: +// - tapecopies parameter for AV setting +// - classification defined in metadata.json +// - policy setting (give warning if this is different from value calculated so far) + +// */ +// if _, ok := metaDataMap["classification"]; !ok { +// // take default from policy settings +// metaDataMap["classification"] = "IN=medium" + ",AV=" + getAVFromPolicy(client, APIServer, user, metaDataMap["ownerGroup"].(string)) + ",CO=low" +// log.Printf("classification field added: %v\n", metaDataMap["classification"]) +// } + +// if *tapecopies == 1 || *tapecopies == 2 { +// var av string +// if *tapecopies == 1 { +// av = "AV=low" +// } +// if *tapecopies == 2 { +// av = "AV=medium" +// } +// var newresult []string +// if val, ok := metaDataMap["classification"]; ok { +// result := strings.Split(val.(string), ",") +// // check for AV field and if existing override +// found := false +// for _, element := range result { +// if element == "" { +// continue +// } +// if strings.HasPrefix(element, "AV=") { +// newresult = append(newresult, av) +// found = true +// } else { +// newresult = append(newresult, element) +// } +// } +// if !found { +// newresult = append(newresult, av) +// } +// metaDataMap["classification"] = strings.Join(newresult, ",") +// } else { +// metaDataMap["classification"] = "IN=medium," + av + ",CO=low" +// } +// log.Printf("classification field adjusted: %s\n", metaDataMap["classification"]) +// } + +// if strings.Contains(metaDataMap["classification"].(string), "AV=medium") { +// color.Set(color.FgYellow) +// log.Printf("Note: this dataset, if archived, will be copied to two tape copies") +// } +// color.Unset() +// } - // check other fields to be added +func UpdateMetaData(client *http.Client, APIServer string, user map[string]string, + originalMap map[string]string, metaDataMap map[string]interface{}, startTime time.Time, endTime time.Time, owner string, tapecopies *int) { + + updateFieldIfDummy(metaDataMap, originalMap, "creationTime", DUMMY_TIME, startTime) + updateFieldIfDummy(metaDataMap, originalMap, "ownerGroup", DUMMY_OWNER, owner) + + if metaDataMap["type"] == "raw" { + updateFieldIfDummy(metaDataMap, originalMap, "endTime", DUMMY_TIME, endTime) + } + + addFieldIfNotExists(metaDataMap, "license", "CC BY-SA 4.0") + addFieldIfNotExists(metaDataMap, "isPublished", false) + + updateClassificationField(client, APIServer, user, metaDataMap, tapecopies) +} - if _, ok := metaDataMap["license"]; !ok { - metaDataMap["license"] = "CC BY-SA 4.0" - log.Printf("license field added: %v\n", metaDataMap["license"]) - } - if _, ok := metaDataMap["isPublished"]; !ok { - metaDataMap["isPublished"] = false - log.Printf("isPublished field added: %v\n", metaDataMap["isPublished"]) +func updateFieldIfDummy(metaDataMap map[string]interface{}, originalMap map[string]string, fieldName string, dummyValue interface{}, newValue interface{}) { + if metaDataMap[fieldName] == dummyValue { + originalMap[fieldName] = metaDataMap[fieldName].(string) + metaDataMap[fieldName] = newValue + log.Printf("%s field added: %v\n", fieldName, metaDataMap[fieldName]) } +} - /* update classification depending on policy settings and tapecopies parameter - precedence order from highest to lowest: - - tapecopies parameter for AV setting - - classification defined in metadata.json - - policy setting (give warning if this is different from value calculated so far) +func addFieldIfNotExists(metaDataMap map[string]interface{}, fieldName string, value interface{}) { + if _, ok := metaDataMap[fieldName]; !ok { + metaDataMap[fieldName] = value + log.Printf("%s field added: %v\n", fieldName, metaDataMap[fieldName]) + } +} - */ +func updateClassificationField(client *http.Client, APIServer string, user map[string]string, metaDataMap map[string]interface{}, tapecopies *int) { if _, ok := metaDataMap["classification"]; !ok { // take default from policy settings metaDataMap["classification"] = "IN=medium" + ",AV=" + getAVFromPolicy(client, APIServer, user, metaDataMap["ownerGroup"].(string)) + ",CO=low" From d80157c1e571825bb5f447c0b3cc9704f21eb8e1 Mon Sep 17 00:00:00 2001 From: "Ali R. Vahdati" Date: Wed, 24 Apr 2024 14:17:57 +0200 Subject: [PATCH 4/5] Add more tests --- datasetIngestor/updateMetaData.go | 122 ++++--------------------- datasetIngestor/updateMetaData_test.go | 69 ++++++++++++++ 2 files changed, 85 insertions(+), 106 deletions(-) diff --git a/datasetIngestor/updateMetaData.go b/datasetIngestor/updateMetaData.go index 394e081..bd8a3c0 100644 --- a/datasetIngestor/updateMetaData.go +++ b/datasetIngestor/updateMetaData.go @@ -79,109 +79,19 @@ The function logs a message each time it updates a field. If the "classification The function does not return a value. */ -// func UpdateMetaData(client *http.Client, APIServer string, user map[string]string, -// originalMap map[string]string, metaDataMap map[string]interface{}, startTime time.Time, endTime time.Time, owner string, tapecopies *int) { -// // add real creationTime if not yet existing -// color.Set(color.FgGreen) -// if metaDataMap["creationTime"] == DUMMY_TIME { -// originalMap["creationTime"] = metaDataMap["creationTime"].(string) -// metaDataMap["creationTime"] = startTime -// log.Printf("creationTime field added: %v\n", metaDataMap["creationTime"]) -// } - -// if metaDataMap["ownerGroup"] == DUMMY_OWNER { -// originalMap["ownerGroup"] = metaDataMap["ownerGroup"].(string) -// metaDataMap["ownerGroup"] = owner -// log.Printf("ownerGroup field added from file info: %v\n", metaDataMap["ownerGroup"]) -// } - -// if metaDataMap["type"] == "raw" { -// if metaDataMap["endTime"] == DUMMY_TIME { -// originalMap["endTime"] = metaDataMap["endTime"].(string) -// metaDataMap["endTime"] = endTime -// log.Printf("endTime field added: %v\n", metaDataMap["endTime"]) -// } -// } - -// // check other fields to be added - -// if _, ok := metaDataMap["license"]; !ok { -// metaDataMap["license"] = "CC BY-SA 4.0" -// log.Printf("license field added: %v\n", metaDataMap["license"]) -// } -// if _, ok := metaDataMap["isPublished"]; !ok { -// metaDataMap["isPublished"] = false -// log.Printf("isPublished field added: %v\n", metaDataMap["isPublished"]) -// } - -// /* update classification depending on policy settings and tapecopies parameter -// precedence order from highest to lowest: -// - tapecopies parameter for AV setting -// - classification defined in metadata.json -// - policy setting (give warning if this is different from value calculated so far) - -// */ -// if _, ok := metaDataMap["classification"]; !ok { -// // take default from policy settings -// metaDataMap["classification"] = "IN=medium" + ",AV=" + getAVFromPolicy(client, APIServer, user, metaDataMap["ownerGroup"].(string)) + ",CO=low" -// log.Printf("classification field added: %v\n", metaDataMap["classification"]) -// } - -// if *tapecopies == 1 || *tapecopies == 2 { -// var av string -// if *tapecopies == 1 { -// av = "AV=low" -// } -// if *tapecopies == 2 { -// av = "AV=medium" -// } -// var newresult []string -// if val, ok := metaDataMap["classification"]; ok { -// result := strings.Split(val.(string), ",") -// // check for AV field and if existing override -// found := false -// for _, element := range result { -// if element == "" { -// continue -// } -// if strings.HasPrefix(element, "AV=") { -// newresult = append(newresult, av) -// found = true -// } else { -// newresult = append(newresult, element) -// } -// } -// if !found { -// newresult = append(newresult, av) -// } -// metaDataMap["classification"] = strings.Join(newresult, ",") -// } else { -// metaDataMap["classification"] = "IN=medium," + av + ",CO=low" -// } -// log.Printf("classification field adjusted: %s\n", metaDataMap["classification"]) -// } - -// if strings.Contains(metaDataMap["classification"].(string), "AV=medium") { -// color.Set(color.FgYellow) -// log.Printf("Note: this dataset, if archived, will be copied to two tape copies") -// } -// color.Unset() -// } - func UpdateMetaData(client *http.Client, APIServer string, user map[string]string, - originalMap map[string]string, metaDataMap map[string]interface{}, startTime time.Time, endTime time.Time, owner string, tapecopies *int) { - - updateFieldIfDummy(metaDataMap, originalMap, "creationTime", DUMMY_TIME, startTime) - updateFieldIfDummy(metaDataMap, originalMap, "ownerGroup", DUMMY_OWNER, owner) - - if metaDataMap["type"] == "raw" { - updateFieldIfDummy(metaDataMap, originalMap, "endTime", DUMMY_TIME, endTime) - } - - addFieldIfNotExists(metaDataMap, "license", "CC BY-SA 4.0") - addFieldIfNotExists(metaDataMap, "isPublished", false) - - updateClassificationField(client, APIServer, user, metaDataMap, tapecopies) + originalMap map[string]string, metaDataMap map[string]interface{}, startTime time.Time, endTime time.Time, owner string, tapecopies *int) { + updateFieldIfDummy(metaDataMap, originalMap, "creationTime", DUMMY_TIME, startTime) + updateFieldIfDummy(metaDataMap, originalMap, "ownerGroup", DUMMY_OWNER, owner) + + if metaDataMap["type"] == "raw" { + updateFieldIfDummy(metaDataMap, originalMap, "endTime", DUMMY_TIME, endTime) + } + + addFieldIfNotExists(metaDataMap, "license", "CC BY-SA 4.0") + addFieldIfNotExists(metaDataMap, "isPublished", false) + + updateClassificationField(client, APIServer, user, metaDataMap, tapecopies) } func updateFieldIfDummy(metaDataMap map[string]interface{}, originalMap map[string]string, fieldName string, dummyValue interface{}, newValue interface{}) { @@ -193,10 +103,10 @@ func updateFieldIfDummy(metaDataMap map[string]interface{}, originalMap map[stri } func addFieldIfNotExists(metaDataMap map[string]interface{}, fieldName string, value interface{}) { - if _, ok := metaDataMap[fieldName]; !ok { - metaDataMap[fieldName] = value - log.Printf("%s field added: %v\n", fieldName, metaDataMap[fieldName]) - } + if _, ok := metaDataMap[fieldName]; !ok { + metaDataMap[fieldName] = value + log.Printf("%s field added: %v\n", fieldName, metaDataMap[fieldName]) + } } func updateClassificationField(client *http.Client, APIServer string, user map[string]string, metaDataMap map[string]interface{}, tapecopies *int) { diff --git a/datasetIngestor/updateMetaData_test.go b/datasetIngestor/updateMetaData_test.go index 1e73da6..f562ff8 100644 --- a/datasetIngestor/updateMetaData_test.go +++ b/datasetIngestor/updateMetaData_test.go @@ -91,3 +91,72 @@ func TestUpdateMetaData(t *testing.T) { t.Errorf("Expected classification to be set, got '%v'", metaDataMap["classification"]) } } + +// Check if updateFieldIfDummy correctly updates the metaDataMap and originalMap when the field is a dummy value. +func TestUpdateFieldIfDummy(t *testing.T) { + // Define test parameters + metaDataMap := map[string]interface{}{ + "testField": "DUMMY", + } + originalMap := map[string]string{} + fieldName := "testField" + dummyValue := "DUMMY" + newValue := "newValue" + + // Call the function + updateFieldIfDummy(metaDataMap, originalMap, fieldName, dummyValue, newValue) + + // Check results + if metaDataMap[fieldName] != newValue { + t.Errorf("Expected %s to be '%v', got '%v'", fieldName, newValue, metaDataMap[fieldName]) + } + if originalMap[fieldName] != dummyValue { + t.Errorf("Expected original %s to be '%v', got '%v'", fieldName, dummyValue, originalMap[fieldName]) + } +} + +// Check if addFieldIfNotExists correctly adds a field to the metaDataMap if it does not already exist. +func TestAddFieldIfNotExists(t *testing.T) { + // Define test parameters + metaDataMap := map[string]interface{}{} + fieldName := "testField" + value := "testValue" + + // Call the function + addFieldIfNotExists(metaDataMap, fieldName, value) + + // Check results + if metaDataMap[fieldName] != value { + t.Errorf("Expected %s to be '%v', got '%v'", fieldName, value, metaDataMap[fieldName]) + } +} + +// Check if updateClassificationField correctly updates the classification field in the metaDataMap. +func TestUpdateClassificationField(t *testing.T) { + // Create a mock server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`[{"TapeRedundancy": "medium", "AutoArchive": false}]`)) // policy list with one policy + })) + defer ts.Close() + + // Create a test client + client := ts.Client() + + // Define test parameters + APIServer := ts.URL // Use the mock server's URL + user := map[string]string{"accessToken": "testToken"} + metaDataMap := map[string]interface{}{ + "ownerGroup": "testOwner", + } + tapecopies := new(int) + *tapecopies = 1 + + // Call the function + updateClassificationField(client, APIServer, user, metaDataMap, tapecopies) + + // Check results + if _, ok := metaDataMap["classification"]; !ok { + t.Errorf("Expected classification to be set, got '%v'", metaDataMap["classification"]) + } +} From a2f36335a3d3511d040a5dd067e55693248f0830 Mon Sep 17 00:00:00 2001 From: "Ali R. Vahdati" <3798865+kavir1698@users.noreply.github.com> Date: Tue, 21 May 2024 14:17:45 +0200 Subject: [PATCH 5/5] Refactor `updateClassificationField` and update tests --- datasetIngestor/updateMetaData.go | 101 +++++++++++++++---------- datasetIngestor/updateMetaData_test.go | 78 +++++++++++++++++++ 2 files changed, 141 insertions(+), 38 deletions(-) diff --git a/datasetIngestor/updateMetaData.go b/datasetIngestor/updateMetaData.go index bd8a3c0..ce883fe 100644 --- a/datasetIngestor/updateMetaData.go +++ b/datasetIngestor/updateMetaData.go @@ -11,6 +11,14 @@ import ( "github.com/fatih/color" ) +const ( + Classification = "classification" + AVLow = "AV=low" + AVMedium = "AV=medium" + INMedium = "IN=medium" + COLow = "CO=low" +) + /* getAVFromPolicy retrieves the AV (?) from a policy. @@ -110,49 +118,66 @@ func addFieldIfNotExists(metaDataMap map[string]interface{}, fieldName string, v } func updateClassificationField(client *http.Client, APIServer string, user map[string]string, metaDataMap map[string]interface{}, tapecopies *int) { - if _, ok := metaDataMap["classification"]; !ok { - // take default from policy settings - metaDataMap["classification"] = "IN=medium" + ",AV=" + getAVFromPolicy(client, APIServer, user, metaDataMap["ownerGroup"].(string)) + ",CO=low" - log.Printf("classification field added: %v\n", metaDataMap["classification"]) + if _, ok := metaDataMap[Classification]; !ok { + addDefaultClassification(client, APIServer, user, metaDataMap) } - + if *tapecopies == 1 || *tapecopies == 2 { - var av string - if *tapecopies == 1 { - av = "AV=low" - } - if *tapecopies == 2 { - av = "AV=medium" - } - var newresult []string - if val, ok := metaDataMap["classification"]; ok { - result := strings.Split(val.(string), ",") - // check for AV field and if existing override - found := false - for _, element := range result { - if element == "" { - continue - } - if strings.HasPrefix(element, "AV=") { - newresult = append(newresult, av) - found = true - } else { - newresult = append(newresult, element) - } - } - if !found { - newresult = append(newresult, av) - } - metaDataMap["classification"] = strings.Join(newresult, ",") + updateAVField(metaDataMap, tapecopies) + } + + if strings.Contains(metaDataMap[Classification].(string), AVMedium) { + logAVMediumMessage() + } +} + +func addDefaultClassification(client *http.Client, APIServer string, user map[string]string, metaDataMap map[string]interface{}) { + metaDataMap[Classification] = INMedium + ",AV=" + getAVFromPolicy(client, APIServer, user, metaDataMap["ownerGroup"].(string)) + "," + COLow + log.Printf("classification field added: %v\n", metaDataMap[Classification]) +} + +func updateAVField(metaDataMap map[string]interface{}, tapecopies *int) { + av := getAVValue(tapecopies) + if _, ok := metaDataMap[Classification]; ok { + newresult := getUpdatedClassification(metaDataMap, av) + metaDataMap[Classification] = strings.Join(newresult, ",") + } else { + metaDataMap[Classification] = INMedium + "," + av + "," + COLow + } + log.Printf("classification field adjusted: %s\n", metaDataMap[Classification]) +} + +func getAVValue(tapecopies *int) string { + if *tapecopies == 1 { + return AVLow + } + return AVMedium +} + +func getUpdatedClassification(metaDataMap map[string]interface{}, av string) []string { + result := strings.Split(metaDataMap[Classification].(string), ",") + // check for AV field and if existing override it + newresult := make([]string, 0, len(result)) + found := false + for _, element := range result { + if element == "" { + continue + } + if strings.HasPrefix(element, "AV=") { + newresult = append(newresult, av) + found = true } else { - metaDataMap["classification"] = "IN=medium," + av + ",CO=low" + newresult = append(newresult, element) } - log.Printf("classification field adjusted: %s\n", metaDataMap["classification"]) } - - if strings.Contains(metaDataMap["classification"].(string), "AV=medium") { - color.Set(color.FgYellow) - log.Printf("Note: this dataset, if archived, will be copied to two tape copies") + if !found { + newresult = append(newresult, av) } + return newresult +} + +func logAVMediumMessage() { + color.Set(color.FgYellow) + log.Printf("Note: this dataset, if archived, will be copied to two tape copies") color.Unset() } diff --git a/datasetIngestor/updateMetaData_test.go b/datasetIngestor/updateMetaData_test.go index f562ff8..945f9fd 100644 --- a/datasetIngestor/updateMetaData_test.go +++ b/datasetIngestor/updateMetaData_test.go @@ -5,6 +5,7 @@ import ( "net/http/httptest" "testing" "time" + "reflect" ) func TestGetAVFromPolicy(t *testing.T) { @@ -151,6 +152,9 @@ func TestUpdateClassificationField(t *testing.T) { } tapecopies := new(int) *tapecopies = 1 + + expectedValue1 := "IN=medium,AV=low,CO=low" + expectedValue2 := "IN=medium,AV=medium,CO=low" // Call the function updateClassificationField(client, APIServer, user, metaDataMap, tapecopies) @@ -158,5 +162,79 @@ func TestUpdateClassificationField(t *testing.T) { // Check results if _, ok := metaDataMap["classification"]; !ok { t.Errorf("Expected classification to be set, got '%v'", metaDataMap["classification"]) + } else if metaDataMap["classification"] != expectedValue1 { + t.Errorf("Expected classification to be '%v', got '%v'", expectedValue1, metaDataMap["classification"]) + } + + // Change tapecopies to 2 and call the function again + *tapecopies = 2 + updateClassificationField(client, APIServer, user, metaDataMap, tapecopies) + + // Check results + if _, ok := metaDataMap["classification"]; !ok { + t.Errorf("Expected classification to be set, got '%v'", metaDataMap["classification"]) + } else if metaDataMap["classification"] != expectedValue2 { + t.Errorf("Expected classification to be '%v', got '%v'", expectedValue2, metaDataMap["classification"]) + } +} + +// checks that the function correctly updates the "AV" field in the classification string +func TestGetUpdatedClassification(t *testing.T) { + metaDataMap := map[string]interface{}{ + "classification": "IN=medium,AV=low,CO=low", + } + av := "AV=medium" + + expected := []string{"IN=medium", "AV=medium", "CO=low"} + result := getUpdatedClassification(metaDataMap, av) + + if !reflect.DeepEqual(result, expected) { + t.Errorf("getUpdatedClassification() = %v; want %v", result, expected) + } +} + +func TestGetAVValue(t *testing.T) { + tapecopies1 := 1 + tapecopies2 := 2 + + expected1 := AVLow + expected2 := AVMedium + + result1 := getAVValue(&tapecopies1) + result2 := getAVValue(&tapecopies2) + + if result1 != expected1 { + t.Errorf("getAVValue() with tapecopies = 1 = %v; want %v", result1, expected1) + } + + if result2 != expected2 { + t.Errorf("getAVValue() with tapecopies = 2 = %v; want %v", result2, expected2) + } +} + +func TestUpdateAVField(t *testing.T) { + tapecopies1 := 1 + tapecopies2 := 2 + + metaDataMap1 := map[string]interface{}{ + "classification": "IN=medium,AV=medium,CO=low", + } + + metaDataMap2 := map[string]interface{}{ + "classification": "IN=medium,AV=low,CO=low", + } + + expected1 := "IN=medium,AV=low,CO=low" + expected2 := "IN=medium,AV=medium,CO=low" + + updateAVField(metaDataMap1, &tapecopies1) + updateAVField(metaDataMap2, &tapecopies2) + + if metaDataMap1["classification"] != expected1 { + t.Errorf("updateAVField() with tapecopies = 1 = %v; want %v", metaDataMap1["classification"], expected1) + } + + if metaDataMap2["classification"] != expected2 { + t.Errorf("updateAVField() with tapecopies = 2 = %v; want %v", metaDataMap2["classification"], expected2) } }