diff --git a/datasetIngestor/updateMetaData.go b/datasetIngestor/updateMetaData.go index 4b79f97..ce883fe 100644 --- a/datasetIngestor/updateMetaData.go +++ b/datasetIngestor/updateMetaData.go @@ -2,7 +2,7 @@ package datasetIngestor import ( "encoding/json" - "io/ioutil" + "io" "log" "net/http" "strings" @@ -11,6 +11,31 @@ import ( "github.com/fatih/color" ) +const ( + Classification = "classification" + AVLow = "AV=low" + AVMedium = "AV=medium" + INMedium = "IN=medium" + COLow = "CO=low" +) + +/* +getAVFromPolicy retrieves the AV (?) from a policy. + +Parameters: +- client: An HTTP client used to send requests. +- APIServer: The URL of the API server. +- user: A map containing user information. It should contain an "accessToken" key. +- owner: The owner of the policy. + +The function constructs a URL using the APIServer, owner, and user's access token, and sends a GET request to this URL. +If the response status code is 200, it reads the response body and unmarshals it into a slice of Policy structs. +If there are no policies available for the owner, it logs a warning and sets the level to "low". +If there are policies available, it sets the level to the TapeRedundancy of the first policy. + +Returns: +- level: The TapeRedundancy level of the first policy if available, otherwise "low". +*/ func getAVFromPolicy(client *http.Client, APIServer string, user map[string]string, owner string) (level string) { var myurl = APIServer + "/Policies?filter=%7B%22where%22%3A%7B%22ownerGroup%22%3A%22" + owner + "%22%7D%7D&access_token=" + user["accessToken"] resp, _ := client.Get(myurl) @@ -18,7 +43,7 @@ func getAVFromPolicy(client *http.Client, APIServer string, user map[string]stri level = "low" if resp.StatusCode == 200 { - body, _ := ioutil.ReadAll(resp.Body) + body, _ := io.ReadAll(resp.Body) type Policy struct { TapeRedundancy string AutoArchive bool @@ -36,91 +61,123 @@ func getAVFromPolicy(client *http.Client, APIServer string, user map[string]stri return level } +/* +UpdateMetaData updates the metadata of a dataset. + +Parameters: +- client: An HTTP client used to send requests. +- APIServer: The URL of the API server. +- user: A map containing user information. It should contain an "accessToken" key. +- originalMap: A map containing the original metadata. +- metaDataMap: A map containing the metadata to be updated. +- startTime: The start time of the dataset. +- endTime: The end time of the dataset. +- owner: The owner of the dataset. +- tapecopies: A pointer to an integer indicating the number of tape copies. + +The function updates the following fields in metaDataMap: +- "creationTime": If it's equal to DUMMY_TIME, it's replaced with startTime. +- "ownerGroup": If it's equal to DUMMY_OWNER, it's replaced with owner. +- "endTime": If the "type" field is "raw" and "endTime" is equal to DUMMY_TIME, it's replaced with endTime. +- "license": If it doesn't exist, it's set to "CC BY-SA 4.0". +- "isPublished": If it doesn't exist, it's set to false. +- "classification": If it doesn't exist, it's set to "IN=medium,AV=,CO=low". If tapecopies is 1 or 2, the "AV" part is replaced with "AV=low" or "AV=medium" respectively. + +The function logs a message each time it updates a field. If the "classification" field contains "AV=medium", it also logs a note that the dataset will be copied to two tape copies. + +The function does not return a value. +*/ func UpdateMetaData(client *http.Client, APIServer string, user map[string]string, originalMap map[string]string, metaDataMap map[string]interface{}, startTime time.Time, endTime time.Time, owner string, tapecopies *int) { - // add real creationTime if not yet existing - color.Set(color.FgGreen) - if metaDataMap["creationTime"] == DUMMY_TIME { - originalMap["creationTime"] = metaDataMap["creationTime"].(string) - metaDataMap["creationTime"] = startTime - log.Printf("creationTime field added: %v\n", metaDataMap["creationTime"]) + updateFieldIfDummy(metaDataMap, originalMap, "creationTime", DUMMY_TIME, startTime) + updateFieldIfDummy(metaDataMap, originalMap, "ownerGroup", DUMMY_OWNER, owner) + + if metaDataMap["type"] == "raw" { + updateFieldIfDummy(metaDataMap, originalMap, "endTime", DUMMY_TIME, endTime) } + + addFieldIfNotExists(metaDataMap, "license", "CC BY-SA 4.0") + addFieldIfNotExists(metaDataMap, "isPublished", false) + + updateClassificationField(client, APIServer, user, metaDataMap, tapecopies) +} - if metaDataMap["ownerGroup"] == DUMMY_OWNER { - originalMap["ownerGroup"] = metaDataMap["ownerGroup"].(string) - metaDataMap["ownerGroup"] = owner - log.Printf("ownerGroup field added from file info: %v\n", metaDataMap["ownerGroup"]) +func updateFieldIfDummy(metaDataMap map[string]interface{}, originalMap map[string]string, fieldName string, dummyValue interface{}, newValue interface{}) { + if metaDataMap[fieldName] == dummyValue { + originalMap[fieldName] = metaDataMap[fieldName].(string) + metaDataMap[fieldName] = newValue + log.Printf("%s field added: %v\n", fieldName, metaDataMap[fieldName]) } +} - if metaDataMap["type"] == "raw" { - if metaDataMap["endTime"] == DUMMY_TIME { - originalMap["endTime"] = metaDataMap["endTime"].(string) - metaDataMap["endTime"] = endTime - log.Printf("endTime field added: %v\n", metaDataMap["endTime"]) - } +func addFieldIfNotExists(metaDataMap map[string]interface{}, fieldName string, value interface{}) { + if _, ok := metaDataMap[fieldName]; !ok { + metaDataMap[fieldName] = value + log.Printf("%s field added: %v\n", fieldName, metaDataMap[fieldName]) } +} - // check other fields to be added - - if _, ok := metaDataMap["license"]; !ok { - metaDataMap["license"] = "CC BY-SA 4.0" - log.Printf("license field added: %v\n", metaDataMap["license"]) +func updateClassificationField(client *http.Client, APIServer string, user map[string]string, metaDataMap map[string]interface{}, tapecopies *int) { + if _, ok := metaDataMap[Classification]; !ok { + addDefaultClassification(client, APIServer, user, metaDataMap) } - if _, ok := metaDataMap["isPublished"]; !ok { - metaDataMap["isPublished"] = false - log.Printf("isPublished field added: %v\n", metaDataMap["isPublished"]) + + if *tapecopies == 1 || *tapecopies == 2 { + updateAVField(metaDataMap, tapecopies) + } + + if strings.Contains(metaDataMap[Classification].(string), AVMedium) { + logAVMediumMessage() } +} + +func addDefaultClassification(client *http.Client, APIServer string, user map[string]string, metaDataMap map[string]interface{}) { + metaDataMap[Classification] = INMedium + ",AV=" + getAVFromPolicy(client, APIServer, user, metaDataMap["ownerGroup"].(string)) + "," + COLow + log.Printf("classification field added: %v\n", metaDataMap[Classification]) +} - /* update classification depending on policy settings and tapecopies parameter - precedence order from highest to lowest: - - tapecopies parameter for AV setting - - classification defined in metadata.json - - policy setting (give warning if this is different from value calculated so far) - - */ - if _, ok := metaDataMap["classification"]; !ok { - // take default from policy settings - metaDataMap["classification"] = "IN=medium" + ",AV=" + getAVFromPolicy(client, APIServer, user, metaDataMap["ownerGroup"].(string)) + ",CO=low" - log.Printf("classification field added: %v\n", metaDataMap["classification"]) +func updateAVField(metaDataMap map[string]interface{}, tapecopies *int) { + av := getAVValue(tapecopies) + if _, ok := metaDataMap[Classification]; ok { + newresult := getUpdatedClassification(metaDataMap, av) + metaDataMap[Classification] = strings.Join(newresult, ",") + } else { + metaDataMap[Classification] = INMedium + "," + av + "," + COLow } + log.Printf("classification field adjusted: %s\n", metaDataMap[Classification]) +} - if *tapecopies == 1 || *tapecopies == 2 { - var av string - if *tapecopies == 1 { - av = "AV=low" - } - if *tapecopies == 2 { - av = "AV=medium" - } - var newresult []string - if val, ok := metaDataMap["classification"]; ok { - result := strings.Split(val.(string), ",") - // check for AV field and if existing override - found := false - for _, element := range result { - if element == "" { - continue - } - if strings.HasPrefix(element, "AV=") { - newresult = append(newresult, av) - found = true - } else { - newresult = append(newresult, element) - } - } - if !found { - newresult = append(newresult, av) - } - metaDataMap["classification"] = strings.Join(newresult, ",") +func getAVValue(tapecopies *int) string { + if *tapecopies == 1 { + return AVLow + } + return AVMedium +} + +func getUpdatedClassification(metaDataMap map[string]interface{}, av string) []string { + result := strings.Split(metaDataMap[Classification].(string), ",") + // check for AV field and if existing override it + newresult := make([]string, 0, len(result)) + found := false + for _, element := range result { + if element == "" { + continue + } + if strings.HasPrefix(element, "AV=") { + newresult = append(newresult, av) + found = true } else { - metaDataMap["classification"] = "IN=medium," + av + ",CO=low" + newresult = append(newresult, element) } - log.Printf("classification field adjusted: %s\n", metaDataMap["classification"]) } - - if strings.Contains(metaDataMap["classification"].(string), "AV=medium") { - color.Set(color.FgYellow) - log.Printf("Note: this dataset, if archived, will be copied to two tape copies") + if !found { + newresult = append(newresult, av) } + return newresult +} + +func logAVMediumMessage() { + color.Set(color.FgYellow) + log.Printf("Note: this dataset, if archived, will be copied to two tape copies") color.Unset() } diff --git a/datasetIngestor/updateMetaData_test.go b/datasetIngestor/updateMetaData_test.go new file mode 100644 index 0000000..945f9fd --- /dev/null +++ b/datasetIngestor/updateMetaData_test.go @@ -0,0 +1,240 @@ +package datasetIngestor + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + "reflect" +) + +func TestGetAVFromPolicy(t *testing.T) { + // Test case 1: No policies available + ts1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`[]`)) // empty policy list + })) + defer ts1.Close() + + client := ts1.Client() + + level := getAVFromPolicy(client, ts1.URL, map[string]string{"accessToken": "testToken"}, "testOwner") + + if level != "low" { + t.Errorf("Expected level to be 'low', got '%s'", level) + } + + // Test case 2: Policies available + ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`[{"TapeRedundancy": "medium", "AutoArchive": false}]`)) // policy list with one policy + })) + defer ts2.Close() + + client = ts2.Client() + + level = getAVFromPolicy(client, ts2.URL, map[string]string{"accessToken": "testToken"}, "testOwner") + + if level != "medium" { + t.Errorf("Expected level to be 'medium', got '%s'", level) + } +} + +// Check whether `UpdateMetaData` correctly updates the metaDataMap +func TestUpdateMetaData(t *testing.T) { + // Create a mock server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`[{"TapeRedundancy": "medium", "AutoArchive": false}]`)) // policy list with one policy + })) + defer ts.Close() + + // Create a test client + client := ts.Client() + + // Define test parameters + APIServer := ts.URL // Use the mock server's URL + + user := map[string]string{"accessToken": "testToken"} + originalMap := map[string]string{} + metaDataMap := map[string]interface{}{ + "creationTime": DUMMY_TIME, + "ownerGroup": DUMMY_OWNER, + "type": "raw", + "endTime": DUMMY_TIME, + } + startTime := time.Now() + endTime := startTime.Add(time.Hour) + owner := "testOwner" + tapecopies := new(int) + *tapecopies = 1 + + // Call the function + UpdateMetaData(client, APIServer, user, originalMap, metaDataMap, startTime, endTime, owner, tapecopies) + + // Check results + if metaDataMap["creationTime"] != startTime { + t.Errorf("Expected creationTime to be '%v', got '%v'", startTime, metaDataMap["creationTime"]) + } + if metaDataMap["ownerGroup"] != owner { + t.Errorf("Expected ownerGroup to be '%s', got '%s'", owner, metaDataMap["ownerGroup"]) + } + if metaDataMap["endTime"] != endTime { + t.Errorf("Expected endTime to be '%v', got '%v'", endTime, metaDataMap["endTime"]) + } + if metaDataMap["license"] != "CC BY-SA 4.0" { + t.Errorf("Expected license to be 'CC BY-SA 4.0', got '%s'", metaDataMap["license"]) + } + if metaDataMap["isPublished"] != false { + t.Errorf("Expected isPublished to be 'false', got '%v'", metaDataMap["isPublished"]) + } + if _, ok := metaDataMap["classification"]; !ok { + t.Errorf("Expected classification to be set, got '%v'", metaDataMap["classification"]) + } +} + +// Check if updateFieldIfDummy correctly updates the metaDataMap and originalMap when the field is a dummy value. +func TestUpdateFieldIfDummy(t *testing.T) { + // Define test parameters + metaDataMap := map[string]interface{}{ + "testField": "DUMMY", + } + originalMap := map[string]string{} + fieldName := "testField" + dummyValue := "DUMMY" + newValue := "newValue" + + // Call the function + updateFieldIfDummy(metaDataMap, originalMap, fieldName, dummyValue, newValue) + + // Check results + if metaDataMap[fieldName] != newValue { + t.Errorf("Expected %s to be '%v', got '%v'", fieldName, newValue, metaDataMap[fieldName]) + } + if originalMap[fieldName] != dummyValue { + t.Errorf("Expected original %s to be '%v', got '%v'", fieldName, dummyValue, originalMap[fieldName]) + } +} + +// Check if addFieldIfNotExists correctly adds a field to the metaDataMap if it does not already exist. +func TestAddFieldIfNotExists(t *testing.T) { + // Define test parameters + metaDataMap := map[string]interface{}{} + fieldName := "testField" + value := "testValue" + + // Call the function + addFieldIfNotExists(metaDataMap, fieldName, value) + + // Check results + if metaDataMap[fieldName] != value { + t.Errorf("Expected %s to be '%v', got '%v'", fieldName, value, metaDataMap[fieldName]) + } +} + +// Check if updateClassificationField correctly updates the classification field in the metaDataMap. +func TestUpdateClassificationField(t *testing.T) { + // Create a mock server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`[{"TapeRedundancy": "medium", "AutoArchive": false}]`)) // policy list with one policy + })) + defer ts.Close() + + // Create a test client + client := ts.Client() + + // Define test parameters + APIServer := ts.URL // Use the mock server's URL + user := map[string]string{"accessToken": "testToken"} + metaDataMap := map[string]interface{}{ + "ownerGroup": "testOwner", + } + tapecopies := new(int) + *tapecopies = 1 + + expectedValue1 := "IN=medium,AV=low,CO=low" + expectedValue2 := "IN=medium,AV=medium,CO=low" + + // Call the function + updateClassificationField(client, APIServer, user, metaDataMap, tapecopies) + + // Check results + if _, ok := metaDataMap["classification"]; !ok { + t.Errorf("Expected classification to be set, got '%v'", metaDataMap["classification"]) + } else if metaDataMap["classification"] != expectedValue1 { + t.Errorf("Expected classification to be '%v', got '%v'", expectedValue1, metaDataMap["classification"]) + } + + // Change tapecopies to 2 and call the function again + *tapecopies = 2 + updateClassificationField(client, APIServer, user, metaDataMap, tapecopies) + + // Check results + if _, ok := metaDataMap["classification"]; !ok { + t.Errorf("Expected classification to be set, got '%v'", metaDataMap["classification"]) + } else if metaDataMap["classification"] != expectedValue2 { + t.Errorf("Expected classification to be '%v', got '%v'", expectedValue2, metaDataMap["classification"]) + } +} + +// checks that the function correctly updates the "AV" field in the classification string +func TestGetUpdatedClassification(t *testing.T) { + metaDataMap := map[string]interface{}{ + "classification": "IN=medium,AV=low,CO=low", + } + av := "AV=medium" + + expected := []string{"IN=medium", "AV=medium", "CO=low"} + result := getUpdatedClassification(metaDataMap, av) + + if !reflect.DeepEqual(result, expected) { + t.Errorf("getUpdatedClassification() = %v; want %v", result, expected) + } +} + +func TestGetAVValue(t *testing.T) { + tapecopies1 := 1 + tapecopies2 := 2 + + expected1 := AVLow + expected2 := AVMedium + + result1 := getAVValue(&tapecopies1) + result2 := getAVValue(&tapecopies2) + + if result1 != expected1 { + t.Errorf("getAVValue() with tapecopies = 1 = %v; want %v", result1, expected1) + } + + if result2 != expected2 { + t.Errorf("getAVValue() with tapecopies = 2 = %v; want %v", result2, expected2) + } +} + +func TestUpdateAVField(t *testing.T) { + tapecopies1 := 1 + tapecopies2 := 2 + + metaDataMap1 := map[string]interface{}{ + "classification": "IN=medium,AV=medium,CO=low", + } + + metaDataMap2 := map[string]interface{}{ + "classification": "IN=medium,AV=low,CO=low", + } + + expected1 := "IN=medium,AV=low,CO=low" + expected2 := "IN=medium,AV=medium,CO=low" + + updateAVField(metaDataMap1, &tapecopies1) + updateAVField(metaDataMap2, &tapecopies2) + + if metaDataMap1["classification"] != expected1 { + t.Errorf("updateAVField() with tapecopies = 1 = %v; want %v", metaDataMap1["classification"], expected1) + } + + if metaDataMap2["classification"] != expected2 { + t.Errorf("updateAVField() with tapecopies = 2 = %v; want %v", metaDataMap2["classification"], expected2) + } +}