From 17bbf51deef536401145287556eb3d59a0f551e9 Mon Sep 17 00:00:00 2001 From: Ali Vahdati <3798865+kavir1698@users.noreply.github.com> Date: Wed, 27 Mar 2024 10:49:07 +0100 Subject: [PATCH] Add test for checkMetadata function (#4) * Add test - not passing * Add comments and testdata dir * Make the tests pass. * Add more tests * Mark a line for future improvement --- datasetIngestor/checkMetadata.go | 32 +++-- datasetIngestor/checkMetadata_test.go | 118 +++++++++++++++++++ datasetIngestor/export_test.go | 4 + datasetIngestor/testdata/metadata-short.json | 8 ++ datasetIngestor/testdata/metadata.json | 20 ++++ go.mod | 1 + go.sum | 3 +- 7 files changed, 175 insertions(+), 11 deletions(-) create mode 100644 datasetIngestor/checkMetadata_test.go create mode 100644 datasetIngestor/export_test.go create mode 100644 datasetIngestor/testdata/metadata-short.json create mode 100644 datasetIngestor/testdata/metadata.json diff --git a/datasetIngestor/checkMetadata.go b/datasetIngestor/checkMetadata.go index d9fc553..ff2da94 100644 --- a/datasetIngestor/checkMetadata.go +++ b/datasetIngestor/checkMetadata.go @@ -18,7 +18,10 @@ import ( const DUMMY_TIME = "2300-01-01T11:11:11.000Z" const DUMMY_OWNER = "x12345" +// getHost is a function that attempts to retrieve and return the fully qualified domain name (FQDN) of the current host. +// If it encounters any error during the process, it gracefully falls back to returning the simple hostname or "unknown". func getHost() string { + // Try to get the hostname of the current machine. hostname, err := os.Hostname() if err != nil { return "unknown" @@ -46,27 +49,34 @@ func getHost() string { return hostname } -func CheckMetadata(client *http.Client, APIServer string, metadatafile string, user map[string]string, - accessGroups []string) (metaDataMap map[string]interface{}, sourceFolder string, beamlineAccount bool) { +// CheckMetadata is a function that validates and augments metadata for a dataset. +// It takes an HTTP client, an API server URL, a metadata file path, a user map, and a list of access groups as input. +// It returns a map of metadata, a source folder string, and a boolean indicating whether the dataset belongs to a beamline account. +func CheckMetadata(client *http.Client, APIServer string, metadatafile string, user map[string]string, accessGroups []string) (metaDataMap map[string]interface{}, sourceFolder string, beamlineAccount bool) { - // read full meta data + // Read the full metadata from the file. b, err := ioutil.ReadFile(metadatafile) // just pass the file name if err != nil { log.Fatal(err) } - var metadataObj interface{} + // Unmarshal the JSON metadata into an interface{} object. + var metadataObj interface{} // Using interface{} allows metadataObj to hold any type of data, since it has no defined methods. err = json.Unmarshal(b, &metadataObj) if err != nil { log.Fatal(err) } - // use type assertion to access f's underlying map - metaDataMap = metadataObj.(map[string]interface{}) + + // Use type assertion to convert the interface{} object to a map[string]interface{}. + metaDataMap = metadataObj.(map[string]interface{}) // `.(` is type assertion: a way to extract the underlying value of an interface and check whether it's of a specific type. beamlineAccount = false + // If the user is not the ingestor, check whether any of the accessGroups equal the ownerGroup. Otherwise, check for beamline-specific accounts. if user["displayName"] != "ingestor" { - if ownerGroup, ok := metaDataMap["ownerGroup"]; ok { + // Check if the metadata contains the "ownerGroup" key. + if ownerGroup, ok := metaDataMap["ownerGroup"]; ok { // type assertion with a comma-ok idiom validOwner := false + // Iterate over accessGroups to validate the owner group. for _, b := range accessGroups { if b == ownerGroup { validOwner = true @@ -76,13 +86,15 @@ func CheckMetadata(client *http.Client, APIServer string, metadatafile string, u if validOwner { log.Printf("OwnerGroup information %s verified successfully.\n", ownerGroup) } else { - // check for beamline specific account if raw data + // If the owner group is not valid, check for beamline-specific accounts. if creationLocation, ok := metaDataMap["creationLocation"]; ok { + // Split the creationLocation string to extract beamline-specific information. parts := strings.Split(creationLocation.(string), "/") expectedAccount := "" if len(parts) == 4 { expectedAccount = strings.ToLower(parts[2]) + strings.ToLower(parts[3]) } + // If the user matches the expected beamline account, grant ingest access. if user["displayName"] == expectedAccount { log.Printf("Beamline specific dataset %s - ingest granted.\n", expectedAccount) beamlineAccount = true @@ -91,7 +103,7 @@ func CheckMetadata(client *http.Client, APIServer string, metadatafile string, u } } else { // for other data just check user name - // this is a quick and dirty test. Should be replaced by test for "globalaccess" role + // this is a quick and dirty test. Should be replaced by test for "globalaccess" role. TODO // facilities: ["SLS", "SINQ", "SWISSFEL", "SmuS"], u := user["displayName"] if strings.HasPrefix(u, "sls") || @@ -131,7 +143,7 @@ func CheckMetadata(client *http.Client, APIServer string, metadatafile string, u log.Printf("sourceFolderHost field added: %s", metaDataMap["sourceFolderHost"]) } } - // far raw data add PI if missing + // for raw data add PI if missing if val, ok := metaDataMap["type"]; ok { dstype := val.(string) if dstype == "raw" { diff --git a/datasetIngestor/checkMetadata_test.go b/datasetIngestor/checkMetadata_test.go new file mode 100644 index 0000000..344b47e --- /dev/null +++ b/datasetIngestor/checkMetadata_test.go @@ -0,0 +1,118 @@ +package datasetIngestor + +import ( + "net/http" + "testing" + "time" + "reflect" +) + +func TestGetHost(t *testing.T) { + // Call the function under test. + host := GetHost() + + // fail the test and report an error if the returned hostname is an empty string. + if len(host) == 0 { + t.Errorf("getHost() returned an empty string") + } + + // fail the test and report an error if the returned hostname is "unknown". + if host == "unknown" { + t.Errorf("getHost() was unable to get the hostname") + } +} + +func TestCheckMetadata(t *testing.T) { + // Define mock parameters for the function + var TEST_API_SERVER string = "https://dacat-qa.psi.ch/api/v3" // TODO: Test Improvement. Change this to a mock server. At the moment, tests will fail if we change this to a mock server. + var APIServer = TEST_API_SERVER + var metadatafile1 = "testdata/metadata.json" + var metadatafile2 = "testdata/metadata-short.json" + + // Mock HTTP client + client := &http.Client{ + Timeout: 5 * time.Second, // Set a timeout for requests + Transport: &http.Transport{ + // Customize the transport settings if needed (e.g., proxy, TLS config) + // For a dummy client, default settings are usually sufficient + }, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + // Customize how redirects are handled if needed + // For a dummy client, default behavior is usually sufficient + return http.ErrUseLastResponse // Use the last response for redirects + }, + } + + // Mock user map + user := map[string]string{ + "displayName": "csaxsswissfel", + "mail": "testuser@example.com", + } + + // Mock access groups + accessGroups := []string{"p17880", "p17301"} + + // Call the function with mock parameters + metaDataMap, sourceFolder, beamlineAccount := CheckMetadata(client, APIServer, metadatafile1, user, accessGroups) + + // Add assertions here based on the expected behavior of the function + if len(metaDataMap) == 0 { + t.Error("Expected non-empty metadata map") + } + if sourceFolder == "" { + t.Error("Expected non-empty source folder") + } + if sourceFolder != "/usr/share/gnome" { + t.Error("sourceFolder should be '/usr/share/gnome'") + } + if reflect.TypeOf(beamlineAccount).Kind() != reflect.Bool { + t.Error("Expected beamlineAccount to be boolean") + } + if beamlineAccount != false { + t.Error("Expected beamlineAccount to be false") + } + if _, ok := metaDataMap["ownerEmail"]; !ok { + t.Error("metaDataMap missing required key 'ownerEmail'") + } + if _, ok := metaDataMap["principalInvestigator"]; !ok { + t.Error("metaDataMap missing required key 'principalInvestigator'") + } + if _, ok := metaDataMap["scientificMetadata"]; !ok { + t.Error("metaDataMap missing required key 'scientificMetadata'") + } + scientificMetadata, ok := metaDataMap["scientificMetadata"].([]interface{}) + if ok { + firstEntry := scientificMetadata[0].(map[string]interface{}) + sample, ok := firstEntry["sample"].(map[string]interface{}) + if ok { + if _, ok := sample["name"]; !ok { + t.Error("Sample is missing 'name' field") + } + if _, ok := sample["description"]; !ok { + t.Error("Sample is missing 'description' field") + } + } + } else { + t.Error("scientificMetadata is not a list") + } + + // test with the second metadata file + metaDataMap2, sourceFolder2, beamlineAccount2 := CheckMetadata(client, APIServer, metadatafile2, user, accessGroups) + + // Add assertions here based on the expected behavior of the function + if len(metaDataMap2) == 0 { + t.Error("Expected non-empty metadata map") + } + if sourceFolder2 == "" { + t.Error("Expected non-empty source folder") + } + if sourceFolder2 != "/tmp/gnome" { + t.Error("sourceFolder should be '/tmp/gnome'") + } + if reflect.TypeOf(beamlineAccount2).Kind() != reflect.Bool { + t.Error("Expected beamlineAccount to be boolean") + } + if beamlineAccount2 != false { + t.Error("Expected beamlineAccount to be false") + } +} diff --git a/datasetIngestor/export_test.go b/datasetIngestor/export_test.go new file mode 100644 index 0000000..0b97c80 --- /dev/null +++ b/datasetIngestor/export_test.go @@ -0,0 +1,4 @@ +// This file exports internal functions for testing purposes. Since the name of the file ends with "_test.go", it will not be included in the final build of the application. +package datasetIngestor + +var GetHost = getHost diff --git a/datasetIngestor/testdata/metadata-short.json b/datasetIngestor/testdata/metadata-short.json new file mode 100644 index 0000000..06b802c --- /dev/null +++ b/datasetIngestor/testdata/metadata-short.json @@ -0,0 +1,8 @@ +{ + "principalInvestigator":"egon.meier@psi.ch", + "creationLocation":"/PSI/SLS/CSAXS/SWISSFEL", + "sourceFolder": "/tmp/gnome", + "owner": "Andreas Menzel", + "type": "raw", + "ownerGroup": "p17880" +} diff --git a/datasetIngestor/testdata/metadata.json b/datasetIngestor/testdata/metadata.json new file mode 100644 index 0000000..100db21 --- /dev/null +++ b/datasetIngestor/testdata/metadata.json @@ -0,0 +1,20 @@ +{ + "creationLocation": "/PSI/SLS/CSAXS/SWISSFEL", + "datasetName": "CMakeCache", + "description": "", + "owner": "Ana Diaz", + "ownerEmail": "ana.diaz@psi.ch", + "ownerGroup": "p17301", + "principalInvestigator": "ana.diaz@psi.ch", + "scientificMetadata": [ + { + "sample": { + "description": "", + "name": "", + "principalInvestigator": "" + } + } + ], + "sourceFolder": "/usr/share/gnome", + "type": "raw" +} diff --git a/go.mod b/go.mod index 61040a7..3eeddb8 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( require ( github.com/creack/pty v1.1.17 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/mattn/go-colorable v0.1.9 // indirect github.com/mattn/go-isatty v0.0.14 // indirect golang.org/x/sys v0.2.0 // indirect diff --git a/go.sum b/go.sum index 00500b7..4887fa9 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,9 @@ github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63n github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=