Skip to content

Commit

Permalink
checkmetadata: make internal functions available, add descriptions
Browse files Browse the repository at this point in the history
  • Loading branch information
@consolethinks
consolethinks committed Aug 23, 2024
1 parent ea9af2e commit b76fbc0
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 24 deletions.
43 changes: 22 additions & 21 deletions datasetIngestor/checkMetadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,41 +25,42 @@ const (
const unknown = "unknown"
const raw = "raw"

func CheckMetadata(client *http.Client, APIServer string, metadatafile string, user map[string]string, accessGroups []string) (metaDataMap map[string]interface{}, sourceFolder string, beamlineAccount bool, err error) {
metaDataMap, err = readMetadataFromFile(metadatafile)
// a combined function that reads and checks metadata, gathers missing metadata and returns the metadata map, source folder and beamline account check
func ReadAndCheckMetadata(client *http.Client, APIServer string, metadatafile string, user map[string]string, accessGroups []string) (metaDataMap map[string]interface{}, sourceFolder string, beamlineAccount bool, err error) {
metaDataMap, err = ReadMetadataFromFile(metadatafile)
if err != nil {
return nil, "", false, err
}

if keys := collectIllegalKeys(metaDataMap); len(keys) > 0 {
if keys := CollectIllegalKeys(metaDataMap); len(keys) > 0 {
return nil, "", false, errors.New(ErrIllegalKeys + ": \"" + strings.Join(keys, "\", \"") + "\"")
}

beamlineAccount, err = checkUserAndOwnerGroup(user, accessGroups, metaDataMap)
beamlineAccount, err = CheckUserAndOwnerGroup(user, accessGroups, metaDataMap)
if err != nil {
return nil, "", false, err
}

err = augmentMissingMetadata(user, metaDataMap, client, APIServer, accessGroups)
err = GatherMissingMetadata(user, metaDataMap, client, APIServer, accessGroups)
if err != nil {
return nil, "", false, err
}

err = checkMetadataValidity(client, APIServer, metaDataMap)
err = CheckMetadataValidity(client, APIServer, metaDataMap)
if err != nil {
return nil, "", false, err
}

sourceFolder, err = getSourceFolder(metaDataMap)
sourceFolder, err = GetSourceFolder(metaDataMap)
if err != nil {
return nil, "", false, err
}

return metaDataMap, sourceFolder, beamlineAccount, nil
}

// readMetadataFromFile reads the metadata from the file and unmarshals it into a map.
func readMetadataFromFile(metadatafile string) (map[string]interface{}, error) {
// ReadMetadataFromFile reads the metadata from the file and unmarshals it into a map.
func ReadMetadataFromFile(metadatafile string) (map[string]interface{}, error) {
b, err := os.ReadFile(metadatafile) // just pass the file name
if err != nil {
return nil, err
Expand All @@ -77,7 +78,7 @@ func readMetadataFromFile(metadatafile string) (map[string]interface{}, error) {
}

// collects keys with illegal characters
func collectIllegalKeys(metadata map[string]interface{}) []string {
func CollectIllegalKeys(metadata map[string]interface{}) []string {
stack := []map[string]interface{}{metadata}
keys := []string{}

Expand All @@ -86,7 +87,7 @@ func collectIllegalKeys(metadata map[string]interface{}) []string {
stack = stack[:len(stack)-1]

for key, value := range item {
if containsIllegalCharacters(key) {
if keyContainsIllegalCharacters(key) {
keys = append(keys, key)
}

Expand All @@ -107,7 +108,7 @@ func collectIllegalKeys(metadata map[string]interface{}) []string {
return keys
}

func containsIllegalCharacters(s string) bool {
func keyContainsIllegalCharacters(s string) bool {
// Check if the string contains periods, brackets, or other illegal characters
// You can adjust this condition based on your specific requirements
for _, char := range s {
Expand All @@ -118,8 +119,8 @@ func containsIllegalCharacters(s string) bool {
return false
}

// checkUserAndOwnerGroup checks the user and owner group and returns whether the user is a beamline account.
func checkUserAndOwnerGroup(user map[string]string, accessGroups []string, metaDataMap map[string]interface{}) (bool, error) {
// CheckUserAndOwnerGroup checks the user and owner group and returns whether the user is a beamline account.
func CheckUserAndOwnerGroup(user map[string]string, accessGroups []string, metaDataMap map[string]interface{}) (bool, error) {
if user["displayName"] == "ingestor" {
return false, nil
}
Expand Down Expand Up @@ -204,12 +205,12 @@ func getHost() string {
return hostname
}

// augmentMissingMetadata augments missing metadata fields.
func augmentMissingMetadata(user map[string]string, metaDataMap map[string]interface{}, client *http.Client, APIServer string, accessGroups []string) error {
// GatherMissingMetadata augments missing metadata fields.
func GatherMissingMetadata(user map[string]string, metaDataMap map[string]interface{}, client *http.Client, APIServer string, accessGroups []string) error {
color.Set(color.FgGreen)
defer color.Unset()

// optionally augment missing owner metadata
// optionally gather missing owner metadata
if _, ok := metaDataMap["owner"]; !ok {
metaDataMap["owner"] = user["displayName"]
//log.Printf("owner field added: %s", metaDataMap["owner"])
Expand Down Expand Up @@ -285,8 +286,8 @@ func addPrincipalInvestigatorFromProposal(user map[string]string, metaDataMap ma
return nil
}

// checkMetadataValidity checks the validity of the metadata by calling the appropriate API.
func checkMetadataValidity(client *http.Client, APIServer string, metaDataMap map[string]interface{}) error {
// CheckMetadataValidity checks the validity of the metadata by calling the appropriate API.
func CheckMetadataValidity(client *http.Client, APIServer string, metaDataMap map[string]interface{}) error {
dstype, ok := metaDataMap["type"].(string)
if !ok {
return fmt.Errorf("metadata type isn't a string")
Expand Down Expand Up @@ -370,8 +371,8 @@ func checkMetadataValidity(client *http.Client, APIServer string, metaDataMap ma
return nil
}

// getSourceFolder gets the source folder from the metadata.
func getSourceFolder(metaDataMap map[string]interface{}) (string, error) {
// GetSourceFolder gets the source folder from the metadata.
func GetSourceFolder(metaDataMap map[string]interface{}) (string, error) {
sourceFolder := ""
val, ok := metaDataMap["sourceFolder"]
if !ok {
Expand Down
6 changes: 3 additions & 3 deletions datasetIngestor/checkMetadata_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func TestCheckMetadata(t *testing.T) {
accessGroups := []string{"group1", "group2"}

// Call the function with mock parameters
metaDataMap, sourceFolder, beamlineAccount, err := CheckMetadata(client, APIServer, metadatafile1, user, accessGroups)
metaDataMap, sourceFolder, beamlineAccount, err := ReadAndCheckMetadata(client, APIServer, metadatafile1, user, accessGroups)
if err != nil {
t.Error("Error in CheckMetadata function: ", err)
}
Expand Down Expand Up @@ -101,7 +101,7 @@ func TestCheckMetadata(t *testing.T) {
}

// test with the second metadata file
metaDataMap2, sourceFolder2, beamlineAccount2, err := CheckMetadata(client, APIServer, metadatafile2, user, accessGroups)
metaDataMap2, sourceFolder2, beamlineAccount2, err := ReadAndCheckMetadata(client, APIServer, metadatafile2, user, accessGroups)
if err != nil {
t.Error("Error in CheckMetadata function: ", err)
}
Expand Down Expand Up @@ -154,7 +154,7 @@ func TestCheckMetadata_CrashCase(t *testing.T) {
accessGroups := []string{"group1", "group2"}

// Call the function that should return an error
_, _, _, err := CheckMetadata(client, APIServer, metadatafile3, user, accessGroups)
_, _, _, err := ReadAndCheckMetadata(client, APIServer, metadatafile3, user, accessGroups)

// Check that the function returned the expected error
if err == nil {
Expand Down

0 comments on commit b76fbc0

Please sign in to comment.