Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes to PREMIS event recording #31

Merged
merged 1 commit into from
Jul 10, 2024
Merged

Fixes to PREMIS event recording #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion internal/activities/add_premis_event.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ type AddPREMISEventParams struct {
PREMISFilePath string
Agent premis.Agent
Type string
Detail string
OutcomeDetail string
Failures []string
}

Expand All @@ -32,7 +34,9 @@ func (md *AddPREMISEventActivity) Execute(
return nil, err
}

eventSummary, err := premis.NewEventSummary(params.Type, params.Failures)
outcome := premis.EventOutcomeForFailures(params.Failures)

eventSummary, err := premis.NewEventSummary(params.Type, params.Detail, outcome, params.OutcomeDetail)
if err != nil {
return nil, err
}
Expand Down
39 changes: 12 additions & 27 deletions internal/activities/add_premis_event_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,41 +15,26 @@ import (
func TestAddPREMISEvent(t *testing.T) {
t.Parallel()

// Normally populated files (for execution expected to work).
ContentFilesNormal := fs.NewDir(t, "",
fs.WithDir("metadata"),
fs.WithDir("content",
fs.WithDir("content",
fs.WithDir("d_0000001",
fs.WithFile("00000001.jp2", ""),
fs.WithFile("00000001_PREMIS.xml", ""),
),
),
),
)
// Normal execution with no failures (for execution expected to work).
PREMISFilePathNormalNoFailures := fs.NewFile(t, "premis.xml",
fs.WithContent(premis.EmptyXML),
).Path()

PREMISFilePathNormal := ContentFilesNormal.Join("metadata", "premis.xml")
// Normal execution with failures (for execution expected to work).
PREMISFilePathNormalWithFailures := fs.NewFile(t, "premis.xml",
fs.WithContent(premis.EmptyXML),
).Path()

// No files (for execution expected to work).
// Creation of PREMIS file in existing directory (for execution expected to work).
ContentNoFiles := fs.NewDir(t, "",
fs.WithDir("metadata"),
fs.WithDir("content",
fs.WithDir("content",
fs.WithDir("d_0000001"),
),
),
)

PREMISFilePathNoFiles := ContentNoFiles.Join("metadata", "premis.xml")

// Non-existent paths (for execution expected to fail).
// Creation of PREMIS file in non-existing directory (for execution expected to fail).
ContentNonExistent := fs.NewDir(t, "",
fs.WithDir("metadata"),
fs.WithDir("content",
fs.WithDir("content",
fs.WithDir("d_0000001"),
),
),
)

PREMISFilePathNonExistent := ContentNonExistent.Join("metadata", "premis.xml")
Expand All @@ -72,7 +57,7 @@ func TestAddPREMISEvent(t *testing.T) {
{
name: "Add PREMIS event for normal content with no failures",
params: activities.AddPREMISEventParams{
PREMISFilePath: PREMISFilePathNormal,
PREMISFilePath: PREMISFilePathNormalNoFailures,
Agent: premis.AgentDefault(),
Type: "someActivity",
Failures: noFailures,
Expand All @@ -82,7 +67,7 @@ func TestAddPREMISEvent(t *testing.T) {
{
name: "Add PREMIS event for normal content with failures",
params: activities.AddPREMISEventParams{
PREMISFilePath: PREMISFilePathNormal,
PREMISFilePath: PREMISFilePathNormalWithFailures,
Agent: premis.AgentDefault(),
Type: "someActivity",
Failures: failures,
Expand Down
15 changes: 11 additions & 4 deletions internal/activities/validate_file_formats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,13 +99,20 @@

// Define PREMIS event.
eventSummary := premis.EventSummary{
Type: "validateFileFormats",
Detail: detail,
Outcome: outcome,
Type: "validation",
Detail: detail,
Outcome: outcome,
OutcomeDetail: "Format allowed",
}

// Get subpath within content.
subpath, err := filepath.Rel(params.ContentPath, p)
if err != nil {
return err

Check warning on line 111 in internal/activities/validate_file_formats.go

View check run for this annotation

Codecov / codecov/patch

internal/activities/validate_file_formats.go#L111 

Added line #L111 was not covered by tests
}

// Append PREMIS event to XML and write results.
originalName := premis.OriginalNameForSubpath(p)
originalName := premis.OriginalNameForSubpath(subpath)

doc, err := premis.ParseOrInitialize(params.PREMISFilePath)
if err != nil {
Expand Down
70 changes: 68 additions & 2 deletions internal/activities/validate_file_formats_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,72 @@ import (

const pngContent = "\x89PNG\r\n\x1a\n\x00\x00\x00\x0DIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x02\x00\x00\x00\x90\x77\x53\xDE\x00\x00\x00\x00IEND\xAE\x42\x60\x82"

const premisValidFormatsContent = `<?xml version="1.0" encoding="UTF-8"?>
<premis:premis xmlns:premis="http://www.loc.gov/premis/v3" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/premis/v3 https://www.loc.gov/standards/premis/premis.xsd" version="3.0">
<premis:object xsi:type="premis:file">
<premis:objectIdentifier>
<premis:objectIdentifierType>uuid</premis:objectIdentifierType>
<premis:objectIdentifierValue>c74a85b7-919b-409e-8209-9c7ebe0e7945</premis:objectIdentifierValue>
</premis:objectIdentifier>
<premis:objectCharacteristics>
<premis:format>
<premis:formatDesignation>
<premis:formatName/>
</premis:formatDesignation>
</premis:format>
</premis:objectCharacteristics>
<premis:originalName>data/objects/dir/file1.txt</premis:originalName>
</premis:object>
<premis:object xsi:type="premis:file">
<premis:objectIdentifier>
<premis:objectIdentifierType>uuid</premis:objectIdentifierType>
<premis:objectIdentifierValue>a74a85b7-919b-409e-8209-9c7ebe0e7945</premis:objectIdentifierValue>
</premis:objectIdentifier>
<premis:objectCharacteristics>
<premis:format>
<premis:formatDesignation>
<premis:formatName/>
</premis:formatDesignation>
</premis:format>
</premis:objectCharacteristics>
<premis:originalName>data/objects/file2.txt</premis:originalName>
</premis:object>
</premis:premis>
`

const premisInvalidFormatsContent = `<?xml version="1.0" encoding="UTF-8"?>
<premis:premis xmlns:premis="http://www.loc.gov/premis/v3" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/premis/v3 https://www.loc.gov/standards/premis/premis.xsd" version="3.0">
<premis:object xsi:type="premis:file">
<premis:objectIdentifier>
<premis:objectIdentifierType>uuid</premis:objectIdentifierType>
<premis:objectIdentifierValue>c74a85b7-919b-409e-8209-9c7ebe0e7945</premis:objectIdentifierValue>
</premis:objectIdentifier>
<premis:objectCharacteristics>
<premis:format>
<premis:formatDesignation>
<premis:formatName/>
</premis:formatDesignation>
</premis:format>
</premis:objectCharacteristics>
<premis:originalName>data/objects/dir/file1.png</premis:originalName>
</premis:object>
<premis:object xsi:type="premis:file">
<premis:objectIdentifier>
<premis:objectIdentifierType>uuid</premis:objectIdentifierType>
<premis:objectIdentifierValue>a74a85b7-919b-409e-8209-9c7ebe0e7945</premis:objectIdentifierValue>
</premis:objectIdentifier>
<premis:objectCharacteristics>
<premis:format>
<premis:formatDesignation>
<premis:formatName/>
</premis:formatDesignation>
</premis:format>
</premis:objectCharacteristics>
<premis:originalName>data/objects/file2.png</premis:originalName>
</premis:object>
</premis:premis>
`

func TestValidateFileFormats(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -42,7 +108,7 @@ func TestValidateFileFormats(t *testing.T) {
fs.WithFile("file2.txt", "content"),
).Path(),
PREMISFilePath: fs.NewFile(t, "premis.xml",
fs.WithContent(premis.EmptyXML),
fs.WithContent(premisValidFormatsContent),
).Path(),
Agent: premis.AgentDefault(),
},
Expand All @@ -53,7 +119,7 @@ func TestValidateFileFormats(t *testing.T) {
params: activities.ValidateFileFormatsParams{
ContentPath: invalidFormatsPath,
PREMISFilePath: fs.NewFile(t, "premis.xml",
fs.WithContent(premis.EmptyXML),
fs.WithContent(premisInvalidFormatsContent),
).Path(),
Agent: premis.AgentDefault(),
},
Expand Down
44 changes: 23 additions & 21 deletions internal/premis/premis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,10 @@
}

type EventSummary struct {
Type string
Detail string
Outcome string
Type string
Detail string
Outcome string
OutcomeDetail string
}

type Event struct {
Expand Down Expand Up @@ -128,30 +129,21 @@
}
}

func NewEventSummary(eventType string, failures []string) (EventSummary, error) {
detail, outcome := assembleEventDetailAndOutcome(failures)

func NewEventSummary(eventType string, detail string, outcome string, outcomeDetail string) (EventSummary, error) {
return EventSummary{
Type: eventType,
Detail: detail,
Outcome: outcome,
Type: eventType,
Detail: detail,
Outcome: outcome,
OutcomeDetail: outcomeDetail,
}, nil
}

func assembleEventDetailAndOutcome(failures []string) (string, string) {
detail := ""
outcome := "valid"

func EventOutcomeForFailures(failures []string) string {
if failures != nil {
// Add failure descriptions to PREMIS event detail.
for _, description := range failures {
detail = detail + description + "\n"
}

outcome = "invalid"
return "invalid"
}

return detail, outcome
return "valid"

Check warning on line 146 in internal/premis/premis.go

View check run for this annotation

Codecov / codecov/patch

internal/premis/premis.go#L146 

Added line #L146 was not covered by tests
}

func AppendObjectXML(doc *etree.Document, object Object) error {
Expand Down Expand Up @@ -182,7 +174,11 @@
return err
}

LinkEventToObject(objectEl, event)
if objectEl != nil {
LinkEventToObject(objectEl, event)
} else {
return fmt.Errorf("append event and link to object: object '%s' not found", originalName)

Check warning on line 180 in internal/premis/premis.go

View check run for this annotation

Codecov / codecov/patch

internal/premis/premis.go#L180 

Added line #L180 was not covered by tests
}

return nil
}
Expand Down Expand Up @@ -276,6 +272,12 @@
outcomeEl := outcomeInfoEl.CreateElement("premis:eventOutcome")
outcomeEl.CreateText(event.Summary.Outcome)

if event.Summary.OutcomeDetail != "" {
outcomeDetailEl := outcomeInfoEl.CreateElement("premis:eventOutcomeDetail")
outcomeDetailNoteEl := outcomeDetailEl.CreateElement("premis:eventOutcomeDetailNote")
outcomeDetailNoteEl.CreateText(event.Summary.OutcomeDetail)
}

addEventAgentIdentifierElement(eventEl, event)
}

Expand Down
15 changes: 9 additions & 6 deletions internal/premis/premis_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ const premisEventAddContent = `<?xml version="1.0" encoding="UTF-8"?>
<premis:eventIdentifierType>UUID</premis:eventIdentifierType>
<premis:eventIdentifierValue/>
</premis:eventIdentifier>
<premis:eventType>validateFiles</premis:eventType>
<premis:eventType>validation</premis:eventType>
<premis:eventDateTime/>
<premis:eventDetailInformation>
<premis:eventDetail>event detail</premis:eventDetail>
Expand Down Expand Up @@ -78,14 +78,16 @@ const premisObjectAndEventAddContent = `<?xml version="1.0" encoding="UTF-8"?>
<premis:eventIdentifierType>UUID</premis:eventIdentifierType>
<premis:eventIdentifierValue/>
</premis:eventIdentifier>
<premis:eventType>someEvent</premis:eventType>
<premis:eventType>validation</premis:eventType>
<premis:eventDateTime/>
<premis:eventDetailInformation>
<premis:eventDetail>some failure
</premis:eventDetail>
<premis:eventDetail>name=&quot;Validate SIP metadata&quot;</premis:eventDetail>
</premis:eventDetailInformation>
<premis:eventOutcomeInformation>
<premis:eventOutcome>invalid</premis:eventOutcome>
<premis:eventOutcomeDetail>
<premis:eventOutcomeDetailNote>Metadata validation successful</premis:eventOutcomeDetailNote>
</premis:eventOutcomeDetail>
</premis:eventOutcomeInformation>
<premis:linkingAgentIdentifier>
<premis:linkingAgentIdentifierType valueURI="http://id.loc.gov/vocabulary/identifiers/local">url</premis:linkingAgentIdentifierType>
Expand Down Expand Up @@ -138,7 +140,7 @@ func TestAppendPREMISEventXML(t *testing.T) {
assert.NilError(t, err)

err = premis.AppendEventXML(doc, premis.EventSummary{
Type: "validateFiles",
Type: "validation",
Detail: "event detail",
Outcome: "valid",
}, premis.AgentDefault())
Expand Down Expand Up @@ -243,9 +245,10 @@ func TestAppendPREMISEventAndLinkToObject(t *testing.T) {
// Define PREMIS event with failure.
var failures []string
failures = append(failures, "some failure")
outcome := premis.EventOutcomeForFailures(failures)

// Add PREMIS event to XML document.
eventSummary, err := premis.NewEventSummary("someEvent", failures)
eventSummary, err := premis.NewEventSummary("validation", "name=\"Validate SIP metadata\"", outcome, "Metadata validation successful")
assert.NilError(t, err)

doc := etree.NewDocument()
Expand Down
28 changes: 11 additions & 17 deletions internal/workflow/preprocessing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,14 +147,21 @@ func (w *PreprocessingWorkflow) Execute(
}

// Add PREMIS event noting validate structure result.
validateStructureOutcomeDetail := fmt.Sprintf(
"SIP structure identified: %s. SIP structure matches validation criteria.",
identifySIP.SIP.Type.String(),
)

var addPREMISEvent activities.AddPREMISEventResult
e = temporalsdk_workflow.ExecuteActivity(
withLocalActOpts(ctx),
activities.AddPREMISEventName,
&activities.AddPREMISEventParams{
PREMISFilePath: premisFilePath,
Agent: premis.AgentDefault(),
Type: "validateStructure",
Type: "validation",
Detail: "name=\"Validate SIP structure\"",
OutcomeDetail: validateStructureOutcomeDetail,
Failures: validateStructure.Failures,
},
).Get(ctx, &addPREMISEvent)
Expand Down Expand Up @@ -198,21 +205,6 @@ func (w *PreprocessingWorkflow) Execute(
}
result.addEvent(validateFileFormatsEvent)

// Add PREMIS event noting validate file formats result.
e = temporalsdk_workflow.ExecuteActivity(
withLocalActOpts(ctx),
activities.AddPREMISEventName,
&activities.AddPREMISEventParams{
PREMISFilePath: premisFilePath,
Agent: premis.AgentDefault(),
Type: "validateFileFormats",
Failures: validateFileFormats.Failures,
},
).Get(ctx, &addPREMISEvent)
if e != nil {
return nil, e
}

// Validate metadata.
validateMetadataEvent := newEvent(ctx, "Validate SIP metadata")
var validateMetadata activities.ValidateMetadataResult
Expand Down Expand Up @@ -257,7 +249,9 @@ func (w *PreprocessingWorkflow) Execute(
&activities.AddPREMISEventParams{
PREMISFilePath: premisFilePath,
Agent: premis.AgentDefault(),
Type: "validateMetadata",
Type: "validation",
Detail: "name=\"Validate SIP metadata\"",
OutcomeDetail: "Metadata validation successful",
Failures: validateMetadata.Failures,
},
).Get(ctx, &addPREMISEvent)
Expand Down
Loading
Loading