Skip to content

Commit

Permalink
Fixes to PREMIS event recording (#19)
Browse files Browse the repository at this point in the history 
* Removed redundant PREMIS event for validate file formats
* Correct use of improper PREMIS event types
* Allowed PREMIS evebt details and PREMIS event outcome detail to be
  specified rather than generated
* Add PREMIS event outcome detail specification to PREMIS events in
  workflow and validate file formats activity
* Generate error when attempting to add a PREMIS event to a
  non-existent PREMIS object
* Fixed issue with validate formats activity
* Removed unneeded code in tests of add PREMIS event activity
  • Loading branch information
@mcantelon
mcantelon committed Jul 10, 2024
1 parent 84e8e73 commit b736af4
Show file tree
Hide file tree
Showing 8 changed files with 145 additions and 92 deletions.
6 changes: 5 additions & 1 deletion internal/activities/add_premis_event.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ type AddPREMISEventParams struct {
PREMISFilePath string
Agent premis.Agent
Type string
Detail string
OutcomeDetail string
Failures []string
}

Expand All @@ -32,7 +34,9 @@ func (md *AddPREMISEventActivity) Execute(
return nil, err
}

eventSummary, err := premis.NewEventSummary(params.Type, params.Failures)
outcome := premis.EventOutcomeForFailures(params.Failures)

eventSummary, err := premis.NewEventSummary(params.Type, params.Detail, outcome, params.OutcomeDetail)
if err != nil {
return nil, err
}
Expand Down
39 changes: 12 additions & 27 deletions internal/activities/add_premis_event_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,41 +15,26 @@ import (
func TestAddPREMISEvent(t *testing.T) {
t.Parallel()

// Normally populated files (for execution expected to work).
ContentFilesNormal := fs.NewDir(t, "",
fs.WithDir("metadata"),
fs.WithDir("content",
fs.WithDir("content",
fs.WithDir("d_0000001",
fs.WithFile("00000001.jp2", ""),
fs.WithFile("00000001_PREMIS.xml", ""),
),
),
),
)
// Normal execution with no failures (for execution expected to work).
PREMISFilePathNormalNoFailures := fs.NewFile(t, "premis.xml",
fs.WithContent(premis.EmptyXML),
).Path()

PREMISFilePathNormal := ContentFilesNormal.Join("metadata", "premis.xml")
// Normal execution with failures (for execution expected to work).
PREMISFilePathNormalWithFailures := fs.NewFile(t, "premis.xml",
fs.WithContent(premis.EmptyXML),
).Path()

// No files (for execution expected to work).
// Creation of PREMIS file in existing directory (for execution expected to work).
ContentNoFiles := fs.NewDir(t, "",
fs.WithDir("metadata"),
fs.WithDir("content",
fs.WithDir("content",
fs.WithDir("d_0000001"),
),
),
)

PREMISFilePathNoFiles := ContentNoFiles.Join("metadata", "premis.xml")

// Non-existent paths (for execution expected to fail).
// Creation of PREMIS file in non-existing directory (for execution expected to fail).
ContentNonExistent := fs.NewDir(t, "",
fs.WithDir("metadata"),
fs.WithDir("content",
fs.WithDir("content",
fs.WithDir("d_0000001"),
),
),
)

PREMISFilePathNonExistent := ContentNonExistent.Join("metadata", "premis.xml")
Expand All @@ -72,7 +57,7 @@ func TestAddPREMISEvent(t *testing.T) {
{
name: "Add PREMIS event for normal content with no failures",
params: activities.AddPREMISEventParams{
PREMISFilePath: PREMISFilePathNormal,
PREMISFilePath: PREMISFilePathNormalNoFailures,
Agent: premis.AgentDefault(),
Type: "someActivity",
Failures: noFailures,
Expand All @@ -82,7 +67,7 @@ func TestAddPREMISEvent(t *testing.T) {
{
name: "Add PREMIS event for normal content with failures",
params: activities.AddPREMISEventParams{
PREMISFilePath: PREMISFilePathNormal,
PREMISFilePath: PREMISFilePathNormalWithFailures,
Agent: premis.AgentDefault(),
Type: "someActivity",
Failures: failures,
Expand Down
15 changes: 11 additions & 4 deletions internal/activities/validate_file_formats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,13 +99,20 @@ func (a *ValidateFileFormats) Execute(

// Define PREMIS event.
eventSummary := premis.EventSummary{
Type: "validateFileFormats",
Detail: detail,
Outcome: outcome,
Type: "validation",
Detail: detail,
Outcome: outcome,
OutcomeDetail: "Format allowed",
}

// Get subpath within content.
subpath, err := filepath.Rel(params.ContentPath, p)
if err != nil {
return err
}

// Append PREMIS event to XML and write results.
originalName := premis.OriginalNameForSubpath(p)
originalName := premis.OriginalNameForSubpath(subpath)

doc, err := premis.ParseOrInitialize(params.PREMISFilePath)
if err != nil {
Expand Down
70 changes: 68 additions & 2 deletions internal/activities/validate_file_formats_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,72 @@ import (

const pngContent = "\x89PNG\r\n\x1a\n\x00\x00\x00\x0DIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x02\x00\x00\x00\x90\x77\x53\xDE\x00\x00\x00\x00IEND\xAE\x42\x60\x82"

const premisValidFormatsContent = `<?xml version="1.0" encoding="UTF-8"?>
<premis:premis xmlns:premis="http://www.loc.gov/premis/v3" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/premis/v3 https://www.loc.gov/standards/premis/premis.xsd" version="3.0">
<premis:object xsi:type="premis:file">
<premis:objectIdentifier>
<premis:objectIdentifierType>uuid</premis:objectIdentifierType>
<premis:objectIdentifierValue>c74a85b7-919b-409e-8209-9c7ebe0e7945</premis:objectIdentifierValue>
</premis:objectIdentifier>
<premis:objectCharacteristics>
<premis:format>
<premis:formatDesignation>
<premis:formatName/>
</premis:formatDesignation>
</premis:format>
</premis:objectCharacteristics>
<premis:originalName>data/objects/dir/file1.txt</premis:originalName>
</premis:object>
<premis:object xsi:type="premis:file">
<premis:objectIdentifier>
<premis:objectIdentifierType>uuid</premis:objectIdentifierType>
<premis:objectIdentifierValue>a74a85b7-919b-409e-8209-9c7ebe0e7945</premis:objectIdentifierValue>
</premis:objectIdentifier>
<premis:objectCharacteristics>
<premis:format>
<premis:formatDesignation>
<premis:formatName/>
</premis:formatDesignation>
</premis:format>
</premis:objectCharacteristics>
<premis:originalName>data/objects/file2.txt</premis:originalName>
</premis:object>
</premis:premis>
`

const premisInvalidFormatsContent = `<?xml version="1.0" encoding="UTF-8"?>
<premis:premis xmlns:premis="http://www.loc.gov/premis/v3" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/premis/v3 https://www.loc.gov/standards/premis/premis.xsd" version="3.0">
<premis:object xsi:type="premis:file">
<premis:objectIdentifier>
<premis:objectIdentifierType>uuid</premis:objectIdentifierType>
<premis:objectIdentifierValue>c74a85b7-919b-409e-8209-9c7ebe0e7945</premis:objectIdentifierValue>
</premis:objectIdentifier>
<premis:objectCharacteristics>
<premis:format>
<premis:formatDesignation>
<premis:formatName/>
</premis:formatDesignation>
</premis:format>
</premis:objectCharacteristics>
<premis:originalName>data/objects/dir/file1.png</premis:originalName>
</premis:object>
<premis:object xsi:type="premis:file">
<premis:objectIdentifier>
<premis:objectIdentifierType>uuid</premis:objectIdentifierType>
<premis:objectIdentifierValue>a74a85b7-919b-409e-8209-9c7ebe0e7945</premis:objectIdentifierValue>
</premis:objectIdentifier>
<premis:objectCharacteristics>
<premis:format>
<premis:formatDesignation>
<premis:formatName/>
</premis:formatDesignation>
</premis:format>
</premis:objectCharacteristics>
<premis:originalName>data/objects/file2.png</premis:originalName>
</premis:object>
</premis:premis>
`

func TestValidateFileFormats(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -42,7 +108,7 @@ func TestValidateFileFormats(t *testing.T) {
fs.WithFile("file2.txt", "content"),
).Path(),
PREMISFilePath: fs.NewFile(t, "premis.xml",
fs.WithContent(premis.EmptyXML),
fs.WithContent(premisValidFormatsContent),
).Path(),
Agent: premis.AgentDefault(),
},
Expand All @@ -53,7 +119,7 @@ func TestValidateFileFormats(t *testing.T) {
params: activities.ValidateFileFormatsParams{
ContentPath: invalidFormatsPath,
PREMISFilePath: fs.NewFile(t, "premis.xml",
fs.WithContent(premis.EmptyXML),
fs.WithContent(premisInvalidFormatsContent),
).Path(),
Agent: premis.AgentDefault(),
},
Expand Down
44 changes: 23 additions & 21 deletions internal/premis/premis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,10 @@ type Object struct {
}

type EventSummary struct {
Type string
Detail string
Outcome string
Type string
Detail string
Outcome string
OutcomeDetail string
}

type Event struct {
Expand Down Expand Up @@ -128,30 +129,21 @@ func eventFromEventSummaryAndAgent(eventSummary EventSummary, agent Agent) Event
}
}

func NewEventSummary(eventType string, failures []string) (EventSummary, error) {
detail, outcome := assembleEventDetailAndOutcome(failures)

func NewEventSummary(eventType string, detail string, outcome string, outcomeDetail string) (EventSummary, error) {
return EventSummary{
Type: eventType,
Detail: detail,
Outcome: outcome,
Type: eventType,
Detail: detail,
Outcome: outcome,
OutcomeDetail: outcomeDetail,
}, nil
}

func assembleEventDetailAndOutcome(failures []string) (string, string) {
detail := ""
outcome := "valid"

func EventOutcomeForFailures(failures []string) string {
if failures != nil {
// Add failure descriptions to PREMIS event detail.
for _, description := range failures {
detail = detail + description + "\n"
}

outcome = "invalid"
return "invalid"
}

return detail, outcome
return "valid"
}

func AppendObjectXML(doc *etree.Document, object Object) error {
Expand Down Expand Up @@ -182,7 +174,11 @@ func AppendEventAndLinkToObject(doc *etree.Document, eventSummary EventSummary,
return err
}

LinkEventToObject(objectEl, event)
if objectEl != nil {
LinkEventToObject(objectEl, event)
} else {
return fmt.Errorf("append event and link to object: object '%s' not found", originalName)
}

return nil
}
Expand Down Expand Up @@ -276,6 +272,12 @@ func addEventElement(PREMISEl *etree.Element, event Event) {
outcomeEl := outcomeInfoEl.CreateElement("premis:eventOutcome")
outcomeEl.CreateText(event.Summary.Outcome)

if event.Summary.OutcomeDetail != "" {
outcomeDetailEl := outcomeInfoEl.CreateElement("premis:eventOutcomeDetail")
outcomeDetailNoteEl := outcomeDetailEl.CreateElement("premis:eventOutcomeDetailNote")
outcomeDetailNoteEl.CreateText(event.Summary.OutcomeDetail)
}

addEventAgentIdentifierElement(eventEl, event)
}

Expand Down
15 changes: 9 additions & 6 deletions internal/premis/premis_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ const premisEventAddContent = `<?xml version="1.0" encoding="UTF-8"?>
<premis:eventIdentifierType>UUID</premis:eventIdentifierType>
<premis:eventIdentifierValue/>
</premis:eventIdentifier>
<premis:eventType>validateFiles</premis:eventType>
<premis:eventType>validation</premis:eventType>
<premis:eventDateTime/>
<premis:eventDetailInformation>
<premis:eventDetail>event detail</premis:eventDetail>
Expand Down Expand Up @@ -78,14 +78,16 @@ const premisObjectAndEventAddContent = `<?xml version="1.0" encoding="UTF-8"?>
<premis:eventIdentifierType>UUID</premis:eventIdentifierType>
<premis:eventIdentifierValue/>
</premis:eventIdentifier>
<premis:eventType>someEvent</premis:eventType>
<premis:eventType>validation</premis:eventType>
<premis:eventDateTime/>
<premis:eventDetailInformation>
<premis:eventDetail>some failure
</premis:eventDetail>
<premis:eventDetail>name=&quot;Validate SIP metadata&quot;</premis:eventDetail>
</premis:eventDetailInformation>
<premis:eventOutcomeInformation>
<premis:eventOutcome>invalid</premis:eventOutcome>
<premis:eventOutcomeDetail>
<premis:eventOutcomeDetailNote>Metadata validation successful</premis:eventOutcomeDetailNote>
</premis:eventOutcomeDetail>
</premis:eventOutcomeInformation>
<premis:linkingAgentIdentifier>
<premis:linkingAgentIdentifierType valueURI="http://id.loc.gov/vocabulary/identifiers/local">url</premis:linkingAgentIdentifierType>
Expand Down Expand Up @@ -138,7 +140,7 @@ func TestAppendPREMISEventXML(t *testing.T) {
assert.NilError(t, err)

err = premis.AppendEventXML(doc, premis.EventSummary{
Type: "validateFiles",
Type: "validation",
Detail: "event detail",
Outcome: "valid",
}, premis.AgentDefault())
Expand Down Expand Up @@ -243,9 +245,10 @@ func TestAppendPREMISEventAndLinkToObject(t *testing.T) {
// Define PREMIS event with failure.
var failures []string
failures = append(failures, "some failure")
outcome := premis.EventOutcomeForFailures(failures)

// Add PREMIS event to XML document.
eventSummary, err := premis.NewEventSummary("someEvent", failures)
eventSummary, err := premis.NewEventSummary("validation", "name=\"Validate SIP metadata\"", outcome, "Metadata validation successful")
assert.NilError(t, err)

doc := etree.NewDocument()
Expand Down
28 changes: 11 additions & 17 deletions internal/workflow/preprocessing.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,14 +147,21 @@ func (w *PreprocessingWorkflow) Execute(
}

// Add PREMIS event noting validate structure result.
validateStructureOutcomeDetail := fmt.Sprintf(
"SIP structure identified: %s. SIP structure matches validation criteria.",
identifySIP.SIP.Type.String(),
)

var addPREMISEvent activities.AddPREMISEventResult
e = temporalsdk_workflow.ExecuteActivity(
withLocalActOpts(ctx),
activities.AddPREMISEventName,
&activities.AddPREMISEventParams{
PREMISFilePath: premisFilePath,
Agent: premis.AgentDefault(),
Type: "validateStructure",
Type: "validation",
Detail: "name=\"Validate SIP structure\"",
OutcomeDetail: validateStructureOutcomeDetail,
Failures: validateStructure.Failures,
},
).Get(ctx, &addPREMISEvent)
Expand Down Expand Up @@ -198,21 +205,6 @@ func (w *PreprocessingWorkflow) Execute(
}
result.addEvent(validateFileFormatsEvent)

// Add PREMIS event noting validate file formats result.
e = temporalsdk_workflow.ExecuteActivity(
withLocalActOpts(ctx),
activities.AddPREMISEventName,
&activities.AddPREMISEventParams{
PREMISFilePath: premisFilePath,
Agent: premis.AgentDefault(),
Type: "validateFileFormats",
Failures: validateFileFormats.Failures,
},
).Get(ctx, &addPREMISEvent)
if e != nil {
return nil, e
}

// Validate metadata.
validateMetadataEvent := newEvent(ctx, "Validate SIP metadata")
var validateMetadata activities.ValidateMetadataResult
Expand Down Expand Up @@ -257,7 +249,9 @@ func (w *PreprocessingWorkflow) Execute(
&activities.AddPREMISEventParams{
PREMISFilePath: premisFilePath,
Agent: premis.AgentDefault(),
Type: "validateMetadata",
Type: "validation",
Detail: "name=\"Validate SIP metadata\"",
OutcomeDetail: "Metadata validation successful",
Failures: validateMetadata.Failures,
},
).Get(ctx, &addPREMISEvent)
Expand Down
Loading

0 comments on commit b736af4

Please sign in to comment.