-
Notifications
You must be signed in to change notification settings - Fork 1.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Add wasm spec factory * Fix workflow tests * Fix and test spec validation for non-YAML specs * Revert use of logger in validate * tidy and reorder inputs * brotli the WASM so it's smaller for the sdk to pass around * Fix so sum is always based on wasm not encoded value * Fix typo and lint imports in wasm file spec factory * Fix sha sum on wasm * Fix go mod merge conflict :( * Fix lint and remove local hack for go version...
- Loading branch information
Showing
21 changed files
with
272 additions
and
180 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| //go:build wasip1 | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "log" | ||
|
|
||
| "github.com/smartcontractkit/chainlink-common/pkg/workflows/wasm" | ||
|
|
||
| "github.com/smartcontractkit/chainlink-common/pkg/capabilities/cli/cmd/testdata/fixtures/capabilities/basictrigger" | ||
| "github.com/smartcontractkit/chainlink-common/pkg/workflows/sdk" | ||
| ) | ||
|
|
||
| func BuildWorkflow(config []byte) *sdk.WorkflowSpecFactory { | ||
| params := sdk.NewWorkflowParams{} | ||
| if err := json.Unmarshal(config, ¶ms); err != nil { | ||
| log.Fatal(err) | ||
| } | ||
|
|
||
| workflow := sdk.NewWorkflowSpecFactory(params) | ||
|
|
||
| triggerCfg := basictrigger.TriggerConfig{Name: "trigger", Number: 100} | ||
| _ = triggerCfg.New(workflow) | ||
|
|
||
| return workflow | ||
| } | ||
|
|
||
| func main() { | ||
| runner := wasm.NewRunner() | ||
| workflow := BuildWorkflow(runner.Config()) | ||
| runner.Run(workflow) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| package job | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "crypto/sha256" | ||
| "errors" | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| "path" | ||
| "strings" | ||
|
|
||
| "github.com/smartcontractkit/chainlink-common/pkg/workflows/sdk" | ||
| "github.com/smartcontractkit/chainlink-common/pkg/workflows/wasm/host" | ||
|
|
||
| "github.com/smartcontractkit/chainlink/v2/core/logger" | ||
|
|
||
| "github.com/andybalholm/brotli" | ||
| ) | ||
|
|
||
| type WasmFileSpecFactory struct{} | ||
|
|
||
| func (w WasmFileSpecFactory) Spec(ctx context.Context, lggr logger.Logger, workflow string, config []byte) (sdk.WorkflowSpec, string, error) { | ||
| compressedBinary, sha, err := w.rawSpecAndSha(workflow, config) | ||
| if err != nil { | ||
| return sdk.WorkflowSpec{}, "", err | ||
| } | ||
|
|
||
| moduleConfig := &host.ModuleConfig{Logger: lggr} | ||
| spec, err := host.GetWorkflowSpec(moduleConfig, compressedBinary, config) | ||
| if err != nil { | ||
| return sdk.WorkflowSpec{}, "", err | ||
| } else if spec == nil { | ||
| return sdk.WorkflowSpec{}, "", errors.New("workflow spec not found when running wasm") | ||
| } | ||
|
|
||
| return *spec, sha, nil | ||
| } | ||
|
|
||
| // rawSpecAndSha returns the brotli compressed version of the raw wasm file, alongside the sha256 hash of the raw wasm file | ||
| func (w WasmFileSpecFactory) rawSpecAndSha(wf string, config []byte) ([]byte, string, error) { | ||
| read, err := os.ReadFile(wf) | ||
| if err != nil { | ||
| return nil, "", err | ||
| } | ||
|
|
||
| extension := strings.ToLower(path.Ext(wf)) | ||
| switch extension { | ||
| case ".wasm", "": | ||
| return w.rawSpecAndShaFromWasm(read, config) | ||
| case ".br": | ||
| return w.rawSpecAndShaFromBrotli(read, config) | ||
| default: | ||
| return nil, "", fmt.Errorf("unsupported file type %s", extension) | ||
| } | ||
| } | ||
|
|
||
| func (w WasmFileSpecFactory) rawSpecAndShaFromBrotli(wasm, config []byte) ([]byte, string, error) { | ||
| brr := brotli.NewReader(bytes.NewReader(wasm)) | ||
| rawWasm, err := io.ReadAll(brr) | ||
| if err != nil { | ||
| return nil, "", err | ||
| } | ||
|
|
||
| return wasm, w.sha(rawWasm, config), nil | ||
| } | ||
|
|
||
| func (w WasmFileSpecFactory) rawSpecAndShaFromWasm(wasm, config []byte) ([]byte, string, error) { | ||
| var b bytes.Buffer | ||
| bwr := brotli.NewWriter(&b) | ||
| if _, err := bwr.Write(wasm); err != nil { | ||
| return nil, "", err | ||
| } | ||
|
|
||
| if err := bwr.Close(); err != nil { | ||
| return nil, "", err | ||
| } | ||
|
|
||
| return b.Bytes(), w.sha(wasm, config), nil | ||
| } | ||
|
|
||
| func (w WasmFileSpecFactory) sha(wasm, config []byte) string { | ||
| sum := sha256.New() | ||
| sum.Write(wasm) | ||
| sum.Write(config) | ||
| return fmt.Sprintf("%x", sum.Sum(nil)) | ||
| } | ||
|
|
||
| var _ WorkflowSpecFactory = (*WasmFileSpecFactory)(nil) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| package job_test | ||
|
|
||
| import ( | ||
| "crypto/sha256" | ||
| "encoding/json" | ||
| "fmt" | ||
| "os" | ||
| "os/exec" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
|
|
||
| "github.com/smartcontractkit/chainlink-common/pkg/workflows/sdk" | ||
| "github.com/smartcontractkit/chainlink-common/pkg/workflows/wasm/host" | ||
|
|
||
| "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" | ||
| "github.com/smartcontractkit/chainlink/v2/core/logger" | ||
| "github.com/smartcontractkit/chainlink/v2/core/services/job" | ||
| ) | ||
|
|
||
| func TestWasmFileSpecFactory(t *testing.T) { | ||
| binaryLocation := createTestBinary(t) | ||
| config, err := json.Marshal(sdk.NewWorkflowParams{ | ||
| Owner: "owner", | ||
| Name: "name", | ||
| }) | ||
| require.NoError(t, err) | ||
|
|
||
| factory := job.WasmFileSpecFactory{} | ||
| actual, actualSha, err := factory.Spec(testutils.Context(t), logger.NullLogger, binaryLocation, config) | ||
| require.NoError(t, err) | ||
|
|
||
| rawBinary, err := os.ReadFile(binaryLocation) | ||
| require.NoError(t, err) | ||
| expected, err := host.GetWorkflowSpec(&host.ModuleConfig{Logger: logger.NullLogger, IsUncompressed: true}, rawBinary, config) | ||
| require.NoError(t, err) | ||
|
|
||
| expectedSha := sha256.New() | ||
| expectedSha.Write(rawBinary) | ||
| expectedSha.Write(config) | ||
| require.Equal(t, fmt.Sprintf("%x", expectedSha.Sum(nil)), actualSha) | ||
|
|
||
| require.Equal(t, *expected, actual) | ||
| } | ||
|
|
||
| func createTestBinary(t *testing.T) string { | ||
| const testBinaryLocation = "testdata/wasm/testmodule.wasm" | ||
|
|
||
| cmd := exec.Command("go", "build", "-o", testBinaryLocation, "github.com/smartcontractkit/chainlink/v2/core/services/job/testdata/wasm") | ||
| cmd.Env = append(os.Environ(), "GOOS=wasip1", "GOARCH=wasm") | ||
|
|
||
| output, err := cmd.CombinedOutput() | ||
| require.NoError(t, err, string(output)) | ||
|
|
||
| return testBinaryLocation | ||
| } |
Oops, something went wrong.