From b5e2654cebcd778736dc1aee4c70e639c9b883d5 Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Fri, 15 Feb 2019 14:58:00 -0500 Subject: [PATCH 1/5] Externalized HTML drivers --- cli/options.go | 4 +- e2e/runner/lib.go | 1 + e2e/runner/runner.go | 15 +- pkg/drivers/cdp/document.go | 146 +++++------------- pkg/drivers/cdp/driver.go | 15 +- pkg/drivers/cdp/element.go | 132 +++++++++------- pkg/drivers/cdp/helpers.go | 2 +- pkg/drivers/common/errors.go | 8 + pkg/drivers/common/getter.go | 101 ++++++++++++ pkg/drivers/common/iterator.go | 37 +++++ pkg/drivers/common/path.go | 17 +++ pkg/drivers/common/setter.go | 90 +++++++++++ pkg/drivers/driver.go | 66 ++++---- pkg/drivers/http/document.go | 204 +++++++++++++++++++++++-- pkg/drivers/http/driver.go | 11 +- pkg/drivers/http/element.go | 202 ++++++++++++++---------- pkg/drivers/options.go | 5 + pkg/drivers/pdf.go | 39 +++++ pkg/drivers/screenshot.go | 47 ++++++ pkg/drivers/type.go | 38 +++++ pkg/drivers/value.go | 124 +++++++++++++++ pkg/runtime/collections/html.go | 55 ------- pkg/runtime/core/errors.go | 1 + pkg/runtime/expressions/data_source.go | 5 - pkg/runtime/values/helpers.go | 139 +++++------------ pkg/runtime/values/html.go | 191 ----------------------- pkg/runtime/values/types/helpers.go | 20 ++- pkg/runtime/values/types/types.go | 20 ++- pkg/stdlib/collections/length.go | 2 - pkg/stdlib/html/click.go | 24 +-- pkg/stdlib/html/click_all.go | 10 +- pkg/stdlib/html/document.go | 60 ++++---- pkg/stdlib/html/element.go | 7 +- pkg/stdlib/html/element_exists.go | 2 +- pkg/stdlib/html/elements.go | 2 +- pkg/stdlib/html/elements_count.go | 2 +- pkg/stdlib/html/hover.go | 16 +- pkg/stdlib/html/inner_html.go | 13 +- pkg/stdlib/html/inner_html_all.go | 12 +- pkg/stdlib/html/inner_text.go | 8 +- pkg/stdlib/html/inner_text_all.go | 12 +- pkg/stdlib/html/input.go | 49 +++--- pkg/stdlib/html/lib.go | 51 +++++-- pkg/stdlib/html/navigate.go | 14 +- pkg/stdlib/html/navigate_back.go | 8 +- pkg/stdlib/html/navigate_forward.go | 8 +- pkg/stdlib/html/pagination.go | 13 +- pkg/stdlib/html/parse.go | 38 ----- pkg/stdlib/html/pdf.go | 5 +- pkg/stdlib/html/screenshot.go | 13 +- pkg/stdlib/html/scroll_bottom.go | 10 +- pkg/stdlib/html/scroll_element.go | 29 ++-- pkg/stdlib/html/scroll_top.go | 10 +- pkg/stdlib/html/select.go | 35 ++--- pkg/stdlib/html/wait_class.go | 33 ++-- pkg/stdlib/html/wait_class_all.go | 11 +- pkg/stdlib/html/wait_element.go | 21 +-- pkg/stdlib/html/wait_navigation.go | 10 +- pkg/stdlib/types/is_html_document.go | 4 +- pkg/stdlib/types/is_html_element.go | 4 +- pkg/stdlib/types/to_array.go | 6 +- 61 files changed, 1282 insertions(+), 995 deletions(-) create mode 100644 pkg/drivers/common/errors.go create mode 100644 pkg/drivers/common/getter.go create mode 100644 pkg/drivers/common/iterator.go create mode 100644 pkg/drivers/common/path.go create mode 100644 pkg/drivers/common/setter.go create mode 100644 pkg/drivers/options.go create mode 100644 pkg/drivers/pdf.go create mode 100644 pkg/drivers/screenshot.go create mode 100644 pkg/drivers/type.go create mode 100644 pkg/drivers/value.go delete mode 100644 pkg/runtime/collections/html.go delete mode 100644 pkg/runtime/values/html.go delete mode 100644 pkg/stdlib/html/parse.go diff --git a/cli/options.go b/cli/options.go index 9f0a4cc7..47ea0608 100644 --- a/cli/options.go +++ b/cli/options.go @@ -18,7 +18,7 @@ type Options struct { func (opts Options) WithContext(ctx context.Context) (context.Context, error) { var err error - ctx = drivers.WithDynamic( + ctx = drivers.WithContext( ctx, cdp.NewDriver( cdp.WithAddress(opts.Cdp), @@ -31,7 +31,7 @@ func (opts Options) WithContext(ctx context.Context) (context.Context, error) { return ctx, err } - ctx = drivers.WithStatic( + ctx = drivers.WithContext( ctx, http.NewDriver( http.WithProxy(opts.Proxy), diff --git a/e2e/runner/lib.go b/e2e/runner/lib.go index a48adb8e..e3e3ef01 100644 --- a/e2e/runner/lib.go +++ b/e2e/runner/lib.go @@ -3,6 +3,7 @@ package runner import ( "context" "fmt" + "github.com/MontFerret/ferret/pkg/runtime/core" "github.com/MontFerret/ferret/pkg/runtime/values" ) diff --git a/e2e/runner/runner.go b/e2e/runner/runner.go index 9e8dae55..2478663c 100644 --- a/e2e/runner/runner.go +++ b/e2e/runner/runner.go @@ -3,6 +3,12 @@ package runner import ( "context" "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "time" + "github.com/MontFerret/ferret/pkg/compiler" "github.com/MontFerret/ferret/pkg/drivers" "github.com/MontFerret/ferret/pkg/drivers/cdp" @@ -10,11 +16,6 @@ import ( "github.com/MontFerret/ferret/pkg/runtime" "github.com/pkg/errors" "github.com/rs/zerolog" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "time" ) type ( @@ -145,12 +146,12 @@ func (r *Runner) runQuery(c *compiler.FqlCompiler, name, script string) Result { } ctx := context.Background() - ctx = drivers.WithDynamic( + ctx = drivers.WithContext( ctx, cdp.NewDriver(cdp.WithAddress(r.settings.CDPAddress)), ) - ctx = drivers.WithStatic(ctx, http.NewDriver()) + ctx = drivers.WithContext(ctx, http.NewDriver()) out, err := p.Run( ctx, diff --git a/pkg/drivers/cdp/document.go b/pkg/drivers/cdp/document.go index b6fc719c..72e68488 100644 --- a/pkg/drivers/cdp/document.go +++ b/pkg/drivers/cdp/document.go @@ -7,8 +7,10 @@ import ( "sync" "time" + "github.com/MontFerret/ferret/pkg/drivers" "github.com/MontFerret/ferret/pkg/drivers/cdp/eval" "github.com/MontFerret/ferret/pkg/drivers/cdp/events" + "github.com/MontFerret/ferret/pkg/drivers/common" "github.com/MontFerret/ferret/pkg/runtime/core" "github.com/MontFerret/ferret/pkg/runtime/logging" "github.com/MontFerret/ferret/pkg/runtime/values" @@ -24,17 +26,15 @@ import ( const BlankPageURL = "about:blank" -type ( - HTMLDocument struct { - sync.Mutex - logger *zerolog.Logger - conn *rpcc.Conn - client *cdp.Client - events *events.EventBroker - url values.String - element *HTMLElement - } -) +type HTMLDocument struct { + sync.Mutex + logger *zerolog.Logger + conn *rpcc.Conn + client *cdp.Client + events *events.EventBroker + url values.String + element *HTMLElement +} func handleLoadError(logger *zerolog.Logger, client *cdp.Client) { err := client.Page.Close(context.Background()) @@ -49,7 +49,7 @@ func LoadHTMLDocument( conn *rpcc.Conn, client *cdp.Client, url string, -) (*HTMLDocument, error) { +) (drivers.HTMLDocument, error) { logger := logging.FromContext(ctx) if conn == nil { @@ -143,7 +143,7 @@ func (doc *HTMLDocument) MarshalJSON() ([]byte, error) { } func (doc *HTMLDocument) Type() core.Type { - return types.HTMLDocument + return drivers.HTMLDocumentType } func (doc *HTMLDocument) String() string { @@ -181,13 +181,26 @@ func (doc *HTMLDocument) Compare(other core.Value) int64 { doc.Lock() defer doc.Unlock() - if other.Type() == types.HTMLDocument { - other := other.(*HTMLDocument) + switch other.Type() { + case drivers.HTMLDocumentType: + other := other.(drivers.HTMLDocument) - return doc.url.Compare(other.url) + return doc.url.Compare(other.GetURL()) + default: + return drivers.Compare(doc.Type(), other.Type()) } +} + +func (doc *HTMLDocument) Iterate(ctx context.Context) (core.Iterator, error) { + return doc.element.Iterate(ctx) +} + +func (doc *HTMLDocument) GetIn(ctx context.Context, path []core.Value) (core.Value, error) { + return common.GetInDocument(ctx, doc, path) +} - return types.Compare(other.Type(), types.HTMLDocument) +func (doc *HTMLDocument) SetIn(ctx context.Context, path []core.Value, value core.Value) error { + return common.SetInDocument(ctx, doc, path, value) } func (doc *HTMLDocument) Close() error { @@ -260,41 +273,6 @@ func (doc *HTMLDocument) Length() values.Int { return doc.element.Length() } -func (doc *HTMLDocument) InnerText() values.String { - doc.Lock() - defer doc.Unlock() - - return doc.element.InnerText() -} - -func (doc *HTMLDocument) InnerHTML() values.String { - doc.Lock() - defer doc.Unlock() - - return doc.element.InnerHTML() -} - -func (doc *HTMLDocument) Value() core.Value { - doc.Lock() - defer doc.Unlock() - - return doc.element.Value() -} - -func (doc *HTMLDocument) GetAttributes() core.Value { - doc.Lock() - defer doc.Unlock() - - return doc.element.GetAttributes() -} - -func (doc *HTMLDocument) GetAttribute(name values.String) core.Value { - doc.Lock() - defer doc.Unlock() - - return doc.element.GetAttribute(name) -} - func (doc *HTMLDocument) GetChildNodes() core.Value { doc.Lock() defer doc.Unlock() @@ -323,39 +301,22 @@ func (doc *HTMLDocument) QuerySelectorAll(selector values.String) core.Value { return doc.element.QuerySelectorAll(selector) } -func (doc *HTMLDocument) URL() core.Value { - doc.Lock() - defer doc.Unlock() - - return doc.url -} - -func (doc *HTMLDocument) InnerHTMLBySelector(selector values.String) values.String { +func (doc *HTMLDocument) DocumentElement() drivers.HTMLElement { doc.Lock() defer doc.Unlock() - return doc.element.InnerHTMLBySelector(selector) + return doc.element } -func (doc *HTMLDocument) InnerHTMLBySelectorAll(selector values.String) *values.Array { +func (doc *HTMLDocument) GetURL() core.Value { doc.Lock() defer doc.Unlock() - return doc.element.InnerHTMLBySelectorAll(selector) + return doc.url } -func (doc *HTMLDocument) InnerTextBySelector(selector values.String) values.String { - doc.Lock() - defer doc.Unlock() - - return doc.element.InnerTextBySelector(selector) -} - -func (doc *HTMLDocument) InnerTextBySelectorAll(selector values.String) *values.Array { - doc.Lock() - defer doc.Unlock() - - return doc.element.InnerTextBySelectorAll(selector) +func (doc *HTMLDocument) SetURL(url values.String) error { + return doc.Navigate(url, values.Int(DefaultTimeout)) } func (doc *HTMLDocument) CountBySelector(selector values.String) values.Int { @@ -377,14 +338,11 @@ func (doc *HTMLDocument) ClickBySelector(selector values.String) (values.Boolean doc.client, fmt.Sprintf(` var el = document.querySelector(%s); - if (el == null) { return false; } - var evt = new window.MouseEvent('click', { bubbles: true, cancelable: true }); el.dispatchEvent(evt); - return true; `, eval.ParamString(selector.String())), true, @@ -407,16 +365,13 @@ func (doc *HTMLDocument) ClickBySelectorAll(selector values.String) (values.Bool doc.client, fmt.Sprintf(` var elements = document.querySelectorAll(%s); - if (elements == null) { return false; } - elements.forEach((el) => { var evt = new window.MouseEvent('click', { bubbles: true, cancelable: true }); el.dispatchEvent(evt); }); - return true; `, eval.ParamString(selector.String())), true, @@ -483,20 +438,15 @@ func (doc *HTMLDocument) SelectBySelector(selector values.String, value *values. doc.client, fmt.Sprintf(` var element = document.querySelector(%s); - if (element == null) { return []; } - var values = %s; - if (element.nodeName.toLowerCase() !== 'select') { throw new Error('Element is not a element.") + return nil, core.Error(core.ErrInvalidOperation, "element is not a element.'); + if (el.nodeName.toLowerCase() !== 'select') { + throw new Error('element is not a