new piece

Author: nathan-vm

pieces/GetHTMLPiece/metadata.json

@@ -0,0 +1,27 @@
+{
+    "name": "GetHTMLPiece",
+    "description": "GetHTMLPiece",
+    "dependency": {
+        "dockerfile": "Dockerfile_1"
+    },
+    "container_resources": {
+        "requests": {
+            "cpu": 100,
+            "memory": 128
+        },
+        "limits": {
+            "cpu": 1000,
+            "memory": 1024
+        }
+    },
+    "tags": [
+        "browser",
+        "chrome",
+        "web",
+        "selenium"
+    ],
+    "style": {
+        "node_label": "Get HTML Piece",
+        "icon_class_name": "skill-icons:selenium"
+    }
+}

pieces/GetHTMLPiece/models.py

@@ -0,0 +1,20 @@
+from pydantic import BaseModel, Field
+from typing import List
+
+class InputModel(BaseModel):
+    """
+    Selenium Web Browser Input Model
+    """
+
+    get_page_html: List[str] = Field(
+        default=["http://www.google.com.br"],
+        description="URL you want to extract HTML",
+    )
+
+class OutputModel(BaseModel):
+    """
+    Selenium Web Browser Output Model
+    """
+    output_file_path: str = Field(
+        description="Path for pickle file with a list of all HTML files combined."
+    )

pieces/GetHTMLPiece/piece.py

@@ -0,0 +1,39 @@
+from domino.base_piece import BasePiece
+from .models import InputModel, OutputModel
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from pathlib import Path
+from typing import List
+import uuid
+import pickle
+
+class GetHTMLPiece(BasePiece):
+
+    def piece_function(self, input_data: InputModel) -> OutputModel:
+        results_path = Path(self.results_path)
+        outputs: List[str] = []
+
+        self.logger.info("Create chrome options arguments.")
+
+        options = Options()
+        options.add_argument("--headless=new")
+        options.add_argument("--no-sandbox")
+
+        self.logger.info("Start chrome web driver.")
+        driver = webdriver.Chrome(options=options)
+
+        for url in input_data.get_page_html:
+            driver.get(url)
+            page_html = driver.page_source
+            outputs.append(page_html)
+
+        driver.quit()
+
+        file_name = f"{results_path}/{uuid.uuid4()}.pkl"
+
+        with open(file_name,"wb") as file:
+            pickle.dump(outputs, file)
+        
+        return OutputModel(
+            output_file_path=file_name
+        )