Refactor clone modal component

bitmakerla · Sep 25, 2023 · 739d443 · 739d443
1 parent 2950a45
commit 739d443
Show file tree

Hide file tree

Showing 11 changed files with 159 additions and 671 deletions.
diff --git a/estela-api/api/serializers/job.py b/estela-api/api/serializers/job.py
@@ -7,6 +7,7 @@
     SpiderJobEnvVarSerializer,
     SpiderJobTagSerializer,
 )
+from api.serializers.spider import SpiderSerializer
 from api.utils import delete_stats_from_redis, update_stats_from_redis
 from config.job_manager import job_manager
 from core.models import (
@@ -30,7 +31,7 @@ class SpiderJobSerializer(serializers.ModelSerializer):
     job_status = serializers.CharField(
         required=False, read_only=True, help_text="Current job status."
     )
-    spider = serializers.SerializerMethodField("get_spider")
+    spider = SpiderSerializer(required=True, help_text="Job spider.")
 
     class Meta:
         model = SpiderJob

diff --git a/estela-api/api/serializers/job_specific.py b/estela-api/api/serializers/job_specific.py
@@ -10,6 +10,11 @@ class Meta:
 
 
 class SpiderJobEnvVarSerializer(serializers.ModelSerializer):
+    masked = serializers.BooleanField(
+        required=True,
+        help_text="Whether the env variable value is masked.",
+    )
+
     class Meta:
         model = SpiderJobEnvVar
         fields = ("evid", "name", "value", "masked")

diff --git a/estela-api/api/utils.py b/estela-api/api/utils.py
@@ -70,5 +70,5 @@ def delete_stats_from_redis(job):
     redis_conn = redis.from_url(settings.REDIS_URL)
     try:
         redis_conn.delete(f"scrapy_stats_{job.key}")
-    except:
+    except Exception:
         pass
diff --git a/estela-api/docs/api.yaml b/estela-api/docs/api.yaml
@@ -1709,6 +1709,7 @@ definitions:
     required:
       - name
       - value
+      - masked
     type: object
     properties:
       evid:
@@ -2184,6 +2185,8 @@ definitions:
           minLength: 1
   SpiderJob:
     description: Project jobs.
+    required:
+      - spider
     type: object
     properties:
       jid:
@@ -2192,9 +2195,7 @@ definitions:
         type: integer
         readOnly: true
       spider:
-        title: Spider
-        type: string
-        readOnly: true
+        $ref: '#/definitions/Spider'
       created:
         title: Created
         description: Job creation date.
@@ -2791,6 +2792,7 @@ definitions:
         $ref: '#/definitions/Stats'
   SpiderJobStats:
     required:
+      - spider
       - stats
     type: object
     properties:
@@ -2800,9 +2802,7 @@ definitions:
         type: integer
         readOnly: true
       spider:
-        title: Spider
-        type: string
-        readOnly: true
+        $ref: '#/definitions/Spider'
       created:
         title: Created
         description: Job creation date.

diff --git a/estela-web/src/pages/JobCreateModal/index.tsx b/estela-web/src/pages/JobCreateModal/index.tsx
@@ -27,6 +27,10 @@ interface JobCreateModalProps {
     openModal: boolean;
     spider: Spider | null;
     projectId: string;
+    envVarsProps: SpiderJobEnvVar[];
+    argsProps: ArgsData[];
+    tagsProps: TagsData[];
+    children: React.ReactNode;
 }
 
 interface MaskedTagProps {
@@ -41,20 +45,14 @@ interface ArgsData {
     key: number;
 }
 
-interface EnvVarsData {
-    name: string;
-    value: string;
-    key: number;
-    masked: boolean;
-}
-
 interface TagsData {
     name: string;
+    key: number;
 }
 
 interface JobData {
     args: ArgsData[];
-    envVars: EnvVarsData[];
+    envVars: SpiderJobEnvVar[];
     tags: TagsData[];
     dataStatus: SpiderDataStatusEnum | undefined;
     dataExpiryDays: number | null | undefined;
@@ -90,7 +88,15 @@ const dataPersistenceOptions = [
     { label: "Forever", key: 7, value: 720 },
 ];
 
-export default function JobCreateModal({ openModal, spider, projectId }: JobCreateModalProps) {
+export default function JobCreateModal({
+    openModal,
+    spider,
+    projectId,
+    envVarsProps,
+    argsProps,
+    tagsProps,
+    children,
+}: JobCreateModalProps) {
     const PAGE_SIZE = 15;
     const apiService = ApiService();
     const [open, setOpen] = useState(openModal);
@@ -99,9 +105,9 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea
     const [spiders, setSpiders] = useState<Spider[]>([]);
     const [externalComponent, setExternalComponent] = useState<React.ReactNode>(<></>);
     const [jobData, setJobData] = useState<JobData>({
-        args: [],
-        envVars: [],
-        tags: [],
+        args: argsProps,
+        envVars: envVarsProps,
+        tags: tagsProps,
         dataStatus: spider ? spider.dataStatus : undefined,
         dataExpiryDays: spider ? spider.dataExpiryDays : 1,
     });
@@ -310,7 +316,7 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea
         }
     };
 
-    const addEnvVar = (): EnvVarsData | null => {
+    const addEnvVar = (): SpiderJobEnvVar | null => {
         const envVars = [...jobData.envVars];
         const newEnvVarName = variable.newEnvVarName.trim();
         const newEnvVarValue = variable.newEnvVarValue.trim();
@@ -319,7 +325,6 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea
                 name: newEnvVarName,
                 value: newEnvVarValue,
                 masked: variable.newEnvVarMasked,
-                key: countKey,
             };
             envVars.push(newEnvVar);
             setCountKey(countKey + 1);
@@ -373,8 +378,8 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea
     const addPendingFormElement = (
         elementName: string,
         elementValue: string,
-        addElementFunction: callable,
-        elementList: Array,
+        addElementFunction: CallableFunction,
+        elementList: Array<ArgsData | SpiderJobEnvVar | TagsData>,
     ): boolean => {
         elementName = elementName.trim();
         elementValue = elementValue.trim();
@@ -414,10 +419,11 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea
                 envVarsData.push(envVar);
             }
         });
-        jobData.envVars.map((envVar: EnvVarsData) => {
+        jobData.envVars.map((envVar: SpiderJobEnvVar) => {
             const index = envVarsData.findIndex((element: SpiderJobEnvVar) => element.name === envVar.name);
             if (index != -1) {
                 envVarsData[index] = {
+                    evid: envVar.evid,
                     name: envVar.name,
                     value: envVar.value,
                     masked: envVar.masked,
@@ -443,10 +449,12 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea
             pid: pid,
             sid: sid,
         };
+
         apiService.apiProjectsSpidersJobsCreate(requests).then(
             (response: SpiderJobCreate) => {
                 setLoading(false);
                 history.push(`/projects/${pid}/spiders/${sid}/jobs/${response.jid}`);
+                location.reload();
             },
             async (error) => {
                 setLoading(false);
@@ -478,7 +486,7 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea
                     }
                 }}
             >
-                Run new job
+                {children}
             </Button>
             {externalComponent}
             <Modal
@@ -571,7 +579,7 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea
                     <Space direction="vertical" className="flex">
                         {projectEnvVars.length > 0 && (
                             <Content className="flex">
-                                <p className="text-sm">Project</p>
+                                <p className="text-sm mr-2">Project</p>
                                 <div className="flex gap-2">
                                     {projectEnvVars.map((envVar: SpiderJobEnvVar, id: number) =>
                                         envVar.masked ? (
@@ -593,8 +601,8 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea
                             </Content>
                         )}
                         {spiderEnvVars.length > 0 && (
-                            <>
-                                <p className="text-sm">Spider</p>
+                            <Content className="flex">
+                                <p className="text-sm mr-2">Spider</p>
                                 <div className="flex gap-2">
                                     {spiderEnvVars.map((envVar: SpiderJobEnvVar, id: number) =>
                                         envVar.masked ? (
@@ -613,10 +621,10 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea
                                         ),
                                     )}
                                 </div>
-                            </>
+                            </Content>
                         )}
                         <Space direction="horizontal" className="mb-2">
-                            {jobData.envVars.map((envVar: EnvVarsData, id: number) =>
+                            {jobData.envVars.map((envVar: SpiderJobEnvVar, id: number) =>
                                 envVar.masked ? (
                                     <Tooltip key={id} placement="top" title="Masked variable" showArrow={false}>
                                         <Tag