diff --git a/estela-api/api/serializers/job.py b/estela-api/api/serializers/job.py index e7d17929..9cc0986c 100644 --- a/estela-api/api/serializers/job.py +++ b/estela-api/api/serializers/job.py @@ -7,6 +7,7 @@ SpiderJobEnvVarSerializer, SpiderJobTagSerializer, ) +from api.serializers.spider import SpiderSerializer from api.utils import delete_stats_from_redis, update_stats_from_redis from config.job_manager import job_manager from core.models import ( @@ -30,7 +31,7 @@ class SpiderJobSerializer(serializers.ModelSerializer): job_status = serializers.CharField( required=False, read_only=True, help_text="Current job status." ) - spider = serializers.SerializerMethodField("get_spider") + spider = SpiderSerializer(required=True, help_text="Job spider.") class Meta: model = SpiderJob diff --git a/estela-api/api/serializers/job_specific.py b/estela-api/api/serializers/job_specific.py index 415f6068..e84b9fcb 100644 --- a/estela-api/api/serializers/job_specific.py +++ b/estela-api/api/serializers/job_specific.py @@ -10,6 +10,11 @@ class Meta: class SpiderJobEnvVarSerializer(serializers.ModelSerializer): + masked = serializers.BooleanField( + required=True, + help_text="Whether the env variable value is masked.", + ) + class Meta: model = SpiderJobEnvVar fields = ("evid", "name", "value", "masked") diff --git a/estela-api/api/utils.py b/estela-api/api/utils.py index b74a2bb6..bd78ab6b 100644 --- a/estela-api/api/utils.py +++ b/estela-api/api/utils.py @@ -70,5 +70,5 @@ def delete_stats_from_redis(job): redis_conn = redis.from_url(settings.REDIS_URL) try: redis_conn.delete(f"scrapy_stats_{job.key}") - except: + except Exception: pass diff --git a/estela-api/docs/api.yaml b/estela-api/docs/api.yaml index dcfaef48..89f5524b 100644 --- a/estela-api/docs/api.yaml +++ b/estela-api/docs/api.yaml @@ -1709,6 +1709,7 @@ definitions: required: - name - value + - masked type: object properties: evid: @@ -2184,6 +2185,8 @@ definitions: minLength: 1 SpiderJob: description: Project jobs. + required: + - spider type: object properties: jid: @@ -2192,9 +2195,7 @@ definitions: type: integer readOnly: true spider: - title: Spider - type: string - readOnly: true + $ref: '#/definitions/Spider' created: title: Created description: Job creation date. @@ -2791,6 +2792,7 @@ definitions: $ref: '#/definitions/Stats' SpiderJobStats: required: + - spider - stats type: object properties: @@ -2800,9 +2802,7 @@ definitions: type: integer readOnly: true spider: - title: Spider - type: string - readOnly: true + $ref: '#/definitions/Spider' created: title: Created description: Job creation date. diff --git a/estela-web/src/components/EnvVarsSettingsPage/index.tsx b/estela-web/src/components/EnvVarsSettingsPage/index.tsx index 7b2c0ba6..61b364d6 100644 --- a/estela-web/src/components/EnvVarsSettingsPage/index.tsx +++ b/estela-web/src/components/EnvVarsSettingsPage/index.tsx @@ -235,7 +235,7 @@ export const EnvVarsSetting: React.FC = ({ projectId, spiderId, e setNewEnvVarValue(""); setNewEnvVarMasked(false); setOpenModal(false); - setActiveUpdateButton(true); + updateEnvVars(); } else { invalidDataNotification("Invalid environment variable name/value pair."); } @@ -394,7 +394,7 @@ export const EnvVarsSetting: React.FC = ({ projectId, spiderId, e onClick={() => updateEnvVars()} className="border-estela bg-estela hover:border-estela hover:text-estela text-white rounded-md text-base h-full" > - Save variables + Save changes diff --git a/estela-web/src/pages/CronjobCreateModal/index.tsx b/estela-web/src/pages/CronjobCreateModal/index.tsx index 8fc7f45b..43cab0bd 100644 --- a/estela-web/src/pages/CronjobCreateModal/index.tsx +++ b/estela-web/src/pages/CronjobCreateModal/index.tsx @@ -91,10 +91,6 @@ interface EnvVarsData { interface TagsData { name: string; -} - -interface Tags { - name: string; key: number; } @@ -111,7 +107,6 @@ interface Cronjob { newEnvVarValue: string; newEnvVarMasked: boolean; newTagName: string; - newTags: Tags[]; } interface OptionDataRepeat { @@ -193,7 +188,6 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro newEnvVarValue: "", newEnvVarMasked: false, newTagName: "", - newTags: [], }); const [crontab, setCrontab] = useState({ expression: "", @@ -365,17 +359,20 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro setCronjobData({ ...cronjobData, args: [...args] }); }; - const addArgument = (): void => { + const addArgument = (): ArgsData | null => { const args = [...cronjobData.args]; const newArgName = newCronjob.newArgName.trim(); const newArgValue = newCronjob.newArgValue.trim(); if (newArgName && newArgValue && newArgName.indexOf(" ") == -1) { - args.push({ name: newArgName, value: newArgValue, key: countKey }); + const arg = { name: newArgName, value: newArgValue, key: countKey }; + args.push(arg); setCountKey(countKey + 1); setCronjobData({ ...cronjobData, args: [...args] }); setNewCronjob({ ...newCronjob, newArgName: "", newArgValue: "" }); + return arg; } else { invalidDataNotification("Invalid argument name/value pair."); + return null; } }; @@ -414,22 +411,25 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro } }; - const addEnvVar = (): void => { + const addEnvVar = (): EnvVarsData | null => { const envVars = [...cronjobData.envVars]; const newEnvVarName = newCronjob.newEnvVarName.trim(); const newEnvVarValue = newCronjob.newEnvVarValue.trim(); if (newEnvVarName && newEnvVarValue && newEnvVarName.indexOf(" ") == -1) { - envVars.push({ + const envVar = { name: newEnvVarName, value: newEnvVarValue, masked: newCronjob.newEnvVarMasked, key: countKey, - }); + }; + envVars.push(envVar); setCountKey(countKey + 1); setCronjobData({ ...cronjobData, envVars: [...envVars] }); setNewCronjob({ ...newCronjob, newEnvVarName: "", newEnvVarValue: "", newEnvVarMasked: false }); + return envVar; } else { invalidDataNotification("Invalid environment variable name/value pair."); + return null; } }; @@ -439,18 +439,19 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro setCronjobData({ ...cronjobData, tags: [...tags] }); }; - const addTag = (): void => { + const addTag = (): TagsData | null => { const tags = [...cronjobData.tags]; - const newTags = [...newCronjob.newTags]; const newTagName = newCronjob.newTagName.trim(); if (newTagName && newTagName.indexOf(" ") == -1) { - newTags.push({ name: newTagName, key: countKey }); + const tag = { name: newTagName, key: countKey }; setCountKey(countKey + 1); - tags.push({ name: newTagName }); + tags.push(tag); setCronjobData({ ...cronjobData, tags: [...tags] }); - setNewCronjob({ ...newCronjob, newTags: [...newTags], newTagName: "" }); + setNewCronjob({ ...newCronjob, newTagName: "" }); + return tag; } else { invalidDataNotification("Invalid tag name."); + return null; } }; @@ -538,8 +539,27 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro setNewCronjob({ ...newCronjob, newEnvVarMasked: checked }); }; + const addPendingFormElement = ( + elementName: string, + elementValue: string, + addElementFunction: callable, + elementList: Array, + ): boolean => { + elementName = elementName.trim(); + elementValue = elementValue.trim(); + if (elementName || elementValue) { + const element = addElementFunction(); + if (!element) { + return false; + } + elementList.push(element); + } + return true; + }; + const handleSubmit = (): void => { setLoading(true); + let expression = ""; if (schedulesFlag[0]) { if (crontab.expression == "") { @@ -556,9 +576,23 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro expression = getExpression(); } - const envVarsData = projectEnvVars.map((envVar: SpiderJobEnvVar) => { - return envVar; - }); + if ( + !( + addPendingFormElement(newCronjob.newArgName, newCronjob.newArgValue, addArgument, cronjobData.args) && + addPendingFormElement( + newCronjob.newEnvVarName, + newCronjob.newEnvVarValue, + addEnvVar, + cronjobData.envVars, + ) && + addPendingFormElement(newCronjob.newTagName, "", addTag, cronjobData.tags) + ) + ) { + setLoading(false); + return; + } + + const envVarsData = [...projectEnvVars]; spiderEnvVars.map((envVar: SpiderJobEnvVar) => { const index = envVarsData.findIndex((element: SpiderJobEnvVar) => element.name === envVar.name); if (index != -1) { @@ -567,7 +601,6 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro envVarsData.push(envVar); } }); - cronjobData.envVars.map((envVar: EnvVarsData) => { const index = envVarsData.findIndex((element: SpiderJobEnvVar) => element.name === envVar.name); if (index != -1) { @@ -589,7 +622,6 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro cronjobData.dataStatus == SpiderDataStatusEnum.Persistent ? SpiderCronJobCreateDataStatusEnum.Persistent : SpiderCronJobCreateDataStatusEnum.Pending; - const requestData = { cargs: [...cronjobData.args], cenvVars: [...envVarsData], @@ -743,48 +775,52 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro

Environment Variables

-
- {projectEnvVars.length > 0 ?

Project:

: <>} -
- {projectEnvVars.map((envVar: SpiderJobEnvVar, id: number) => - envVar.masked ? ( - - {envVar.name} - - ) : ( - handleRemoveProjectEnvVar(id, true)} - > - {envVar.name}: {envVar.value} - - ), - )} -
-
-
- {spiderEnvVars.length > 0 ?

Spider:

: <>} -
- {spiderEnvVars.map((envVar: SpiderJobEnvVar, id: number) => - envVar.masked ? ( - - {envVar.name} - - ) : ( - handleRemoveProjectEnvVar(id, false)} - > - {envVar.name}: {envVar.value} - - ), - )} -
-
+ {projectEnvVars.length > 0 && ( + +

Project:

+
+ {projectEnvVars.map((envVar: SpiderJobEnvVar, id: number) => + envVar.masked ? ( + + {envVar.name} + + ) : ( + handleRemoveProjectEnvVar(id, true)} + > + {envVar.name}: {envVar.value} + + ), + )} +
+ + )} + {spiderEnvVars.length > 0 && ( + +

Spider:

+
+ {spiderEnvVars.map((envVar: SpiderJobEnvVar, id: number) => + envVar.masked ? ( + + {envVar.name} + + ) : ( + handleRemoveProjectEnvVar(id, false)} + > + {envVar.name}: {envVar.value} + + ), + )} +
+ + )} {cronjobData.envVars.map((envVar: EnvVarsData, id: number) => envVar.masked ? ( @@ -839,8 +875,8 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro - -

Tags

+ +

Tags

{cronjobData.tags.map((tag: TagsData, id) => ( - +

Select a period

diff --git a/estela-web/src/pages/DeployListPage/index.tsx b/estela-web/src/pages/DeployListPage/index.tsx index 2ba4000d..47202ebd 100644 --- a/estela-web/src/pages/DeployListPage/index.tsx +++ b/estela-web/src/pages/DeployListPage/index.tsx @@ -69,14 +69,35 @@ export class DeployListPage extends Component, <> {spiders.length === 0 ? ( "-/-" - ) : spiders.length > 1 ? ( + ) : spiders.length > 2 ? ( <> - {spiders[0].name}  - +1 + {spiders.map((spider: Spider, id: number) => { + if (id > 1) { + return; + } + return ( + + {spider.name} + + ); + })} + + +{spiders.length - 2} + ) : ( <> - {spiders[0].name} + {spiders.map((spider: Spider, id: number) => ( + + {spider.name} + + ))} )} diff --git a/estela-web/src/pages/JobCreateModal/index.tsx b/estela-web/src/pages/JobCreateModal/index.tsx index f85642c5..5a02932b 100644 --- a/estela-web/src/pages/JobCreateModal/index.tsx +++ b/estela-web/src/pages/JobCreateModal/index.tsx @@ -1,5 +1,5 @@ import React, { useState, useEffect } from "react"; -import { Modal, Button, message, Row, Select, Space, Input, Tag, Checkbox, Tooltip } from "antd"; +import { Modal, Layout, Button, message, Row, Select, Space, Input, Tag, Checkbox, Tooltip } from "antd"; import type { CheckboxChangeEvent } from "antd/es/checkbox"; import { ApiProjectsSpidersJobsCreateRequest, @@ -20,12 +20,17 @@ import { checkExternalError } from "../../defaultComponents"; import Run from "../../assets/icons/play.svg"; import Add from "../../assets/icons/add.svg"; +const { Content } = Layout; const { Option } = Select; interface JobCreateModalProps { openModal: boolean; spider: Spider | null; projectId: string; + envVarsProps: SpiderJobEnvVar[]; + argsProps: ArgsData[]; + tagsProps: TagsData[]; + children: React.ReactNode; } interface MaskedTagProps { @@ -40,25 +45,14 @@ interface ArgsData { key: number; } -interface EnvVarsData { - name: string; - value: string; - key: number; - masked: boolean; -} - interface TagsData { name: string; -} - -interface Tags { - name: string; key: number; } interface JobData { args: ArgsData[]; - envVars: EnvVarsData[]; + envVars: SpiderJobEnvVar[]; tags: TagsData[]; dataStatus: SpiderDataStatusEnum | undefined; dataExpiryDays: number | null | undefined; @@ -71,7 +65,6 @@ interface Variable { newEnvVarValue: string; newEnvVarMasked: boolean; newTagName: string; - newTags: Tags[]; } interface Request { @@ -95,7 +88,15 @@ const dataPersistenceOptions = [ { label: "Forever", key: 7, value: 720 }, ]; -export default function JobCreateModal({ openModal, spider, projectId }: JobCreateModalProps) { +export default function JobCreateModal({ + openModal, + spider, + projectId, + envVarsProps, + argsProps, + tagsProps, + children, +}: JobCreateModalProps) { const PAGE_SIZE = 15; const apiService = ApiService(); const [open, setOpen] = useState(openModal); @@ -104,9 +105,9 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea const [spiders, setSpiders] = useState([]); const [externalComponent, setExternalComponent] = useState(<>); const [jobData, setJobData] = useState({ - args: [], - envVars: [], - tags: [], + args: argsProps, + envVars: envVarsProps, + tags: tagsProps, dataStatus: spider ? spider.dataStatus : undefined, dataExpiryDays: spider ? spider.dataExpiryDays : 1, }); @@ -119,7 +120,6 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea newEnvVarValue: "", newEnvVarMasked: false, newTagName: "", - newTags: [], }); const [request, setRequest] = useState({ pid: projectId, @@ -299,51 +299,57 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea setJobData({ ...jobData, tags: [...tags] }); }; - const addArgument = (): void => { + const addArgument = (): ArgsData | null => { const args = [...jobData.args]; const newArgName = variable.newArgName.trim(); const newArgValue = variable.newArgValue.trim(); if (newArgName && newArgValue && newArgName.indexOf(" ") == -1) { - args.push({ name: newArgName, value: newArgValue, key: countKey }); + const arg = { name: newArgName, value: newArgValue, key: countKey }; + args.push(arg); setCountKey(countKey + 1); setJobData({ ...jobData, args: [...args] }); setVariable({ ...variable, newArgName: "", newArgValue: "" }); + return arg; } else { invalidDataNotification("Invalid argument name/value pair."); + return null; } }; - const addEnvVar = (): void => { + const addEnvVar = (): SpiderJobEnvVar | null => { const envVars = [...jobData.envVars]; const newEnvVarName = variable.newEnvVarName.trim(); const newEnvVarValue = variable.newEnvVarValue.trim(); if (newEnvVarName && newEnvVarValue && newEnvVarName.indexOf(" ") == -1) { - envVars.push({ + const newEnvVar = { name: newEnvVarName, value: newEnvVarValue, masked: variable.newEnvVarMasked, - key: countKey, - }); + }; + envVars.push(newEnvVar); setCountKey(countKey + 1); setJobData({ ...jobData, envVars: [...envVars] }); setVariable({ ...variable, newEnvVarName: "", newEnvVarValue: "", newEnvVarMasked: false }); + return newEnvVar; } else { invalidDataNotification("Invalid environment variable name/value pair."); + return null; } }; - const addTag = (): void => { + const addTag = (): TagsData | null => { const tags = [...jobData.tags]; - const newTags = [...variable.newTags]; const newTagName = variable.newTagName.trim(); if (newTagName && newTagName.indexOf(" ") == -1) { - newTags.push({ name: newTagName, key: countKey }); + const tag = { name: newTagName, key: countKey }; + tags.push(tag); setCountKey(countKey + 1); - tags.push({ name: newTagName }); setJobData({ ...jobData, tags: [...tags] }); - setVariable({ ...variable, newTags: [...newTags], newTagName: "" }); + setVariable({ ...variable, newTagName: "" }); + return tag; } else { invalidDataNotification("Invalid tag name."); + return null; } }; @@ -369,14 +375,42 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea setVariable({ ...variable, newEnvVarMasked: checked }); }; + const addPendingFormElement = ( + elementName: string, + elementValue: string, + addElementFunction: CallableFunction, + elementList: Array, + ): boolean => { + elementName = elementName.trim(); + elementValue = elementValue.trim(); + if (elementName || elementValue) { + const element = addElementFunction(); + if (!element) { + return false; + } + elementList.push(element); + } + return true; + }; + const handleSubmit = (): void => { setLoading(true); + + if ( + !( + addPendingFormElement(variable.newArgName, variable.newArgValue, addArgument, jobData.args) && + addPendingFormElement(variable.newEnvVarName, variable.newEnvVarValue, addEnvVar, jobData.envVars) && + addPendingFormElement(variable.newTagName, "", addTag, jobData.tags) + ) + ) { + setLoading(false); + return; + } + const { args, tags, dataStatus, dataExpiryDays } = jobData; const { pid, sid } = request; - const envVarsData = projectEnvVars.map((envVar: SpiderJobEnvVar) => { - return envVar; - }); + const envVarsData = [...projectEnvVars]; spiderEnvVars.map((envVar: SpiderJobEnvVar) => { const index = envVarsData.findIndex((element: SpiderJobEnvVar) => element.name === envVar.name); if (index != -1) { @@ -385,11 +419,11 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea envVarsData.push(envVar); } }); - - jobData.envVars.map((envVar: EnvVarsData) => { + jobData.envVars.map((envVar: SpiderJobEnvVar) => { const index = envVarsData.findIndex((element: SpiderJobEnvVar) => element.name === envVar.name); if (index != -1) { envVarsData[index] = { + evid: envVar.evid, name: envVar.name, value: envVar.value, masked: envVar.masked, @@ -415,10 +449,12 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea pid: pid, sid: sid, }; + apiService.apiProjectsSpidersJobsCreate(requests).then( (response: SpiderJobCreate) => { setLoading(false); history.push(`/projects/${pid}/spiders/${sid}/jobs/${response.jid}`); + location.reload(); }, async (error) => { setLoading(false); @@ -450,7 +486,7 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea } }} > - Run new job + {children} {externalComponent} - +

Environment Variables

- -
- {projectEnvVars.length > 0 &&

Project

} -
- {projectEnvVars.map((envVar: SpiderJobEnvVar, id: number) => - envVar.masked ? ( - - {envVar.name} - - ) : ( - handleRemoveProjectEnvVar(id, true)} - > - {envVar.name}: {envVar.value} - - ), - )} -
-
-
- {spiderEnvVars.length > 0 &&

Spider

} -
- {spiderEnvVars.map((envVar: SpiderJobEnvVar, id: number) => - envVar.masked ? ( - - {envVar.name} - - ) : ( - handleRemoveProjectEnvVar(id, false)} - > - {envVar.name}: {envVar.value} - - ), - )} -
-
+ + {projectEnvVars.length > 0 && ( + +

Project

+
+ {projectEnvVars.map((envVar: SpiderJobEnvVar, id: number) => + envVar.masked ? ( + + {envVar.name} + + ) : ( + handleRemoveProjectEnvVar(id, true)} + > + {envVar.name}: {envVar.value} + + ), + )} +
+ + )} + {spiderEnvVars.length > 0 && ( + +

Spider

+
+ {spiderEnvVars.map((envVar: SpiderJobEnvVar, id: number) => + envVar.masked ? ( + + {envVar.name} + + ) : ( + handleRemoveProjectEnvVar(id, false)} + > + {envVar.name}: {envVar.value} + + ), + )} +
+ + )} - {jobData.envVars.map((envVar: EnvVarsData, id: number) => + {jobData.envVars.map((envVar: SpiderJobEnvVar, id: number) => envVar.masked ? ( - +

Tags

diff --git a/estela-web/src/pages/JobDetailPage/index.tsx b/estela-web/src/pages/JobDetailPage/index.tsx index 40106371..ef24ad44 100644 --- a/estela-web/src/pages/JobDetailPage/index.tsx +++ b/estela-web/src/pages/JobDetailPage/index.tsx @@ -1,57 +1,31 @@ import React, { Component } from "react"; -import moment from "moment"; import { Chart as ChartJS, ArcElement, Tooltip, Legend, LinearScale } from "chart.js"; import { Doughnut } from "react-chartjs-2"; -import { - Layout, - Typography, - Row, - Col, - Space, - Tag, - Button, - DatePickerProps, - Tabs, - Card, - Modal, - Select, - Input, - Tooltip as AntdTooltip, - Checkbox, -} from "antd"; -import type { RangePickerProps } from "antd/es/date-picker"; -import type { CheckboxChangeEvent } from "antd/es/checkbox"; +import JobCreateModal from "../JobCreateModal"; +import { Layout, Typography, Row, Col, Space, Tag, Button, Tabs, Card, Modal, Tooltip as AntdTooltip } from "antd"; import { Link, RouteComponentProps } from "react-router-dom"; import "./styles.scss"; -import history from "../../history"; import { ApiService } from "../../services"; import { BytesMetric, parseDuration, durationToString, formatBytes } from "../../utils"; -import Copy from "../../assets/icons/copy.svg"; import Pause from "../../assets/icons/pause.svg"; -import Add from "../../assets/icons/add.svg"; import { ApiProjectsSpidersJobsReadRequest, SpiderJobUpdateStatusEnum, - ApiProjectsSpidersJobsCreateRequest, ApiProjectsSpidersJobsUpdateRequest, ApiProjectsSpidersJobsDataListRequest, - ApiProjectsSpidersListRequest, SpiderJob, - SpiderJobCreate, SpiderJobEnvVar, - SpiderJobUpdate, - SpiderJobUpdateDataStatusEnum, - Spider, + SpiderJobArg, + SpiderJobTag, } from "../../services/api"; import { JobItemsData, JobRequestsData, JobLogsData, JobStatsData } from "../JobDataPage"; -import { resourceNotAllowedNotification, incorrectDataNotification, invalidDataNotification, Spin } from "../../shared"; +import { resourceNotAllowedNotification, incorrectDataNotification, Spin } from "../../shared"; import { convertDateToString } from "../../utils"; const { Content } = Layout; const { Text } = Typography; -const { Option } = Select; ChartJS.register(ArcElement, LinearScale, Tooltip, Legend); @@ -59,17 +33,6 @@ interface Dictionary { [Key: string]: string; } -interface OptionDataPersistence { - label: string; - key: number; - value: number; -} - -interface ArgsData { - name: string; - value: string; -} - interface Args { name: string; value: string; @@ -79,14 +42,10 @@ interface Args { interface EnvVars { name: string; value: string; - masked: boolean | undefined; + masked: boolean; key: number; } -interface TagsData { - name: string; -} - interface Tags { name: string; key: number; @@ -97,9 +56,9 @@ interface JobDetailPageState { name: string | undefined; lifespan: number | undefined; totalResponseBytes: number | undefined; - args: ArgsData[]; + args: SpiderJobArg[]; envVars: SpiderJobEnvVar[]; - tags: TagsData[]; + tags: SpiderJobTag[]; date: string; activeKey: string; created: string | undefined; @@ -113,24 +72,12 @@ interface JobDetailPageState { dataStatus: string | undefined; dataExpiryDays: number | undefined; loading_status: boolean; - modified: boolean; modalStop: boolean; modalClone: boolean; - spiders: Spider[]; - loadedSpiders: boolean; spiderName: string; - newSpiderId: string; - newDataExpireDays: number; - newDataStatus: string; newTags: Tags[]; - newTagName: string; newArgs: Args[]; - newArgName: string; - newArgValue: string; newEnvVars: EnvVars[]; - newEnvVarName: string; - newEnvVarValue: string; - newEnvVarMasked: boolean; } interface RouteParams { @@ -244,24 +191,12 @@ export class JobDetailPage extends Component, J dataStatus: "", dataExpiryDays: 0, loading_status: false, - modified: false, modalStop: false, modalClone: false, - spiders: [], - loadedSpiders: false, spiderName: "", - newSpiderId: "", - newDataExpireDays: 1, - newDataStatus: "PENDING", newTags: [], - newTagName: "", newArgs: [], - newArgName: "", - newArgValue: "", newEnvVars: [], - newEnvVarName: "", - newEnvVarValue: "", - newEnvVarMasked: false, }; apiService = ApiService(); projectId: string = this.props.match.params.projectId; @@ -289,16 +224,35 @@ export class JobDetailPage extends Component, J this.apiService.apiProjectsSpidersJobsRead(requestParams).then( async (response: SpiderJob) => { const args = response.args || []; + const newArgs = args.map((arg: SpiderJobArg, id: number) => { + return { name: arg.name, value: arg.value, key: id }; + }); const envVars = response.envVars || []; + const newEnvVars = envVars.map((envVar: SpiderJobEnvVar, id: number) => { + return { + evid: envVar.evid, + name: envVar.name, + value: envVar.masked ? "__MASKED__" : envVar.value, + masked: envVar.masked, + key: id, + }; + }); const tags = response.tags || []; - const lifespan = parseDuration(response.lifespan); + const newTags = tags.map((tag: SpiderJobTag, id: number) => { + return { name: tag.name, key: id }; + }); + const lifespan = response.lifespan; this.setState({ name: response.name, lifespan: lifespan, totalResponseBytes: response.totalResponseBytes, args: [...args], + newArgs: [...newArgs], envVars: [...envVars], + newEnvVars: [...newEnvVars], tags: [...tags], + newTags: [...newTags], + spiderName: response.spider.name, date: convertDateToString(response.created), created: `${response.created}`, status: response.jobStatus, @@ -313,244 +267,10 @@ export class JobDetailPage extends Component, J resourceNotAllowedNotification(); }, ); - this.getProjectSpiders(); await this.getItems(1); this.setState({ loadedItemsFirstTime: true }); } - getProjectSpiders = async (): Promise => { - const requestParams: ApiProjectsSpidersListRequest = { pid: this.projectId }; - this.apiService.apiProjectsSpidersList(requestParams).then( - (results) => { - if (results.results.length == 0 || results.results == undefined) { - this.setState({ spiders: [], loadedSpiders: true }); - } else { - const spiderName = results.results.find( - (spider: Spider) => String(spider?.sid) === this.spiderId, - )?.name; - this.setState({ - spiders: [...results.results], - spiderName: spiderName || "", - newSpiderId: String(results.results[0].sid), - loadedSpiders: true, - }); - } - }, - (error: unknown) => { - error; - resourceNotAllowedNotification(); - }, - ); - }; - - getData = async (): Promise => { - const requestParams: ApiProjectsSpidersJobsDataListRequest = { - pid: this.projectId, - sid: this.spiderId, - jid: `${this.jobId}`, - }; - this.apiService.apiProjectsSpidersList(requestParams).then( - (results) => { - if (results.results.length == 0 || results.results == undefined) { - this.setState({ spiders: [], loadedSpiders: true }); - } else { - const spiderName = results.results.find( - (spider: Spider) => String(spider?.sid) === this.spiderId, - )?.name; - this.setState({ - spiders: [...results.results], - spiderName: spiderName || "", - newSpiderId: String(results.results[0].sid), - loadedSpiders: true, - }); - } - }, - (error: unknown) => { - error; - resourceNotAllowedNotification(); - }, - ); - }; - - handlePersistenceChange = (value: number): void => { - if (value == 720) { - this.setState({ newDataStatus: "PERSISTENT" }); - } else { - this.setState({ newDataExpireDays: value }); - } - }; - - handleRemoveTag = (id: number): void => { - const newTags = [...this.state.newTags]; - newTags.splice(id, 1); - this.setState({ newTags: [...newTags] }); - }; - - handleRemoveArg = (id: number): void => { - const newArgs = [...this.state.newArgs]; - newArgs.splice(id, 1); - this.setState({ newArgs: [...newArgs] }); - }; - - handleRemoveEnvVar = (id: number): void => { - const newEnvVars = [...this.state.newEnvVars]; - newEnvVars.splice(id, 1); - this.setState({ newEnvVars: [...newEnvVars] }); - }; - - onChangeEnvVarMasked = (e: CheckboxChangeEvent) => { - this.setState({ newEnvVarMasked: e.target.checked }); - }; - - handleInputChange = (event: React.ChangeEvent): void => { - const { - target: { value, name }, - } = event; - if (name === "newArgName") { - this.setState({ newArgName: value }); - } else if (name === "newArgValue") { - this.setState({ newArgValue: value }); - } else if (name === "newEnvVarName") { - this.setState({ newEnvVarName: value }); - } else if (name === "newEnvVarValue") { - this.setState({ newEnvVarValue: value }); - } else if (name === "newTagName") { - this.setState({ newTagName: value }); - } - }; - - handleSubmit = (): void => { - const futureDate: Date = new Date(); - futureDate.setDate(futureDate.getDate() + this.state.newDataExpireDays); - const requestData = { - args: [...this.state.newArgs], - envVars: [...this.state.newEnvVars], - tags: [...this.state.newTags], - dataStatus: this.state.newDataStatus, - dataExpiryDays: this.state.newDataExpireDays, - }; - const request: ApiProjectsSpidersJobsCreateRequest = { - data: requestData, - pid: this.projectId, - sid: this.state.newSpiderId, - }; - this.apiService.apiProjectsSpidersJobsCreate(request).then( - (response: SpiderJobCreate) => { - history.push(`/projects/${this.projectId}/spiders/${this.state.newSpiderId}/jobs/${response.jid}`); - location.reload(); - }, - (error: unknown) => { - error; - incorrectDataNotification(); - }, - ); - }; - - handleSpiderChange = (value: string): void => { - const spiderId = this.state.spiders.find((spider) => { - return spider.name === value; - }); - this.setState({ newSpiderId: String(spiderId?.sid) }); - }; - - addTag = (): void => { - const newTags = [...this.state.newTags]; - const newTagName = this.state.newTagName.trim(); - if (newTagName && newTagName.indexOf(" ") == -1) { - newTags.push({ name: newTagName, key: this.countKey++ }); - this.setState({ newTags: [...newTags], newTagName: "" }); - } else { - invalidDataNotification("Invalid tag name."); - } - }; - - addArgument = (): void => { - const newArgs = [...this.state.newArgs]; - const newArgName = this.state.newArgName.trim(); - const newArgValue = this.state.newArgValue.trim(); - if (newArgName && newArgValue && newArgName.indexOf(" ") == -1) { - newArgs.push({ name: newArgName, value: newArgValue, key: this.countKey++ }); - this.setState({ newArgs: [...newArgs], newArgName: "", newArgValue: "" }); - } else { - invalidDataNotification("Invalid argument name/value pair."); - } - }; - - addEnvVar = (): void => { - const newEnvVars = [...this.state.newEnvVars]; - const newEnvVarName = this.state.newEnvVarName.trim(); - const newEnvVarValue = this.state.newEnvVarValue.trim(); - const newEnvVarMasked = this.state.newEnvVarMasked; - if (newEnvVarName && newEnvVarValue && newEnvVarName.indexOf(" ") == -1) { - newEnvVars.push({ - name: newEnvVarName, - value: newEnvVarValue, - masked: newEnvVarMasked, - key: this.countKey++, - }); - this.setState({ - newEnvVars: [...newEnvVars], - newEnvVarName: "", - newEnvVarValue: "", - newEnvVarMasked: false, - }); - } else { - invalidDataNotification("Invalid environment variable name/value pair."); - } - }; - - stopJob = (): void => { - const request: ApiProjectsSpidersJobsUpdateRequest = { - jid: this.jobId, - sid: this.spiderId, - pid: this.projectId, - data: { - jid: this.jobId, - status: SpiderJobUpdateStatusEnum.Stopped, - }, - }; - this.apiService.apiProjectsSpidersJobsUpdate(request).then( - (response: SpiderJobUpdate) => { - this.setState({ status: response.status }); - }, - (error: unknown) => { - error; - incorrectDataNotification(); - }, - ); - }; - - updateDataExpiry = (): void => { - this.setState({ loading_status: true }); - const requestData: SpiderJobUpdate = { - dataStatus: - this.state.dataStatus == SpiderJobUpdateDataStatusEnum.Persistent - ? this.state.dataStatus - : SpiderJobUpdateDataStatusEnum.Pending, - dataExpiryDays: this.state.dataExpiryDays, - }; - const request: ApiProjectsSpidersJobsUpdateRequest = { - jid: this.jobId, - pid: this.projectId, - sid: this.spiderId, - data: requestData, - }; - this.apiService.apiProjectsSpidersJobsUpdate(request).then( - (response: SpiderJobUpdate) => { - this.setState({ - dataStatus: response.dataStatus, - dataExpiryDays: response.dataExpiryDays == null ? 1 : response.dataExpiryDays, - modified: false, - loading_status: false, - }); - }, - (error: unknown) => { - error; - incorrectDataNotification(); - }, - ); - }; - updateStatus = (_status: SpiderJobUpdateStatusEnum): void => { this.setState({ loading_status: true }); const request: ApiProjectsSpidersJobsUpdateRequest = { @@ -572,27 +292,6 @@ export class JobDetailPage extends Component, J ); }; - onChangeData = (): void => { - const _dataStatus = - this.state.dataStatus == SpiderJobUpdateDataStatusEnum.Persistent - ? SpiderJobUpdateDataStatusEnum.Pending - : SpiderJobUpdateDataStatusEnum.Persistent; - this.setState({ dataStatus: _dataStatus, modified: true }); - }; - - onChangeDay = (value: number): void => { - this.setState({ dataExpiryDays: value, modified: true }); - }; - - disabledDate: RangePickerProps["disabledDate"] = (current) => { - return current && current < moment().endOf("day"); - }; - - onChangeDate: DatePickerProps["onChange"] = (date) => { - const days = moment.duration(moment(date, "llll").diff(moment(this.state.created, "llll"))).asDays(); - this.setState({ dataExpiryDays: days, modified: true }); - }; - getItems = async (page: number, pageSize?: number): Promise => { const requestParams: ApiProjectsSpidersJobsDataListRequest = { pid: this.projectId, @@ -753,7 +452,7 @@ export class JobDetailPage extends Component, J {date} - + Scheduled job @@ -770,13 +469,13 @@ export class JobDetailPage extends Component, J )} - + Tags - - - {tags.map((tag: TagsData, id) => ( + + + {tags.map((tag: SpiderJobTag, id) => ( , J - + Environment variables @@ -822,13 +521,13 @@ export class JobDetailPage extends Component, J - + Arguments - - - {args.map((arg: ArgsData, id) => ( + + + {args.map((arg: SpiderJobArg, id) => ( , J )} {status == "WAITING" && ( - + {status} @@ -943,7 +642,7 @@ export class JobDetailPage extends Component, J - {durationToString(lifespan || 0)} + {durationToString(parseDuration(String(lifespan || 0)))} @@ -996,28 +695,19 @@ export class JobDetailPage extends Component, J render(): JSX.Element { const { loaded, - newTagName, - status, modalStop, - modalClone, - spiders, activeKey, - loadedSpiders, newTags, newArgs, - newArgName, - newArgValue, newEnvVars, - newEnvVarName, - newEnvVarValue, - newEnvVarMasked, itemsCurrent, loadedItems, loadedItemsFirstTime, + status, } = this.state; return ( - {loaded && loadedSpiders ? ( + {loaded ? ( @@ -1027,42 +717,16 @@ export class JobDetailPage extends Component, J - + - {modalStop && ( - CONFIRM ACTION

- } - onCancel={() => this.setState({ modalStop: false })} - footer={null} - > - - - Are you sure you want to stop this job? - - - - - - - - )} - {modalClone && ( - NEW JOB

} - onCancel={() => this.setState({ modalClone: false })} - footer={null} - > - - - -

Spider

- - - -

Data persistence

- - - -

Tags

- - {newTags.map((tag: Tags, id: number) => { - return ( - this.handleRemoveTag(id)} - > - {tag.name} - - ); - })} - - - - - - - -

Arguments

- - {newArgs.map((arg: Args, id) => ( - this.handleRemoveArg(id)} - > - {arg.name}: {arg.value} - - ))} - - - - - - - - -

Environment Variables

- - {newEnvVars.map((envVar: SpiderJobEnvVar, id: number) => - envVar.masked ? ( - - this.handleRemoveEnvVar(id)} - className="environment-variables" - key={id} - > - {envVar.name} - - - ) : ( - this.handleRemoveEnvVar(id)} - className="environment-variables" - key={id} - > - {envVar.name}: {envVar.value} - - ), - )} - - - - Masked - - - - - - - - - - - - - - )} + CONFIRM ACTION

} + onCancel={() => this.setState({ modalStop: false })} + footer={null} + > + + + Are you sure you want to stop this job? + + + + + + + diff --git a/estela-web/src/pages/ProjectCronJobListPage/index.tsx b/estela-web/src/pages/ProjectCronJobListPage/index.tsx index 76f5395c..91192ef8 100644 --- a/estela-web/src/pages/ProjectCronJobListPage/index.tsx +++ b/estela-web/src/pages/ProjectCronJobListPage/index.tsx @@ -1,5 +1,5 @@ import React, { Component, ReactElement } from "react"; -import { Layout, Pagination, Row, Space, Table, Col, Button, Switch, Tag, message, ConfigProvider } from "antd"; +import { Layout, Pagination, Row, Space, Table, Col, Button, Switch, Tag, message } from "antd"; import { Link, RouteComponentProps } from "react-router-dom"; import "./styles.scss"; import history from "../../history"; @@ -24,6 +24,7 @@ import { RouteParams, } from "../../shared"; import CronjobCreateModal from "../CronjobCreateModal"; +import FolderDotted from "../../assets/icons/folderDotted.svg"; import { convertDateToString } from "../../utils"; const { Content } = Layout; @@ -68,6 +69,7 @@ interface ProjectCronJobListPageState { current: number; loading: boolean; page: number; + isEmpty: boolean; } export class ProjectCronJobListPage extends Component, ProjectCronJobListPageState> { @@ -81,6 +83,7 @@ export class ProjectCronJobListPage extends Component ( + + +

No cronjobs yet.

+ + ); + render(): JSX.Element { const { loadedCronjobs, cronjobs, selectedRows, count, current } = this.state; return ( @@ -354,8 +365,10 @@ export class ProjectCronJobListPage extends Component - -

No scheduled jobs yet.

}> + {this.state.isEmpty ? ( + {this.emptyText()} + ) : ( + - - + + )} - + + Run new job + @@ -591,6 +612,7 @@ export class ProjectJobListPage extends Component )} + {isEmpty && this.emptyText()} STATUS this.onChangeStatus(waiting, waitingJobs.length)} > @@ -623,6 +646,7 @@ export class ProjectJobListPage extends Component
this.onChangeStatus(queued, queueJobs.length)} > @@ -637,6 +661,7 @@ export class ProjectJobListPage extends Component
this.onChangeStatus(running, runningJobs.length)} > @@ -651,6 +676,7 @@ export class ProjectJobListPage extends Component
this.onChangeStatus(completed, completedJobs.length)} > @@ -665,6 +691,7 @@ export class ProjectJobListPage extends Component
this.onChangeStatus(stopped, stoppedJobs.length)} > @@ -679,6 +706,7 @@ export class ProjectJobListPage extends Component
this.onChangeStatus(withError, errorJobs.length)} > diff --git a/estela-web/src/pages/ProjectListPage/index.tsx b/estela-web/src/pages/ProjectListPage/index.tsx index 95b1becf..0f53f0f5 100644 --- a/estela-web/src/pages/ProjectListPage/index.tsx +++ b/estela-web/src/pages/ProjectListPage/index.tsx @@ -216,19 +216,19 @@ export class ProjectListPage extends Component { { this.setState({ modalWelcome: false }); }} > - + WELCOME SCRAPER! Start by creating a project to be able to deploy your spiders and start with your scraping. - + Remember to install the  {  to be able to deploy your spiders! - - estela - - {this.getFramework()} - - - - {this.renderNotificationIcon(path === "/notifications/inbox", news)} - - - -
- - -
{this.getUser()} - {this.getUserRole() !== "" && ( - - {this.getUserRole()} - - )} - - - - - - - - - ) : ( - <> - )} - +
+ +
+ + estela + + {this.getFramework()} + + + + {this.renderNotificationIcon(path === "/notifications/inbox", news)} + + + +
+ + +
{this.getUser()} + {this.getUserRole() !== "" && ( + + {this.getUserRole()} + + )} + + + + + + + + ); } }