From faa258dcf73b9d785b87f0cc4ac47b49ef64a85e Mon Sep 17 00:00:00 2001 From: cblades-tc <41635127+cblades-tc@users.noreply.github.com> Date: Wed, 22 Feb 2023 10:02:30 -0500 Subject: [PATCH 1/8] =?UTF-8?q?APP-3899=20[CLI]=20Updated=20error=20handli?= =?UTF-8?q?ng=20on=20CLI=20when=20downloading=20templat=E2=80=A6=20(#290)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * APP-3899 [CLI] Updated error handling on CLI when downloading template files * APP-3900 [CLI] Updated proxy support for CLI --- bin/tcex | 57 +++++++++++++++++++++++++++++++++++++++++--- release_notes.md | 5 ++++ tcex/bin/dep.py | 9 ++++++- tcex/bin/deploy.py | 2 +- tcex/bin/template.py | 35 ++++++++++++++++++++++++--- 5 files changed, 100 insertions(+), 8 deletions(-) diff --git a/bin/tcex b/bin/tcex index e8061461b..315f8a28d 100644 --- a/bin/tcex +++ b/bin/tcex @@ -182,12 +182,29 @@ def init( app_builder: Optional[bool] = typer.Option( False, help='Include .appbuilderconfig file in template download.' ), + proxy_host: Optional[str] = typer.Option( + None, help='(Advanced) Hostname for the proxy server.' + ), + proxy_port: Optional[int] = typer.Option( + None, help='(Advanced) Port number for the proxy server.' + ), + proxy_user: Optional[str] = typer.Option( + None, help='(Advanced) Username for the proxy server.' + ), + proxy_pass: Optional[str] = typer.Option( + None, help='(Advanced) Password for the proxy server.' + ), ): """Initialize a new App from a template. Templates can be found at: https://github.com/ThreatConnect-Inc/tcex-app-templates """ - tt = Template() + tt = Template( + proxy_host, + proxy_port, + proxy_user, + proxy_pass, + ) if os.listdir(os.getcwd()) and force is False: tt.print_block( 'The current directory does not appear to be empty. Apps should ' @@ -230,6 +247,18 @@ def _list( branch: Optional[str] = typer.Option( 'main', help='The git branch of the tcex-app-template repository to use.' ), + proxy_host: Optional[str] = typer.Option( + None, help='(Advanced) Hostname for the proxy server.' + ), + proxy_port: Optional[int] = typer.Option( + None, help='(Advanced) Port number for the proxy server.' + ), + proxy_user: Optional[str] = typer.Option( + None, help='(Advanced) Username for the proxy server.' + ), + proxy_pass: Optional[str] = typer.Option( + None, help='(Advanced) Password for the proxy server.' + ), ): """List templates @@ -237,7 +266,12 @@ def _list( is provided it will be used instead of the value in the tcex.json file. The tcex.json file will also be updated with new values. """ - tt = Template() + tt = Template( + proxy_host, + proxy_port, + proxy_user, + proxy_pass, + ) try: tt.list(branch, type_) tt.print_list(branch) @@ -312,6 +346,18 @@ def update( force: bool = typer.Option( False, help="Update files from template even if they haven't changed." ), + proxy_host: Optional[str] = typer.Option( + None, help='(Advanced) Hostname for the proxy server.' + ), + proxy_port: Optional[int] = typer.Option( + None, help='(Advanced) Port number for the proxy server.' + ), + proxy_user: Optional[str] = typer.Option( + None, help='(Advanced) Username for the proxy server.' + ), + proxy_pass: Optional[str] = typer.Option( + None, help='(Advanced) Password for the proxy server.' + ), ): """Update a project with the latest template files. @@ -329,7 +375,12 @@ def update( ) raise typer.Exit(code=1) - tt = Template() + tt = Template( + proxy_host, + proxy_port, + proxy_user, + proxy_pass, + ) try: downloads = tt.update(branch, template, type_, force) tt.print_title('Updating template files ...', divider=False, fg_color='white') diff --git a/release_notes.md b/release_notes.md index 7bd6536d3..735e409c8 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,5 +1,10 @@ # Release Notes +### 3.0.8 + +- APP-3899 [CLI] Updated error handling on CLI when downloading template files +- APP-3900 [CLI] Updated proxy support for CLI + ### 3.0.7 - APP-3859 - [API] Enhancements for ThreatConnect 7.x diff --git a/tcex/bin/dep.py b/tcex/bin/dep.py index 685b1acf8..494b12010 100644 --- a/tcex/bin/dep.py +++ b/tcex/bin/dep.py @@ -9,7 +9,7 @@ from distutils.version import StrictVersion # pylint: disable=no-name-in-module from pathlib import Path from typing import List, Optional -from urllib.parse import quote +from urllib.parse import quote, urlsplit # third-party import typer @@ -45,6 +45,13 @@ def __init__( self.proxy_user = proxy_user self.proxy_pass = proxy_pass + if not self.proxy_host and os.environ.get('https_proxy'): + parsed_proxy_url = urlsplit(os.environ.get('https_proxy')) + self.proxy_host = parsed_proxy_url.hostname + self.proxy_port = parsed_proxy_url.port + self.proxy_user = parsed_proxy_url.username + self.proxy_pass = parsed_proxy_url.password + # properties self.env = self._env self.latest_version = None diff --git a/tcex/bin/deploy.py b/tcex/bin/deploy.py index 6f2276b4b..bbafe8994 100644 --- a/tcex/bin/deploy.py +++ b/tcex/bin/deploy.py @@ -138,6 +138,6 @@ def session(self): proxy_host=self.proxy_host, proxy_port=self.proxy_port, proxy_user=self.proxy_user, - proxy_pass=self.proxy_pass, + proxy_pass=Sensitive(self.proxy_pass), ) return TcSession(auth=self.auth, base_url=self.base_url, proxies=_proxies) diff --git a/tcex/bin/template.py b/tcex/bin/template.py index 40656e208..8e2f734bd 100644 --- a/tcex/bin/template.py +++ b/tcex/bin/template.py @@ -18,6 +18,8 @@ from tcex.app_config.models import TemplateConfigModel from tcex.backports import cached_property from tcex.bin.bin_abc import BinABC +from tcex.input.field_types import Sensitive +from tcex.pleb.proxies import proxies if TYPE_CHECKING: # pragma: no cover # third-party @@ -27,7 +29,13 @@ class Template(BinABC): """Install dependencies for App.""" - def __init__(self): + def __init__( + self, + proxy_host, + proxy_port, + proxy_user, + proxy_pass, + ): """Initialize class properties.""" super().__init__() @@ -43,6 +51,10 @@ def __init__(self): self.template_data = {} self.template_manifest_fqfn = Path('.template_manifest.json') self.username = os.getenv('GITHUB_USER') + self.proxy_host = proxy_host + self.proxy_port = proxy_port + self.proxy_user = proxy_user + self.proxy_pass = proxy_pass @cached_property def cache_valid(self) -> bool: @@ -85,6 +97,7 @@ def contents( f'response={r.text or r.reason}' ) self.errors = True + yield from [] else: for content in r.json(): # exclusion - this file is only needed for building App Builder templates @@ -173,6 +186,10 @@ def download_template_file(self, item: dict): f'status_code={r.status_code}, headers={r.headers}, ' f'response={r.text or r.reason}' ) + raise RuntimeError( + f'action=get-template-config, url={r.request.url}, status_code=' + f'{r.status_code}, reason={r.reason}' + ) # get the relative path to the file and create the parent directory if it does not exist destination = item.get('relative_path') @@ -226,7 +243,10 @@ def get_template_config( f'response={r.text or r.reason}' ) self.errors = True - return None + raise RuntimeError( + f'action=get-template-config, url={r.request.url}, status_code=' + f'{r.status_code}, reason={r.reason}' + ) try: template_config_data = yaml.safe_load(r.text) @@ -368,7 +388,10 @@ def project_sha(self) -> Optional[str]: f'response={r.text or r.reason}' ) self.errors = True - return None + raise RuntimeError( + f'action=get-template-config, url={r.request.url}, status_code=' + f'{r.status_code}, reason={r.reason}' + ) try: commits_data = r.json() @@ -383,6 +406,12 @@ def session(self) -> 'Session': """Return session object""" session = Session() session.headers.update({'Cache-Control': 'no-cache'}) + session.proxies = proxies( + proxy_host=self.proxy_host, + proxy_port=self.proxy_port, + proxy_user=self.proxy_user, + proxy_pass=Sensitive(self.proxy_pass), + ) # add auth if set (typically not require since default site is public) if self.username is not None and self.password is not None: From 7495f23a5a3b4d87dc85cff11a2b41024a237455 Mon Sep 17 00:00:00 2001 From: cblades-tc <41635127+cblades-tc@users.noreply.github.com> Date: Wed, 1 Mar 2023 15:21:30 -0500 Subject: [PATCH 2/8] APP-3906 [CLI] Don't create requirements.lock file if any errors occurred during tcex deps. (#291) --- release_notes.md | 2 ++ tcex/__metadata__.py | 2 +- tcex/bin/dep.py | 12 ++++++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/release_notes.md b/release_notes.md index 735e409c8..51798d9e5 100644 --- a/release_notes.md +++ b/release_notes.md @@ -4,6 +4,8 @@ - APP-3899 [CLI] Updated error handling on CLI when downloading template files - APP-3900 [CLI] Updated proxy support for CLI +- APP-3906 [CLI] Don't create requirements.lock file if any errors occurred during tcex deps. + ### 3.0.7 diff --git a/tcex/__metadata__.py b/tcex/__metadata__.py index c8f922053..d38dc9333 100644 --- a/tcex/__metadata__.py +++ b/tcex/__metadata__.py @@ -5,5 +5,5 @@ __license__ = 'Apache License, Version 2' __package_name__ = 'tcex' __url__ = 'https://github.com/ThreatConnect-Inc/tcex' -__version__ = '3.0.7' +__version__ = '3.0.8' __download_url__ = f'https://github.com/ThreatConnect-Inc/tcex/tarball/{__version__}' diff --git a/tcex/bin/dep.py b/tcex/bin/dep.py index 494b12010..7ccbba519 100644 --- a/tcex/bin/dep.py +++ b/tcex/bin/dep.py @@ -207,6 +207,8 @@ def has_requirements_lock(self): def install_deps(self): """Install Required Libraries using pip.""" + error = False # track if any errors have occurred and if so, don't create lock file. + # check for requirements.txt if not self.requirements_fqfn.is_file(): self.handle_error( @@ -225,7 +227,7 @@ def install_deps(self): not lib_version.python_executable.is_file() and not lib_version.python_executable.is_symlink() ): - + error = True # display error typer.secho( f'The Python executable ({lib_version.python_executable}) could not be found. ' @@ -286,7 +288,13 @@ def install_deps(self): self._create_lib_latest() if self.has_requirements_lock is False: - self.create_requirements_lock() + if error: + typer.secho( + 'Not creating requirements.lock file due to errors.', + fg=typer.colors.YELLOW, + ) + else: + self.create_requirements_lock() @property def lib_versions(self) -> List[LibVersionModel]: From 9a0178b0c359a5fb2574471fd4e9ac46174fb214 Mon Sep 17 00:00:00 2001 From: bpurdy-tc <98357712+bpurdy-tc@users.noreply.github.com> Date: Mon, 13 Mar 2023 11:15:06 -0400 Subject: [PATCH 3/8] Adding support for security labels in transformations (#296) --- tcex/api/tc/ti_transform/transform_abc.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/tcex/api/tc/ti_transform/transform_abc.py b/tcex/api/tc/ti_transform/transform_abc.py index 0c0d4d614..52686b550 100644 --- a/tcex/api/tc/ti_transform/transform_abc.py +++ b/tcex/api/tc/ti_transform/transform_abc.py @@ -237,17 +237,9 @@ def _process_attributes(self, attributes: List['AttributeTransformModel']): param_keys = ['type_', 'value', 'displayed', 'source'] params = [dict(zip(param_keys, p)) for p in zip(types, values, displayed, source)] - # self.log.trace( - # f'feature=transform, action=process-attributes, values={values}, ' - # f'types={types}, source={source}, displayed={displayed}, params={params}' - # ) for param in params: param = self.utils.remove_none(param) if 'value' not in param: - self.log.warning( - 'feature=transform, action=process-attribute, ' - f'transform={attribute.dict(exclude_unset=True)}, error=no-value' - ) continue if 'type_' not in param: @@ -405,20 +397,18 @@ def _process_security_labels(self, labels: List['SecurityLabelTransformModel']): descriptions = self._process_metadata_transform_model( label.description, expected_length=len(names) ) - colors = self._process_metadata_transform_model( - label.colors, expected_length=len(names) - ) + colors = self._process_metadata_transform_model(label.color, expected_length=len(names)) param_keys = ['color', 'description', 'name'] params = [dict(zip(param_keys, p)) for p in zip(colors, descriptions, names)] for kwargs in params: + kwargs = self.utils.remove_none(kwargs) if 'name' not in kwargs: - self.log.warning(f'Attribute transform {label.dict()} did not yield a name') continue # strip out None params so that required params are enforced and optional # params with default values are respected. - self.add_security_label(**self.utils.remove_none(kwargs)) + self.add_security_label(**kwargs) def _process_tags(self, tags: List['TagTransformModel']): """Process Tag data""" From e4e090b166f4920d6a18c52946137c7882147853 Mon Sep 17 00:00:00 2001 From: bpurdy-tc <98357712+bpurdy-tc@users.noreply.github.com> Date: Fri, 24 Mar 2023 13:37:51 -0400 Subject: [PATCH 4/8] adding support for indicator active (#298) --- tcex/api/tc/ti_transform/model/transform_model.py | 1 + tcex/api/tc/ti_transform/transform_abc.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/tcex/api/tc/ti_transform/model/transform_model.py b/tcex/api/tc/ti_transform/model/transform_model.py index 90f63a4b0..91d2f89c5 100644 --- a/tcex/api/tc/ti_transform/model/transform_model.py +++ b/tcex/api/tc/ti_transform/model/transform_model.py @@ -177,6 +177,7 @@ class IndicatorTransformModel(TiTransformModel, extra=Extra.forbid): # host dns_active: Optional[MetadataTransformModel] = Field(None, description='') whois_active: Optional[MetadataTransformModel] = Field(None, description='') + active: Optional[MetadataTransformModel] = Field(None, description='') # root validator that ensure at least one indicator value/summary is set @root_validator() diff --git a/tcex/api/tc/ti_transform/transform_abc.py b/tcex/api/tc/ti_transform/transform_abc.py index 52686b550..08adefc7b 100644 --- a/tcex/api/tc/ti_transform/transform_abc.py +++ b/tcex/api/tc/ti_transform/transform_abc.py @@ -332,6 +332,9 @@ def _process_indicator(self): # handle the 3 possible indicator fields self._process_indicator_values() + if self.transform.active: + self._process_metadata('active', self.transform.active) + self._process_confidence(self.transform.confidence) self._process_rating(self.transform.rating) From 6efc27d80c0787755c631104a2016748dddb9094 Mon Sep 17 00:00:00 2001 From: cblades-tc <41635127+cblades-tc@users.noreply.github.com> Date: Fri, 24 Mar 2023 13:45:42 -0400 Subject: [PATCH 5/8] Develop (#299) * APP-3922 update transform to use datetime for event_date field --- release_notes.md | 2 +- tcex/api/tc/ti_transform/transform_abc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/release_notes.md b/release_notes.md index 51798d9e5..84105dc59 100644 --- a/release_notes.md +++ b/release_notes.md @@ -5,7 +5,7 @@ - APP-3899 [CLI] Updated error handling on CLI when downloading template files - APP-3900 [CLI] Updated proxy support for CLI - APP-3906 [CLI] Don't create requirements.lock file if any errors occurred during tcex deps. - + APP-3922 [API] Update TI Transforms to treat event_date field on some group types as a datetime transform. ### 3.0.7 diff --git a/tcex/api/tc/ti_transform/transform_abc.py b/tcex/api/tc/ti_transform/transform_abc.py index 08adefc7b..46209912c 100644 --- a/tcex/api/tc/ti_transform/transform_abc.py +++ b/tcex/api/tc/ti_transform/transform_abc.py @@ -314,7 +314,7 @@ def _process_group(self): self._process_metadata('to', self.transform.to_addr) if self.transformed_item['type'] in ('Event', 'Incident'): - self._process_metadata('eventDate', self.transform.event_date) + self._process_metadata_datetime('eventDate', self.transform.event_date) self._process_metadata('status', self.transform.status) if self.transformed_item['type'] == 'Report': From 5077444b53f1f54c78a2164c47314abdb6ef7568 Mon Sep 17 00:00:00 2001 From: bpurdy-tc <98357712+bpurdy-tc@users.noreply.github.com> Date: Thu, 6 Apr 2023 12:34:03 -0400 Subject: [PATCH 6/8] Running gen script to update v3 api (#304) --- tcex/api/tc/v3/_gen/_gen_abc.py | 10 +++++-- tcex/api/tc/v3/_gen/models/_property_model.py | 27 ++++++++++++++++++- tcex/api/tc/v3/cases/case_filter.py | 10 +++++++ tcex/api/tc/v3/cases/case_model.py | 7 +++++ .../tc/v3/group_attributes/group_attribute.py | 1 + .../group_attributes/group_attribute_model.py | 14 ++++++++++ tcex/api/tc/v3/groups/group_filter.py | 9 +++++++ tcex/api/tc/v3/groups/group_model.py | 2 +- .../indicator_attribute.py | 1 + .../indicator_attribute_model.py | 14 ++++++++++ tcex/api/tc/v3/indicators/indicator_model.py | 7 +++++ tcex/api/tc/v3/security/users/user_filter.py | 21 --------------- 12 files changed, 98 insertions(+), 25 deletions(-) diff --git a/tcex/api/tc/v3/_gen/_gen_abc.py b/tcex/api/tc/v3/_gen/_gen_abc.py index 061c8b6f4..3ee06bef1 100644 --- a/tcex/api/tc/v3/_gen/_gen_abc.py +++ b/tcex/api/tc/v3/_gen/_gen_abc.py @@ -175,6 +175,10 @@ def _prop_contents_updated(self) -> dict: # update "bad" data self._prop_content_update(_properties) + # Temp issue + if self.type_ == 'indicators': + _properties['enrichment']['data'][0]['type'] = 'Enrichment' + # critical fix for breaking API change if self.type_ in [ 'case_attributes', @@ -381,8 +385,10 @@ def _prop_content_update(self, properties: dict): """Update "bad" data in properties.""" if self.type_ in ['groups']: # fixed fields that are missing readOnly property - properties['downVoteCount']['readOnly'] = True - properties['upVoteCount']['readOnly'] = True + if 'downVoteCount' in properties: + properties['downVoteCount']['readOnly'] = True + if 'upVoteCount' in properties: + properties['upVoteCount']['readOnly'] = True if self.type_ in ['victims']: # ownerName is readOnly, but readOnly is not defined in response from OPTIONS endpoint diff --git a/tcex/api/tc/v3/_gen/models/_property_model.py b/tcex/api/tc/v3/_gen/models/_property_model.py index b34632670..e2055b703 100644 --- a/tcex/api/tc/v3/_gen/models/_property_model.py +++ b/tcex/api/tc/v3/_gen/models/_property_model.py @@ -141,6 +141,11 @@ def __extra_data(self) -> Dict[str, str]: # process special types self.__process_special_types(self, extra) + if extra.get('typing_type') is None: + raise RuntimeError( + f'Unable to determine typing_type for name={self.name}, type={self.type}.' + ) + return extra def __calculate_methods(self): @@ -170,7 +175,7 @@ def __process_dict_types(cls, pm: 'PropertyModel', extra: Dict[str, str]): types = [ 'AttributeSource', 'DNSResolutions', - 'Enrichment', + 'Enrichments', 'GeoLocation', 'InvestigationLinks', 'Links', @@ -242,6 +247,16 @@ def __process_special_types(cls, pm: 'PropertyModel', extra: Dict[str, str]): 'typing_type': cls.__extra_format_type('datetime'), } ) + elif pm.type == 'Group': + bi += 'groups.group_model' + extra.update( + { + 'import_data': f'{bi} import GroupModel', + 'import_source': 'first-party-forward-reference', + 'model': f'{pm.type}Model', + 'typing_type': cls.__extra_format_type_model(pm.type), + } + ) elif pm.type == 'GroupAttributes': bi += 'group_attributes.group_attribute_model' extra.update( @@ -252,6 +267,16 @@ def __process_special_types(cls, pm: 'PropertyModel', extra: Dict[str, str]): 'typing_type': cls.__extra_format_type_model(pm.type), } ) + elif pm.type == 'Indicator': + bi += 'indicators.indicator_model' + extra.update( + { + 'import_data': f'{bi} import IndicatorModel', + 'import_source': 'first-party-forward-reference', + 'model': f'{pm.type}Model', + 'typing_type': cls.__extra_format_type_model(pm.type), + } + ) elif pm.type == 'IndicatorAttributes': bi += 'indicator_attributes.indicator_attribute_model' extra.update( diff --git a/tcex/api/tc/v3/cases/case_filter.py b/tcex/api/tc/v3/cases/case_filter.py index 51e6f5289..fc3672bb2 100644 --- a/tcex/api/tc/v3/cases/case_filter.py +++ b/tcex/api/tc/v3/cases/case_filter.py @@ -289,6 +289,16 @@ def id_as_string(self, operator: Enum, id_as_string: str): """ self._tql.add_filter('idAsString', operator, id_as_string, TqlType.STRING) + def last_updated(self, operator: Enum, last_updated: str): + """Filter Last Updated based on **lastUpdated** keyword. + + Args: + operator: The operator enum for the filter. + last_updated: The date the case was last updated in the system. + """ + last_updated = self.utils.any_to_datetime(last_updated).strftime('%Y-%m-%d %H:%M:%S') + self._tql.add_filter('lastUpdated', operator, last_updated, TqlType.STRING) + def missing_artifact_count(self, operator: Enum, missing_artifact_count: int): """Filter Missing Artifact Count For Tasks based on **missingArtifactCount** keyword. diff --git a/tcex/api/tc/v3/cases/case_model.py b/tcex/api/tc/v3/cases/case_model.py index 0f50d957f..7c0143911 100644 --- a/tcex/api/tc/v3/cases/case_model.py +++ b/tcex/api/tc/v3/cases/case_model.py @@ -193,6 +193,13 @@ class CaseModel( read_only=True, title='id', ) + last_updated: Optional[datetime] = Field( + None, + allow_mutation=False, + description='The date and time that the Case was last updated.', + read_only=True, + title='lastUpdated', + ) name: Optional[str] = Field( None, description='The name of the Case.', diff --git a/tcex/api/tc/v3/group_attributes/group_attribute.py b/tcex/api/tc/v3/group_attributes/group_attribute.py index 2783144ec..ff6744222 100644 --- a/tcex/api/tc/v3/group_attributes/group_attribute.py +++ b/tcex/api/tc/v3/group_attributes/group_attribute.py @@ -63,6 +63,7 @@ class GroupAttribute(ObjectABC): Args: default (bool, kwargs): A flag indicating that this is the default attribute of its type within the object. Only applies to certain attribute and data types. + group (Group, kwargs): Details of group associated with attribute. group_id (int, kwargs): Group associated with attribute. pinned (bool, kwargs): A flag indicating that the attribute has been noted for importance. security_labels (SecurityLabels, kwargs): A list of Security Labels corresponding to the diff --git a/tcex/api/tc/v3/group_attributes/group_attribute_model.py b/tcex/api/tc/v3/group_attributes/group_attribute_model.py index c5046c778..fc12f1847 100644 --- a/tcex/api/tc/v3/group_attributes/group_attribute_model.py +++ b/tcex/api/tc/v3/group_attributes/group_attribute_model.py @@ -90,6 +90,13 @@ class GroupAttributeModel( read_only=False, title='default', ) + group: Optional['GroupModel'] = Field( + None, + description='Details of group associated with attribute.', + methods=['POST'], + read_only=False, + title='group', + ) group_id: Optional[int] = Field( None, description='Group associated with attribute.', @@ -150,6 +157,12 @@ class GroupAttributeModel( title='value', ) + @validator('group', always=True) + def _validate_group(cls, v): + if not v: + return GroupModel() + return v + @validator('security_labels', always=True) def _validate_security_labels(cls, v): if not v: @@ -164,6 +177,7 @@ def _validate_user(cls, v): # first-party +from tcex.api.tc.v3.groups.group_model import GroupModel from tcex.api.tc.v3.security.users.user_model import UserModel from tcex.api.tc.v3.security_labels.security_label_model import SecurityLabelsModel diff --git a/tcex/api/tc/v3/groups/group_filter.py b/tcex/api/tc/v3/groups/group_filter.py index 12b0a7ccd..22323a03a 100644 --- a/tcex/api/tc/v3/groups/group_filter.py +++ b/tcex/api/tc/v3/groups/group_filter.py @@ -236,6 +236,15 @@ def has_indicator(self): self._tql.add_filter('hasIndicator', TqlOperator.EQ, indicators, TqlType.SUB_QUERY) return indicators + def has_intel_query(self, operator: Enum, has_intel_query: int): + """Filter Associated User Queries based on **hasIntelQuery** keyword. + + Args: + operator: The operator enum for the filter. + has_intel_query: A nested query for association to User Queries. + """ + self._tql.add_filter('hasIntelQuery', operator, has_intel_query, TqlType.INTEGER) + @property def has_security_label(self): """Return **SecurityLabel** for further filtering.""" diff --git a/tcex/api/tc/v3/groups/group_model.py b/tcex/api/tc/v3/groups/group_model.py index 46b9bae73..e31dd55a0 100644 --- a/tcex/api/tc/v3/groups/group_model.py +++ b/tcex/api/tc/v3/groups/group_model.py @@ -137,7 +137,7 @@ class GroupModel( date_added: Optional[datetime] = Field( None, allow_mutation=False, - description='The date and time that the Entity was first created.', + description='The date and time that the item was first created.', read_only=True, title='dateAdded', ) diff --git a/tcex/api/tc/v3/indicator_attributes/indicator_attribute.py b/tcex/api/tc/v3/indicator_attributes/indicator_attribute.py index 548c53266..68584a3e9 100644 --- a/tcex/api/tc/v3/indicator_attributes/indicator_attribute.py +++ b/tcex/api/tc/v3/indicator_attributes/indicator_attribute.py @@ -63,6 +63,7 @@ class IndicatorAttribute(ObjectABC): Args: default (bool, kwargs): A flag indicating that this is the default attribute of its type within the object. Only applies to certain attribute and data types. + indicator (Indicator, kwargs): Details of indicator associated with attribute. indicator_id (int, kwargs): Indicator associated with attribute. pinned (bool, kwargs): A flag indicating that the attribute has been noted for importance. security_labels (SecurityLabels, kwargs): A list of Security Labels corresponding to the diff --git a/tcex/api/tc/v3/indicator_attributes/indicator_attribute_model.py b/tcex/api/tc/v3/indicator_attributes/indicator_attribute_model.py index 2698e2edd..b2f61bb9f 100644 --- a/tcex/api/tc/v3/indicator_attributes/indicator_attribute_model.py +++ b/tcex/api/tc/v3/indicator_attributes/indicator_attribute_model.py @@ -96,6 +96,13 @@ class IndicatorAttributeModel( read_only=True, title='id', ) + indicator: Optional['IndicatorModel'] = Field( + None, + description='Details of indicator associated with attribute.', + methods=['POST'], + read_only=False, + title='indicator', + ) indicator_id: Optional[int] = Field( None, description='Indicator associated with attribute.', @@ -150,6 +157,12 @@ class IndicatorAttributeModel( title='value', ) + @validator('indicator', always=True) + def _validate_indicator(cls, v): + if not v: + return IndicatorModel() + return v + @validator('security_labels', always=True) def _validate_security_labels(cls, v): if not v: @@ -164,6 +177,7 @@ def _validate_user(cls, v): # first-party +from tcex.api.tc.v3.indicators.indicator_model import IndicatorModel from tcex.api.tc.v3.security.users.user_model import UserModel from tcex.api.tc.v3.security_labels.security_label_model import SecurityLabelsModel diff --git a/tcex/api/tc/v3/indicators/indicator_model.py b/tcex/api/tc/v3/indicators/indicator_model.py index 37cb7503d..c72df338e 100644 --- a/tcex/api/tc/v3/indicators/indicator_model.py +++ b/tcex/api/tc/v3/indicators/indicator_model.py @@ -429,6 +429,13 @@ class IndicatorModel( read_only=True, title='threatAssessScoreObserved', ) + tracked_users: Optional[dict] = Field( + None, + allow_mutation=False, + description='List of tracked users and their observation and false positive stats.', + read_only=True, + title='trackedUsers', + ) type: Optional[str] = Field( None, description='The **type** for the Indicator.', diff --git a/tcex/api/tc/v3/security/users/user_filter.py b/tcex/api/tc/v3/security/users/user_filter.py index 11559b3de..a47c38363 100644 --- a/tcex/api/tc/v3/security/users/user_filter.py +++ b/tcex/api/tc/v3/security/users/user_filter.py @@ -110,18 +110,6 @@ def locked(self, operator: Enum, locked: bool): """ self._tql.add_filter('locked', operator, locked, TqlType.BOOLEAN) - def logout_interval_minutes(self, operator: Enum, logout_interval_minutes: int): - """Filter Logout Interval based on **logoutIntervalMinutes** keyword. - - Args: - operator: The operator enum for the filter. - logout_interval_minutes: The configured period of time to wait for idle activity before - being logged out. - """ - self._tql.add_filter( - 'logoutIntervalMinutes', operator, logout_interval_minutes, TqlType.INTEGER - ) - def password_reset_required(self, operator: Enum, password_reset_required: bool): """Filter Password Reset Required based on **passwordResetRequired** keyword. @@ -182,15 +170,6 @@ def tql_timeout(self, operator: Enum, tql_timeout: int): """ self._tql.add_filter('tqlTimeout', operator, tql_timeout, TqlType.INTEGER) - def ui_theme(self, operator: Enum, ui_theme: str): - """Filter UI Theme based on **uiTheme** keyword. - - Args: - operator: The operator enum for the filter. - ui_theme: The user's configured theme (e.g. light/dark). - """ - self._tql.add_filter('uiTheme', operator, ui_theme, TqlType.STRING) - def user_name(self, operator: Enum, user_name: str): """Filter User Name based on **userName** keyword. From 7e48e8b13e25ad0f37c24c5fdaf76e081d05a211 Mon Sep 17 00:00:00 2001 From: cblades-tc <41635127+cblades-tc@users.noreply.github.com> Date: Thu, 6 Apr 2023 12:39:52 -0400 Subject: [PATCH 7/8] pin dev dependencies to specific versions that support 3.6 (#301) * pin dev dependencies to specific versions that support 3.6 --- setup.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index de8ea72be..2c8c04efa 100644 --- a/setup.py +++ b/setup.py @@ -21,21 +21,21 @@ readme = f.read() dev_packages = [ - 'black', - 'codespell', + 'black==22.8.0', + 'codespell==2.2.1', 'fakeredis==1.7.0', - 'flake8', - 'isort>=5.0.0', - 'mako', - 'pre-commit', + 'flake8==5.0.4', + 'isort==5.10.1', + 'mako==1.1.6', + 'pre-commit==2.17.0', 'pydocstyle', 'pylint>=2.5.0,<2.14.0', - 'pytest', + 'pytest==7.0.1', 'pytest-cov', 'pytest-html', 'pytest-ordering', - 'pytest-xdist>=2.5.0', - 'pyupgrade', + 'pytest-xdist==3.0.2', + 'pyupgrade==2.31.0', ] if sys.version_info <= (3, 7): # these packages dropped support for 3.6 From 783e0b44607d32beccf240d37e77dd2be76763df Mon Sep 17 00:00:00 2001 From: Bracey Summers Date: Thu, 6 Apr 2023 17:44:18 -0500 Subject: [PATCH 8/8] APP-3898 - updated release notes; addressed failing test cases; minor update to V3 create/delete/update methods --- pyproject.toml | 1 + release_notes.md | 12 ++++++++---- tcex/api/tc/v3/object_abc.py | 3 +++ tests/api/tc/v3/groups/test_group_interface.py | 2 +- .../api/tc/v3/indicators/test_indicator_snippets.py | 6 +++--- tests/api/tc/v3/v3_helpers.py | 8 ++++---- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fff425600..2650805c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ disable = "C0103,C0302,C0330,C0415,E0401,R0205,R0801,R0902,R0903,R0904,R0912,R09 extension-pkg-whitelist = "pydantic" [tool.pytest.ini_options] +# addopts = "-n auto" junit_family = "xunit2" testpaths = [ "tests", diff --git a/release_notes.md b/release_notes.md index 84105dc59..5e879a485 100644 --- a/release_notes.md +++ b/release_notes.md @@ -2,10 +2,14 @@ ### 3.0.8 -- APP-3899 [CLI] Updated error handling on CLI when downloading template files -- APP-3900 [CLI] Updated proxy support for CLI -- APP-3906 [CLI] Don't create requirements.lock file if any errors occurred during tcex deps. - APP-3922 [API] Update TI Transforms to treat event_date field on some group types as a datetime transform. +- APP-3899 - [CLI] Updated error handling on CLI when downloading template files +- APP-3900 - [CLI] Updated proxy support for CLI +- APP-3906 - [CLI] Updated Deps command to no create the requirements.lock file on error +- APP-3922 - [API] Updated TI Transforms to treat event_date field on some group types as a datetime transform +- APP-3937 - [TcEx] Pinned dependencies to prevent pickup up packages that dropped support for Python 3.6 +- APP-3938 - [API] Updated TI Transforms to support active field on indicators +- APP-3939 - [API] Updated TI Transforms to support Security Labels +- APP-3940 - [API] Updated `api.tc.v3` module to support changes in TC 7.1 ### 3.0.7 diff --git a/tcex/api/tc/v3/object_abc.py b/tcex/api/tc/v3/object_abc.py index c2f253989..254e96701 100644 --- a/tcex/api/tc/v3/object_abc.py +++ b/tcex/api/tc/v3/object_abc.py @@ -162,6 +162,8 @@ def create(self, params: Optional[dict] = None) -> 'Response': """ method = 'POST' body = self.model.gen_body_json(method=method) + + params = self.gen_params(params or {}) self._request( method, self.url(method), @@ -345,6 +347,7 @@ def update(self, mode: Optional[str] = None, params: Optional[dict] = None) -> ' # validate an id is available self._validate_id(unique_id, self.url(method)) + params = self.gen_params(params or {}) self._request( method, self.url(method, unique_id=unique_id), diff --git a/tests/api/tc/v3/groups/test_group_interface.py b/tests/api/tc/v3/groups/test_group_interface.py index 3b48d178f..999b4d332 100644 --- a/tests/api/tc/v3/groups/test_group_interface.py +++ b/tests/api/tc/v3/groups/test_group_interface.py @@ -197,7 +197,7 @@ def test_victim_asset_associations(self): ) staged_victim.stage_victim_asset(asset) - staged_victim.update(params={'owner': 'TCI'}) + staged_victim.update(params={'owner': 'TCI', 'fields': ['_all_']}) assert asset.as_entity.get('type') == 'Victim Asset : EmailAddress' assert asset.as_entity.get('value') == 'Trojan : malware@example.com' diff --git a/tests/api/tc/v3/indicators/test_indicator_snippets.py b/tests/api/tc/v3/indicators/test_indicator_snippets.py index 73b84cafa..140b0a491 100644 --- a/tests/api/tc/v3/indicators/test_indicator_snippets.py +++ b/tests/api/tc/v3/indicators/test_indicator_snippets.py @@ -296,7 +296,7 @@ def test_add_indicator_file_actions(self): # Begin Snippet relationship_type = 'File DNS Query' indicator.stage_file_action({'relationship': relationship_type, 'indicator': host.model}) - indicator.update() + indicator.update(params={'fields': ['_all_']}) # End Snippet assert len(indicator.model.file_actions.data) == 1 @@ -323,7 +323,7 @@ def test_remove_indicator_file_actions(self): # [Stage Testing] - Stage an the file action to be removed indicator.stage_file_action({'relationship': relationship_type, 'indicator': host.model}) # [Delete Testing] - Delete the newly staged file action - indicator.update(mode='delete') + indicator.update(mode='delete', params={'fields': ['_all_']}) # End Snippet assert len(indicator.model.file_actions.data) == 1 @@ -349,7 +349,7 @@ def test_replace_indicator_file_actions(self): # [Stage Testing] - Stage an the file action. This will replace the existing file actions. indicator.stage_file_action({'relationship': relationship_type, 'indicator': host_2.model}) # [Replace Testing] - Replace all the current file actions with the new file actions. - indicator.update(mode='replace') + indicator.update(mode='replace', params={'fields': ['_all_']}) # End Snippet assert len(indicator.model.file_actions.data) == 1 diff --git a/tests/api/tc/v3/v3_helpers.py b/tests/api/tc/v3/v3_helpers.py index aa06e19ee..0d9f5ce63 100644 --- a/tests/api/tc/v3/v3_helpers.py +++ b/tests/api/tc/v3/v3_helpers.py @@ -409,7 +409,7 @@ def create_case(self, **kwargs): case.stage_task(self.v3.task(**task)) # create object - case.create(kwargs.get('params', {})) + case.create(params={'fields': ['_all_']}) # store case id for cleanup self._v3_objects.append(case) @@ -482,7 +482,7 @@ def create_group(self, type_: Optional[str] = 'Adversary', **kwargs): group.stage_tag(self.v3.tag(**tag)) # create object - group.create() + group.create(params={'fields': ['_all_']}) # store case id for cleanup self._v3_objects.append(group) @@ -589,7 +589,7 @@ def value_3_map(): indicator.stage_tag(self.v3.tag(**tag)) # create object - indicator.create() + indicator.create(params={'fields': ['_all_']}) # store case id for cleanup self._v3_objects.append(indicator) @@ -669,7 +669,7 @@ def create_victim(self, **kwargs): victim.stage_tag(self.v3.tag(**tag)) # create object - victim.create() + victim.create(params={'fields': ['_all_']}) # store case id for cleanup self._v3_objects.append(victim)