Merge pull request #306 from ThreatConnect-Inc/develop

3.0.8 Release

bin/tcex

@@ -182,12 +182,29 @@ def init(
     app_builder: Optional[bool] = typer.Option(
         False, help='Include .appbuilderconfig file in template download.'
     ),
+    proxy_host: Optional[str] = typer.Option(
+        None, help='(Advanced) Hostname for the proxy server.'
+    ),
+    proxy_port: Optional[int] = typer.Option(
+        None, help='(Advanced) Port number for the proxy server.'
+    ),
+    proxy_user: Optional[str] = typer.Option(
+        None, help='(Advanced) Username for the proxy server.'
+    ),
+    proxy_pass: Optional[str] = typer.Option(
+        None, help='(Advanced) Password for the proxy server.'
+    ),
 ):
     """Initialize a new App from a template.
 
     Templates can be found at: https://github.com/ThreatConnect-Inc/tcex-app-templates
     """
-    tt = Template()
+    tt = Template(
+        proxy_host,
+        proxy_port,
+        proxy_user,
+        proxy_pass,
+    )
     if os.listdir(os.getcwd()) and force is False:
         tt.print_block(
             'The current directory does not appear to be empty. Apps should '
@@ -230,14 +247,31 @@ def _list(
     branch: Optional[str] = typer.Option(
         'main', help='The git branch of the tcex-app-template repository to use.'
     ),
+    proxy_host: Optional[str] = typer.Option(
+        None, help='(Advanced) Hostname for the proxy server.'
+    ),
+    proxy_port: Optional[int] = typer.Option(
+        None, help='(Advanced) Port number for the proxy server.'
+    ),
+    proxy_user: Optional[str] = typer.Option(
+        None, help='(Advanced) Username for the proxy server.'
+    ),
+    proxy_pass: Optional[str] = typer.Option(
+        None, help='(Advanced) Password for the proxy server.'
+    ),
 ):
     """List templates
 
     The template name will be pulled from tcex.json by default. If the template option
     is provided it will be used instead of the value in the tcex.json file. The tcex.json
     file will also be updated with new values.
     """
-    tt = Template()
+    tt = Template(
+        proxy_host,
+        proxy_port,
+        proxy_user,
+        proxy_pass,
+    )
     try:
         tt.list(branch, type_)
         tt.print_list(branch)
@@ -312,6 +346,18 @@ def update(
     force: bool = typer.Option(
         False, help="Update files from template even if they haven't changed."
     ),
+    proxy_host: Optional[str] = typer.Option(
+        None, help='(Advanced) Hostname for the proxy server.'
+    ),
+    proxy_port: Optional[int] = typer.Option(
+        None, help='(Advanced) Port number for the proxy server.'
+    ),
+    proxy_user: Optional[str] = typer.Option(
+        None, help='(Advanced) Username for the proxy server.'
+    ),
+    proxy_pass: Optional[str] = typer.Option(
+        None, help='(Advanced) Password for the proxy server.'
+    ),
 ):
     """Update a project with the latest template files.
 
@@ -329,7 +375,12 @@ def update(
         )
         raise typer.Exit(code=1)
 
-    tt = Template()
+    tt = Template(
+        proxy_host,
+        proxy_port,
+        proxy_user,
+        proxy_pass,
+    )
     try:
         downloads = tt.update(branch, template, type_, force)
         tt.print_title('Updating template files ...', divider=False, fg_color='white')

pyproject.toml

@@ -27,6 +27,7 @@ disable = "C0103,C0302,C0330,C0415,E0401,R0205,R0801,R0902,R0903,R0904,R0912,R09
 extension-pkg-whitelist = "pydantic"
 
 [tool.pytest.ini_options]
+# addopts = "-n auto"
 junit_family = "xunit2"
 testpaths = [
   "tests",

release_notes.md

@@ -1,5 +1,16 @@
 # Release Notes
 
+### 3.0.8
+
+-   APP-3899 - [CLI] Updated error handling on CLI when downloading template files
+-   APP-3900 - [CLI] Updated proxy support for CLI
+-   APP-3906 - [CLI] Updated Deps command to no create the requirements.lock file on error
+-   APP-3922 - [API] Updated TI Transforms to treat event_date field on some group types as a datetime transform
+-   APP-3937 - [TcEx] Pinned dependencies to prevent pickup up packages that dropped support for Python 3.6
+-   APP-3938 - [API] Updated TI Transforms to support active field on indicators
+-   APP-3939 - [API] Updated TI Transforms to support Security Labels
+-   APP-3940 - [API] Updated `api.tc.v3` module to support changes in TC 7.1
+
 ### 3.0.7
 
 -   APP-3859 - [API] Enhancements for ThreatConnect 7.x

setup.py

@@ -21,21 +21,21 @@
     readme = f.read()
 
 dev_packages = [
-    'black',
-    'codespell',
+    'black==22.8.0',
+    'codespell==2.2.1',
     'fakeredis==1.7.0',
-    'flake8',
-    'isort>=5.0.0',
-    'mako',
-    'pre-commit',
+    'flake8==5.0.4',
+    'isort==5.10.1',
+    'mako==1.1.6',
+    'pre-commit==2.17.0',
     'pydocstyle',
     'pylint>=2.5.0,<2.14.0',
-    'pytest',
+    'pytest==7.0.1',
     'pytest-cov',
     'pytest-html',
     'pytest-ordering',
-    'pytest-xdist>=2.5.0',
-    'pyupgrade',
+    'pytest-xdist==3.0.2',
+    'pyupgrade==2.31.0',
 ]
 if sys.version_info <= (3, 7):
     # these packages dropped support for 3.6

tcex/__metadata__.py

@@ -5,5 +5,5 @@
 __license__ = 'Apache License, Version 2'
 __package_name__ = 'tcex'
 __url__ = 'https://github.com/ThreatConnect-Inc/tcex'
-__version__ = '3.0.7'
+__version__ = '3.0.8'
 __download_url__ = f'https://github.com/ThreatConnect-Inc/tcex/tarball/{__version__}'

tcex/api/tc/ti_transform/model/transform_model.py

@@ -177,6 +177,7 @@ class IndicatorTransformModel(TiTransformModel, extra=Extra.forbid):
     # host
     dns_active: Optional[MetadataTransformModel] = Field(None, description='')
     whois_active: Optional[MetadataTransformModel] = Field(None, description='')
+    active: Optional[MetadataTransformModel] = Field(None, description='')
 
     # root validator that ensure at least one indicator value/summary is set
     @root_validator()

tcex/api/tc/ti_transform/transform_abc.py

@@ -237,17 +237,9 @@ def _process_attributes(self, attributes: List['AttributeTransformModel']):
             param_keys = ['type_', 'value', 'displayed', 'source']
             params = [dict(zip(param_keys, p)) for p in zip(types, values, displayed, source)]
 
-            # self.log.trace(
-            #     f'feature=transform, action=process-attributes, values={values}, '
-            #     f'types={types}, source={source}, displayed={displayed}, params={params}'
-            # )
             for param in params:
                 param = self.utils.remove_none(param)
                 if 'value' not in param:
-                    self.log.warning(
-                        'feature=transform, action=process-attribute, '
-                        f'transform={attribute.dict(exclude_unset=True)}, error=no-value'
-                    )
                     continue
 
                 if 'type_' not in param:
@@ -322,7 +314,7 @@ def _process_group(self):
             self._process_metadata('to', self.transform.to_addr)
 
         if self.transformed_item['type'] in ('Event', 'Incident'):
-            self._process_metadata('eventDate', self.transform.event_date)
+            self._process_metadata_datetime('eventDate', self.transform.event_date)
             self._process_metadata('status', self.transform.status)
 
         if self.transformed_item['type'] == 'Report':
@@ -340,6 +332,9 @@ def _process_indicator(self):
         # handle the 3 possible indicator fields
         self._process_indicator_values()
 
+        if self.transform.active:
+            self._process_metadata('active', self.transform.active)
+
         self._process_confidence(self.transform.confidence)
         self._process_rating(self.transform.rating)
 
@@ -405,20 +400,18 @@ def _process_security_labels(self, labels: List['SecurityLabelTransformModel']):
             descriptions = self._process_metadata_transform_model(
                 label.description, expected_length=len(names)
             )
-            colors = self._process_metadata_transform_model(
-                label.colors, expected_length=len(names)
-            )
+            colors = self._process_metadata_transform_model(label.color, expected_length=len(names))
 
             param_keys = ['color', 'description', 'name']
             params = [dict(zip(param_keys, p)) for p in zip(colors, descriptions, names)]
 
             for kwargs in params:
+                kwargs = self.utils.remove_none(kwargs)
                 if 'name' not in kwargs:
-                    self.log.warning(f'Attribute transform {label.dict()} did not yield a name')
                     continue
                 # strip out None params so that required params are enforced and optional
                 # params with default values are respected.
-                self.add_security_label(**self.utils.remove_none(kwargs))
+                self.add_security_label(**kwargs)
 
     def _process_tags(self, tags: List['TagTransformModel']):
         """Process Tag data"""

tcex/api/tc/v3/_gen/_gen_abc.py

@@ -175,6 +175,10 @@ def _prop_contents_updated(self) -> dict:
         # update "bad" data
         self._prop_content_update(_properties)
 
+        # Temp issue
+        if self.type_ == 'indicators':
+            _properties['enrichment']['data'][0]['type'] = 'Enrichment'
+
         # critical fix for breaking API change
         if self.type_ in [
             'case_attributes',
@@ -381,8 +385,10 @@ def _prop_content_update(self, properties: dict):
         """Update "bad" data in properties."""
         if self.type_ in ['groups']:
             # fixed fields that are missing readOnly property
-            properties['downVoteCount']['readOnly'] = True
-            properties['upVoteCount']['readOnly'] = True
+            if 'downVoteCount' in properties:
+                properties['downVoteCount']['readOnly'] = True
+            if 'upVoteCount' in properties:
+                properties['upVoteCount']['readOnly'] = True
 
         if self.type_ in ['victims']:
             # ownerName is readOnly, but readOnly is not defined in response from OPTIONS endpoint

tcex/api/tc/v3/_gen/models/_property_model.py

@@ -141,6 +141,11 @@ def __extra_data(self) -> Dict[str, str]:
         # process special types
         self.__process_special_types(self, extra)
 
+        if extra.get('typing_type') is None:
+            raise RuntimeError(
+                f'Unable to determine typing_type for name={self.name}, type={self.type}.'
+            )
+
         return extra
 
     def __calculate_methods(self):
@@ -170,7 +175,7 @@ def __process_dict_types(cls, pm: 'PropertyModel', extra: Dict[str, str]):
         types = [
             'AttributeSource',
             'DNSResolutions',
-            'Enrichment',
+            'Enrichments',
             'GeoLocation',
             'InvestigationLinks',
             'Links',
@@ -242,6 +247,16 @@ def __process_special_types(cls, pm: 'PropertyModel', extra: Dict[str, str]):
                     'typing_type': cls.__extra_format_type('datetime'),
                 }
             )
+        elif pm.type == 'Group':
+            bi += 'groups.group_model'
+            extra.update(
+                {
+                    'import_data': f'{bi} import GroupModel',
+                    'import_source': 'first-party-forward-reference',
+                    'model': f'{pm.type}Model',
+                    'typing_type': cls.__extra_format_type_model(pm.type),
+                }
+            )
         elif pm.type == 'GroupAttributes':
             bi += 'group_attributes.group_attribute_model'
             extra.update(
@@ -252,6 +267,16 @@ def __process_special_types(cls, pm: 'PropertyModel', extra: Dict[str, str]):
                     'typing_type': cls.__extra_format_type_model(pm.type),
                 }
             )
+        elif pm.type == 'Indicator':
+            bi += 'indicators.indicator_model'
+            extra.update(
+                {
+                    'import_data': f'{bi} import IndicatorModel',
+                    'import_source': 'first-party-forward-reference',
+                    'model': f'{pm.type}Model',
+                    'typing_type': cls.__extra_format_type_model(pm.type),
+                }
+            )
         elif pm.type == 'IndicatorAttributes':
             bi += 'indicator_attributes.indicator_attribute_model'
             extra.update(

tcex/api/tc/v3/cases/case_filter.py

@@ -289,6 +289,16 @@ def id_as_string(self, operator: Enum, id_as_string: str):
         """
         self._tql.add_filter('idAsString', operator, id_as_string, TqlType.STRING)
 
+    def last_updated(self, operator: Enum, last_updated: str):
+        """Filter Last Updated based on **lastUpdated** keyword.
+
+        Args:
+            operator: The operator enum for the filter.
+            last_updated: The date the case was last updated in the system.
+        """
+        last_updated = self.utils.any_to_datetime(last_updated).strftime('%Y-%m-%d %H:%M:%S')
+        self._tql.add_filter('lastUpdated', operator, last_updated, TqlType.STRING)
+
     def missing_artifact_count(self, operator: Enum, missing_artifact_count: int):
         """Filter Missing Artifact Count For Tasks based on **missingArtifactCount** keyword.
 

tcex/api/tc/v3/cases/case_model.py

@@ -193,6 +193,13 @@ class CaseModel(
         read_only=True,
         title='id',
     )
+    last_updated: Optional[datetime] = Field(
+        None,
+        allow_mutation=False,
+        description='The date and time that the Case was last updated.',
+        read_only=True,
+        title='lastUpdated',
+    )
     name: Optional[str] = Field(
         None,
         description='The name of the Case.',

tcex/api/tc/v3/group_attributes/group_attribute.py

@@ -63,6 +63,7 @@ class GroupAttribute(ObjectABC):
     Args:
         default (bool, kwargs): A flag indicating that this is the default attribute of its type
             within the object. Only applies to certain attribute and data types.
+        group (Group, kwargs): Details of group associated with attribute.
         group_id (int, kwargs): Group associated with attribute.
         pinned (bool, kwargs): A flag indicating that the attribute has been noted for importance.
         security_labels (SecurityLabels, kwargs): A list of Security Labels corresponding to the

tcex/api/tc/v3/group_attributes/group_attribute_model.py

@@ -90,6 +90,13 @@ class GroupAttributeModel(
         read_only=False,
         title='default',
     )
+    group: Optional['GroupModel'] = Field(
+        None,
+        description='Details of group associated with attribute.',
+        methods=['POST'],
+        read_only=False,
+        title='group',
+    )
     group_id: Optional[int] = Field(
         None,
         description='Group associated with attribute.',
@@ -150,6 +157,12 @@ class GroupAttributeModel(
         title='value',
     )
 
+    @validator('group', always=True)
+    def _validate_group(cls, v):
+        if not v:
+            return GroupModel()
+        return v
+
     @validator('security_labels', always=True)
     def _validate_security_labels(cls, v):
         if not v:
@@ -164,6 +177,7 @@ def _validate_user(cls, v):
 
 
 # first-party
+from tcex.api.tc.v3.groups.group_model import GroupModel
 from tcex.api.tc.v3.security.users.user_model import UserModel
 from tcex.api.tc.v3.security_labels.security_label_model import SecurityLabelsModel
 

tcex/api/tc/v3/groups/group_filter.py

@@ -236,6 +236,15 @@ def has_indicator(self):
         self._tql.add_filter('hasIndicator', TqlOperator.EQ, indicators, TqlType.SUB_QUERY)
         return indicators
 
+    def has_intel_query(self, operator: Enum, has_intel_query: int):
+        """Filter Associated User Queries based on **hasIntelQuery** keyword.
+
+        Args:
+            operator: The operator enum for the filter.
+            has_intel_query: A nested query for association to User Queries.
+        """
+        self._tql.add_filter('hasIntelQuery', operator, has_intel_query, TqlType.INTEGER)
+
     @property
     def has_security_label(self):
         """Return **SecurityLabel** for further filtering."""