Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactored FilterTags.tsx to sort filters labels in a predefined order (CRASM-931) #741

Open
wants to merge 8 commits into
base: develop
Choose a base branch
from
3 changes: 3 additions & 0 deletions backend/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ syncdb:
syncdb-populate:
docker compose exec backend python manage.py syncdb --populate

syncdb-dangerously-force:
docker compose exec backend python manage.py syncdb --dangerouslyforce

# Pytest
# i.e. make pytest FILE=xfd_api/tests/test_domain.py
pytest:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Populate command."""
# Third-Party Libraries
from django.core.management.base import BaseCommand
from xfd_api.tasks.searchSync import handler as sync_es_domains
from xfd_api.tasks.syncdb_helpers import (
drop_all_tables,
manage_elasticsearch_indices,
Expand Down Expand Up @@ -53,5 +54,8 @@ def handle(self, *args, **options):
populate_sample_data()
self.stdout.write("Sample data population complete.")

# Step 4: Sync organizations in ES
sync_es_organizations()
# Step 4: Sync organizations in ES
sync_es_organizations()

# Step 5: Sync domains in ES
sync_es_domains({})
36 changes: 21 additions & 15 deletions backend/src/xfd_django/xfd_api/tasks/es_client.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
"""ES client."""
# Standard Python Libraries
import logging
import os

# Third-Party Libraries
Expand All @@ -10,9 +9,6 @@
DOMAINS_INDEX = "domains-5"
ORGANIZATIONS_INDEX = "organizations-1"

# Configure logging
logging.basicConfig(level=logging.INFO)

# Define mappings
organization_mapping = {
"properties": {"name": {"type": "text"}, "suggest": {"type": "completion"}}
Expand Down Expand Up @@ -41,7 +37,7 @@ def sync_organizations_index(self):
"""Create or updates the organizations index with mappings."""
try:
if not self.client.indices.exists(index=ORGANIZATIONS_INDEX):
logging.info(f"Creating index {ORGANIZATIONS_INDEX}...")
print(f"Creating index {ORGANIZATIONS_INDEX}...")
self.client.indices.create(
index=ORGANIZATIONS_INDEX,
body={
Expand All @@ -50,19 +46,19 @@ def sync_organizations_index(self):
},
)
else:
logging.info(f"Updating index {ORGANIZATIONS_INDEX}...")
print(f"Updating index {ORGANIZATIONS_INDEX}...")
self.client.indices.put_mapping(
index=ORGANIZATIONS_INDEX, body=organization_mapping
)
except Exception as e:
logging.error(f"Error syncing organizations index: {e}")
print(f"Error syncing organizations index: {e}")
raise e

def sync_domains_index(self):
"""Create or updates the domains index with mappings."""
try:
if not self.client.indices.exists(index=DOMAINS_INDEX):
logging.info(f"Creating index {DOMAINS_INDEX}...")
print(f"Creating index {DOMAINS_INDEX}...")
self.client.indices.create(
index=DOMAINS_INDEX,
body={
Expand All @@ -71,7 +67,7 @@ def sync_domains_index(self):
},
)
else:
logging.info(f"Updating index {DOMAINS_INDEX}...")
print(f"Updating index {DOMAINS_INDEX}...")
self.client.indices.put_mapping(
index=DOMAINS_INDEX, body=domain_mapping
)
Expand All @@ -80,7 +76,7 @@ def sync_domains_index(self):
index=DOMAINS_INDEX, body={"settings": {"refresh_interval": "1800s"}}
)
except Exception as e:
logging.error(f"Error syncing domains index: {e}")
print(f"Error syncing domains index: {e}")
raise e

def update_organizations(self, organizations):
Expand Down Expand Up @@ -140,10 +136,10 @@ def update_webpages(self, webpages):
def delete_all(self):
"""Delete all indices in Elasticsearch."""
try:
logging.info("Deleting all indices...")
print("Deleting all indices...")
self.client.indices.delete(index="*")
except Exception as e:
logging.error(f"Error deleting all indices: {e}")
print(f"Error deleting all indices: {e}")
raise e

def search_domains(self, body):
Expand All @@ -157,8 +153,18 @@ def search_organizations(self, body):
def _bulk_update(self, actions):
"""Update to Elasticsearch."""
try:
helpers.bulk(self.client, actions, raise_on_error=True)
logging.info("Bulk update completed successfully.")
success_count, response = helpers.bulk(
self.client, actions, raise_on_error=False
)
print(f"Bulk operation success count: {success_count}")

for idx, item in enumerate(response):
if "update" in item and item["update"].get("error"):
print(f"Error indexing document {idx}: {item['update']['error']}")
else:
print(f"Successfully indexed document {idx}: {item}")

self.client.indices.refresh(index="domains-5")
except Exception as e:
logging.error(f"Bulk operation error: {e}")
print(f"Bulk operation error: {e}")
raise e
2 changes: 1 addition & 1 deletion backend/src/xfd_django/xfd_api/tasks/searchSync.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def chunked_queryset(queryset, chunk_size):
yield [first] + list(islice(it, chunk_size - 1))


async def handler(command_options):
def handler(command_options):
"""Handle the synchronization of domains with Elasticsearch."""
organization_id = command_options.get("organizationId")
domain_id = command_options.get("domainId")
Expand Down
20 changes: 19 additions & 1 deletion frontend/src/pages/Search/FilterTags.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,23 @@ type FlatFilters = {
type: 'all' | 'none' | 'any';
}[];

const filterOrder = [
'Region',
'Organization',
'IP',
'Name',
'Root Domain(s)',
'Port',
'CVE',
'Severity'
];

const sortFiltersByOrder = (filters: FlatFilters) => {
return filters.sort((a, b) => {
return filterOrder.indexOf(a.label) - filterOrder.indexOf(b.label);
});
};

export const FilterTags: React.FC<Props> = ({ filters, removeFilter }) => {
const { userLevel } = useUserLevel();

Expand All @@ -134,7 +151,7 @@ export const FilterTags: React.FC<Props> = ({ filters, removeFilter }) => {
}, [userLevel]);

const filtersByColumn: FlatFilters = useMemo(() => {
return filters.reduce((acc, nextFilter) => {
const processedFilters = filters.reduce((acc, nextFilter) => {
const fieldAccessors = FIELD_TO_LABEL_MAP[nextFilter.field] ?? null;
const value = fieldAccessors
? ellipsisPastIndex(
Expand All @@ -156,6 +173,7 @@ export const FilterTags: React.FC<Props> = ({ filters, removeFilter }) => {
}
];
}, []);
return sortFiltersByOrder(processedFilters);
}, [filters]);

return (
Expand Down
Loading