Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Clean up CLX references in docstring #1049

Merged
merged 7 commits into from
Jul 13, 2023
Merged

Clean up CLX references in docstring #1049

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4185936
fix input type in DFPVizPostprocStage
efajardo-nv Jul 12, 2023
7c7ff8b
update viz submodule to latest commit
efajardo-nv Jul 12, 2023
b7a4c84
Merge branch 'branch-23.07' of https://github.com/nv-morpheus/Morpheu…
efajardo-nv Jul 12, 2023
1830c88
remove clx references
efajardo-nv Jul 12, 2023
19fa80d
pylint fixes
efajardo-nv Jul 13, 2023
c82efdd
Merge branch 'branch-23.07' of https://github.com/nv-morpheus/Morpheu…
efajardo-nv Jul 13, 2023
52a104c
fix error from merge
efajardo-nv Jul 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 71 additions & 71 deletions morpheus/parsers/ip.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@

import cudf

# pylint: disable=invalid-name,use-dict-literal


def ip_to_int(values):
"""
Expand All @@ -36,11 +34,11 @@ def ip_to_int(values):

Examples
--------
>>> import clx.ip
>>> import morpheus.parsers.ip as ip
>>> import cudf
>>> clx.ip.ip_to_int(cudf.Series(["192.168.0.1","10.0.0.1"]))
0 89088434
1 1585596973
>>> ip.ip_to_int(cudf.Series(["192.168.0.1","10.0.0.1"]))
0 3232235521
1 167772161
dtype: int64
"""
return cudf.Series(values.str.ip2int())
Expand All @@ -63,11 +61,11 @@ def int_to_ip(values):

Examples
--------
>>> import clx.ip
>>> import morpheus.parsers.ip as ip
>>> import cudf
>>> clx.ip.int_to_ip(cudf.Series([3232235521, 167772161]))
0 5.79.97.178
1 94.130.74.45
>>> ip.int_to_ip(cudf.Series([3232235521, 167772161]))
0 192.168.0.1
1 10.0.0.1
dtype: object
"""
return cudf.Series(values._column.int2ip())
Expand All @@ -80,25 +78,25 @@ def is_ip(ips: str):

Parameters
----------
ips : IPv4 address
ips : cudf.Series
IPv4 addresses to be checked

Returns
-------
rtype : cudf.Series, booleans
rtype : cudf.Series
Boolean values true or false

Examples
--------
>>> import clx.ip
>>> import morpheus.parsers.ip as ip
>>> import cudf
>>> clx.ip.is_ip(cudf.Series(["192.168.0.1","10.123.0"]))
>>> ip.is_ip(cudf.Series(["192.168.0.1","10.123.0"]))
0 True
1 False
dtype: bool
"""
is_ip_REGEX = r"^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$"
return ips.str.match(is_ip_REGEX)
is_ip_regex = r"^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$"
return ips.str.match(is_ip_regex)


def is_reserved(ips):
Expand All @@ -118,17 +116,17 @@ def is_reserved(ips):

Examples
--------
>>> import clx.ip
>>> import morpheus.parsers.ip as ip
>>> import cudf
>>> clx.ip.is_reserved(cudf.Series(["127.0.0.1","10.0.0.1"]))
>>> ip.is_reserved(cudf.Series(["127.0.0.1","10.0.0.1"]))
0 False
1 False
dtype: bool
"""
reserved_ipv4_REGEX = (
reserved_ipv4_regex = (
r"^(2(4[0-9]|5[0-5]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]|5[0-5]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])"
r"|2([0-4][0-9]|5[0-5]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]|5[0-5]))$")
return ips.str.match(reserved_ipv4_REGEX)
return ips.str.match(reserved_ipv4_regex)


def is_loopback(ips):
Expand All @@ -148,17 +146,17 @@ def is_loopback(ips):

Examples
--------
>>> import clx.ip
>>> import morpheus.parsers.ip as ip
>>> import cudf
>>> clx.ip.is_loopback(cudf.Series(["127.0.0.1","10.0.0.1"]))
>>> ip.is_loopback(cudf.Series(["127.0.0.1","10.0.0.1"]))
0 True
1 False
dtype: bool
"""
loopback_ipv4_REGEX = (
loopback_ipv4_regex = (
r"^127\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]|5[0-5]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]"
r"|5[0-5]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]|5[0-5]))$")
return ips.str.match(loopback_ipv4_REGEX)
return ips.str.match(loopback_ipv4_regex)


def is_link_local(ips):
Expand All @@ -178,17 +176,17 @@ def is_link_local(ips):

Examples
--------
>>> import clx.ip
>>> import morpheus.parsers.ip as ip
>>> import cudf
>>> clx.ip.is_link_local(cudf.Series(["127.0.0.1","169.254.123.123"]))
>>> ip.is_link_local(cudf.Series(["127.0.0.1","169.254.123.123"]))
0 False
1 True
dtype: bool
"""
link_local_ipv4_REGEX = (
link_local_ipv4_regex = (
r"^169\.254\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]|5[0-5]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4]"
r"[0-9]|5[0-5]))$")
return ips.str.match(link_local_ipv4_REGEX)
return ips.str.match(link_local_ipv4_regex)


def is_unspecified(ips):
Expand All @@ -208,15 +206,15 @@ def is_unspecified(ips):

Examples
--------
>>> import clx.ip
>>> import morpheus.parsers.ip as ip
>>> import cudf
>>> clx.ip.is_unspecified(cudf.Series(["127.0.0.1","10.0.0.1"]))
>>> ip.is_unspecified(cudf.Series(["127.0.0.1","10.0.0.1"]))
0 False
1 False
dtype: bool
"""
unspecified_REGEX = r"^0\.0\.0\.0$"
return ips.str.match(unspecified_REGEX)
unspecified_regex = r"^0\.0\.0\.0$"
return ips.str.match(unspecified_regex)


def is_multicast(ips):
Expand All @@ -236,17 +234,17 @@ def is_multicast(ips):

Examples
--------
>>> import clx.ip
>>> import morpheus.parsers.ip as ip
>>> import cudf
>>> clx.ip.is_multicast(cudf.Series(["127.0.0.1","224.0.0.0"]))
>>> ip.is_multicast(cudf.Series(["127.0.0.1","224.0.0.0"]))
0 False
1 True
dtype: bool
"""
is_multicast_ipv4_REGEX = (
is_multicast_ipv4_regex = (
r"^(2(2[4-9]|3[0-9]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]|5[0-5]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])"
r"|2([0-4][0-9]|5[0-5]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]|5[0-5]))$")
return ips.str.match(is_multicast_ipv4_REGEX)
return ips.str.match(is_multicast_ipv4_regex)


def is_private(ips):
Expand All @@ -266,14 +264,14 @@ def is_private(ips):

Examples
--------
>>> import clx.ip
>>> import morpheus.parsers.ip as ip
>>> import cudf
>>> clx.ip.is_private(cudf.Series(["127.0.0.1","207.46.13.151"]))
>>> ip.is_private(cudf.Series(["127.0.0.1","207.46.13.151"]))
0 True
1 False
dtype: bool
"""
private_REGEX = (
private_regex = (
r"((^0\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]|5[0-5]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]"
r"|5[0-5]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]|5[0-5]))$)|(^10\.([0-9]|[1-9][0-9]|1([0-9][0-9])|"
r"2([0-4][0-9]|5[0-5]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]|5[0-5]))\.([0-9]|[1-9][0-9]|1([0-9]"
Expand All @@ -289,7 +287,7 @@ def is_private(ips):
r"[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]|5[0-5]))$)|(^(2(4[0-9]|5[0-5]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2("
r"[0-4][0-9]|5[0-5]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]|5[0-5]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])"
r"|2([0-4][0-9]|5[0-5]))$)|(^255\.255\.255\.255$))")
return ips.str.match(private_REGEX)
return ips.str.match(private_regex)


def is_global(ips):
Expand All @@ -309,17 +307,17 @@ def is_global(ips):

Examples
--------
>>> import clx.ip
>>> import morpheus.parsers.ip as ip
>>> import cudf
>>> clx.ip.is_global(cudf.Series(["127.0.0.1","207.46.13.151"]))
>>> ip.is_global(cudf.Series(["127.0.0.1","207.46.13.151"]))
0 False
1 True
dtype: bool
"""
is_global_REGEX = (
is_global_regex = (
r"^(100\.(6[4-9]|[7-9][0-9]|1([0-1][0-9]|2[0-7]))\.([0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]|5[0-5]))\.("
r"[0-9]|[1-9][0-9]|1([0-9][0-9])|2([0-4][0-9]|5[0-5]))$)")
part1 = ips.str.match(is_global_REGEX)
part1 = ips.str.match(is_global_regex)
part2 = is_private(ips)
result = ~part1 & ~part2
return result
Expand Down Expand Up @@ -353,22 +351,24 @@ def netmask(ips, prefixlen=16):

Examples
--------
>>> import clx.ip
>>> import morpheus.parsers.ip as ip
>>> import cudf
>>> clx.ip.netmask(cudf.Series(["192.168.0.1","10.0.0.1"]), prefixlen=16)
>>> ip.netmask(cudf.Series(["192.168.0.1","10.0.0.1"]), prefixlen=16)
0 255.255.0.0
1 255.255.0.0
Name: net_mask, dtype: object
"""
_ALL_ONES = (2**32) - 1
mask_int = _ALL_ONES ^ (_ALL_ONES >> prefixlen)
all_ones = (2**32) - 1
mask_int = all_ones ^ (all_ones >> prefixlen)
df = cudf.DataFrame()
df["idx"] = ips.index
x = df.apply_rows(
_netmask_kernel,
incols=["idx"],
outcols=dict(out1=np.int64, out2=np.int64, out3=np.int64, out4=np.int64),
kwargs=dict(kwarg1=mask_int),
outcols={
"out1": np.int64, "out2": np.int64, "out3": np.int64, "out4": np.int64
},
kwargs={"kwarg1": mask_int},
)

out1 = x["out1"].astype(str)
Expand Down Expand Up @@ -406,23 +406,23 @@ def hostmask(ips, prefixlen=16):

Examples
--------
>>> import clx.ip
>>> import morpheus.parsers.ip as ip
>>> import cudf
>>> clx.ip.hostmask(cudf.Series(["192.168.0.1","10.0.0.1"], prefixlen=16)
>>> ip.hostmask(cudf.Series(["192.168.0.1","10.0.0.1"]), prefixlen=16)
0 0.0.255.255
1 0.0.255.255
Name: hostmask, dtype: object
"""
_ALL_ONES = (2**32) - 1
host_mask_int = int(_ALL_ONES ^ (_ALL_ONES >> prefixlen)) ^ _ALL_ONES
all_ones = (2**32) - 1
host_mask_int = int(all_ones ^ (all_ones >> prefixlen)) ^ all_ones
df = cudf.DataFrame()
df["idx"] = ips.index
x = df.apply_rows(
_hostmask_kernel,
incols=["idx"],
outcols=dict(out1=np.int64, out2=np.int64, out3=np.int64, out4=np.int64),
kwargs=dict(kwarg1=host_mask_int),
)
x = df.apply_rows(_hostmask_kernel,
incols=["idx"],
outcols={
"out1": np.int64, "out2": np.int64, "out3": np.int64, "out4": np.int64
},
kwargs={"kwarg1": host_mask_int})

out1 = x["out1"].astype(str)
out2 = x["out2"].astype(str)
Expand Down Expand Up @@ -459,26 +459,26 @@ def mask(ips, masks):

Examples
--------
>>> import clx.ip
>>> import morpheus.parsers.ip as ip
>>> import cudf
>>> input_ips = cudf.Series(["192.168.0.1","10.0.0.1"])
>>> input_masks = cudf.Series(["255.255.0.0", "255.255.0.0"])
>>> clx.ip.mask(input_ips, input_masks)
... 0 192.168.0.0
... 1 10.0.0.0
... Name: mask, dtype: object
>>> ip.mask(input_ips, input_masks)
0 192.168.0.0
1 10.0.0.0
Name: mask, dtype: object
"""
df = cudf.DataFrame()
df["int_mask"] = masks.str.ip2int()
df["int_ip"] = ips.str.ip2int()
df["masked_ip_int"] = df["int_mask"] & df["int_ip"]

x = df.apply_rows(
_mask_kernel,
incols=["masked_ip_int"],
outcols=dict(out1=np.int64, out2=np.int64, out3=np.int64, out4=np.int64),
kwargs=dict(kwarg1=0),
)
x = df.apply_rows(_mask_kernel,
incols=["masked_ip_int"],
outcols={
"out1": np.int64, "out2": np.int64, "out3": np.int64, "out4": np.int64
},
kwargs={"kwarg1": 0})

out1 = x["out1"].astype(str)
out2 = x["out2"].astype(str)
Expand Down
2 changes: 1 addition & 1 deletion tests/test_zeek.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ def test_parse_log_file(tmpdir):

footer = "#close^I2015-01-24-16-50-35"

fname = tmpdir.mkdir("tmp_clx_zeek_test").join("tst_zeek_conn_log.csv")
fname = tmpdir.mkdir("tmp_zeek_test").join("tst_zeek_conn_log.csv")
actual.to_csv(fname, sep="\t", index=False, header=False)

with open(fname, "r+", encoding='UTF-8') as f:
Expand Down