SunnyQjm/cicflowmeter
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Revert "🔨 fix module error"

Browse Source 
This reverts commit 2c16978220.
This commit is contained in:
Hieu Le
2020-11-10 23:40:49 +07:00
parent 2c16978220
commit e54781bb35
11 changed files with 75 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1 -3
View File
@@ -7,6 +7,4 @@ venv/
*.ods#
*.pcap
*.pyc
.idea
.pytest_cache/
.tox/
.idea
Makefile
-11
View File
@@ -1,11 +0,0 @@
test:
PYTHONPATH=./src pytest
install:
python setup.py install
uninstall:
pip uninstall cicflowmeter -y
clean:
rm -rf *.egg-info/ dist/ build/
requirements.txt
+5
View File
@@ -1,3 +1,8 @@
numpy~=1.18
scipy~=1.4.1
scapy~=2.4.3
#matplotlib==3.1.2
#scikit-learn==0.22.1
#Keras==2.3.1
#tensorflow==2.1.0
#ijson==3.0
setup.py
+1 -125
View File
@@ -1,125 +1 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Note: To use the 'upload' functionality of this file, you must:
# $ pipenv install twine --dev
import io
import os
import sys
from shutil import rmtree
from setuptools import find_packages, setup, Command
# Package meta-data.
NAME = "cicflowmeter"
DESCRIPTION = "CICFlowMeter V3 Python Implementation"
URL = "https://github.com/me/myproject"
EMAIL = "hieulw99@gmail.com"
AUTHOR = "Le Hieu"
REQUIRES_PYTHON = ">=3.7.0"
VERSION = "0.1.0"
# What packages are required for this module to be executed?
REQUIRED = ["numpy", "scapy"]
# What packages are optional?
EXTRAS = {
# 'fancy feature': ['django'],
}
# The rest you shouldn't have to touch too much :)
# ------------------------------------------------
# Except, perhaps the License and Trove Classifiers!
# If you do change the License, remember to change the Trove Classifier for that!
here = os.path.abspath(os.path.dirname(__file__))
# Import the README and use it as the long-description.
# Note: this will only work if 'README.md' is present in your MANIFEST.in file!
try:
with io.open(os.path.join(here, "README.md"), encoding="utf-8") as f:
long_description = "\n" + f.read()
except FileNotFoundError:
long_description = DESCRIPTION
# Load the package's __version__.py module as a dictionary.
about = {}
if not VERSION:
project_slug = NAME.lower().replace("-", "_").replace(" ", "_")
with open(os.path.join(here, project_slug, "__version__.py")) as f:
exec(f.read(), about)
else:
about["__version__"] = VERSION
class UploadCommand(Command):
"""Support setup.py upload."""
description = "Build and publish the package."
user_options = []
@staticmethod
def status(s):
"""Prints things in bold."""
print("\033[1m{0}\033[0m".format(s))
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self):
try:
self.status("Removing previous builds…")
rmtree(os.path.join(here, "dist"))
except OSError:
pass
self.status("Building Source and Wheel (universal) distribution…")
os.system("{0} setup.py sdist bdist_wheel --universal".format(sys.executable))
self.status("Uploading the package to PyPI via Twine…")
os.system("twine upload dist/*")
self.status("Pushing git tags…")
os.system("git tag v{0}".format(about["__version__"]))
os.system("git push --tags")
sys.exit()
# Where the magic happens:
setup(
name=NAME,
version=about["__version__"],
description=DESCRIPTION,
long_description=long_description,
long_description_content_type="text/markdown",
author=AUTHOR,
author_email=EMAIL,
python_requires=REQUIRES_PYTHON,
url=URL,
packages=find_packages("src"),
package_dir={"cicflowmeter": "src/cicflowmeter"},
entry_points={
"console_scripts": ["cicflowmeter=cicflowmeter.sniffer:main"],
},
install_requires=REQUIRED,
extras_require=EXTRAS,
include_package_data=True,
license="MIT",
classifiers=[
"Topic :: System :: Monitoring",
"Topic :: System :: Networking",
"License :: OSI Approved :: MIT License",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.7",
],
# $ setup.py publish support.
cmdclass={
"upload": UploadCommand,
},
)
from setuptools import setup, find_packages
src/cicflowmeter/features/packet_time.py
+1
View File
@@ -2,6 +2,7 @@ from datetime import datetime
import numpy
from scipy import stats as stat
from utils import get_statistics
class PacketTime:
src/cicflowmeter/flow.py
+11 -10
View File
@@ -1,16 +1,17 @@
from enum import Enum
from typing import Any
from . import constants
from .utils import get_statistics
from .features.context.packet_direction import PacketDirection
from .features.context import packet_flow_key
from .features.flow_bytes import FlowBytes
from .features.flag_count import FlagCount
from .features.packet_count import PacketCount
from .features.packet_length import PacketLength
from .features.packet_time import PacketTime
from .features.response_time import ResponseTime
import constants
from utils import get_statistics
from features.context.packet_direction import PacketDirection
from features.context import packet_flow_key
from features.flow_bytes import FlowBytes
from features.flag_count import FlagCount
from features.packet_count import PacketCount
from features.packet_length import PacketLength
from features.packet_time import PacketTime
from features.response_time import ResponseTime
class Flow:
src/cicflowmeter/flow_session.py
+41 -14
View File
@@ -1,11 +1,14 @@
import csv
import os
from collections import defaultdict
from scapy.layers.tls.record import TLS, TLSApplicationData
from scapy.sessions import DefaultSession
from .features.context.packet_direction import PacketDirection
from .features.context.packet_flow_key import get_packet_flow_key
from .flow import Flow
from features.context.packet_direction import PacketDirection
from features.context.packet_flow_key import get_packet_flow_key
from flow import Flow
from time_series.processor import Processor
EXPIRED_UPDATE = 40
@@ -41,6 +44,16 @@ class FlowSession(DefaultSession):
if "IP" not in packet:
return
# if TLS not in packet:
# return
# if TLSApplicationData not in packet:
# return
# if len(packet[TLSApplicationData]) < 40:
# # PING frame (len = 34) or other useless frames
# return
self.packets_count += 1
# Creates a key variable to check
@@ -106,17 +119,31 @@ class FlowSession(DefaultSession):
for k in keys:
flow = self.flows.get(k)
if (
latest_time is None
or latest_time - flow.latest_timestamp > EXPIRED_UPDATE
or flow.duration > 90
):
data = flow.get_data()
if self.csv_line == 0:
self.csv_writer.writerow(data.keys())
self.csv_writer.writerow(data.values())
self.csv_line += 1
del self.flows[k]
if self.output_mode == "flow":
if (
latest_time is None
or latest_time - flow.latest_timestamp > EXPIRED_UPDATE
or flow.duration > 90
):
data = flow.get_data()
if self.csv_line == 0:
self.csv_writer.writerow(data.keys())
self.csv_writer.writerow(data.values())
self.csv_line += 1
del self.flows[k]
else:
if (
latest_time is None
or latest_time - flow.latest_timestamp > EXPIRED_UPDATE
):
output_dir = os.path.join(
self.output_file, "doh" if flow.is_doh() else "ndoh"
)
os.makedirs(output_dir, exist_ok=True)
proc = Processor(flow)
flow_clumps = proc.create_flow_clumps_container()
flow_clumps.to_json_file(output_dir)
del self.flows[k]
print("Garbage Collection Finished. Flows = {}".format(len(self.flows)))
src/cicflowmeter/sniffer.py
+15 -5
View File
@@ -1,8 +1,9 @@
import argparse
from scapy.all import load_layer
from scapy.sendrecv import AsyncSniffer
from .flow_session import generate_session_class
from flow_session import generate_session_class
def create_sniffer(input_file, input_interface, output_mode, output_file):
@@ -33,7 +34,8 @@ def main():
input_group = parser.add_mutually_exclusive_group(required=True)
input_group.add_argument(
"-i",
"-n",
"--online",
"--interface",
action="store",
dest="input_interface",
@@ -41,6 +43,7 @@ def main():
)
input_group.add_argument(
"-f",
"--offline",
"--file",
action="store",
dest="input_file",
@@ -54,14 +57,21 @@ def main():
"--flow",
action="store_const",
const="flow",
default="flow",
dest="output_mode",
help="output flows as csv",
)
output_group.add_argument(
"-s",
"--json",
"--sequence",
action="store_const",
const="sequence",
dest="output_mode",
help="output flow segments as json",
)
parser.add_argument(
"output",
help="output file name in flow mode",
"output", help="output file name (in flow mode) or directory (in sequence mode)"
)
args = parser.parse_args()
tests/data/test.pcapfile
BIN
View File
Binary file not shown.
tests/test_features.py
-28
View File
@@ -1,28 +0,0 @@
from scapy.all import IP, TCP, UDP, ICMP
from cicflowmeter.features.context.packet_flow_key import get_packet_flow_key
from cicflowmeter.features.context.packet_direction import PacketDirection
import pytest
def test_packet_flow_key():
icmp_packet = IP(src="192.168.1.2", dst="192.168.1.1") / ICMP()
tcp_packet = IP(src="192.168.1.2", dst="192.168.1.1") / TCP()
udp_packet = IP(src="192.168.1.2", dst="192.168.1.1") / UDP()
with pytest.raises(Exception):
get_packet_flow_key(icmp_packet, PacketDirection.FORWARD)
"""
get_packet_flow_key return a tuple (dest_ip, src_ip, src_port, dest_port)
"""
tcp_forward = get_packet_flow_key(tcp_packet, PacketDirection.FORWARD)
tcp_backward = get_packet_flow_key(tcp_packet, PacketDirection.REVERSE)
udp_forward = get_packet_flow_key(udp_packet, PacketDirection.FORWARD)
udp_backward = get_packet_flow_key(udp_packet, PacketDirection.REVERSE)
# Test IP match source and destination
assert tcp_forward[0] == udp_forward[0]
assert tcp_forward[0] == tcp_backward[1]
# Test Port match source and destination
assert udp_forward[2] == udp_backward[3]
assert tcp_forward[2] == tcp_backward[3]
tox.ini
-16
View File
@@ -1,16 +0,0 @@
# tox (https://tox.readthedocs.io/) is a tool for running tests
# in multiple virtualenvs. This configuration file will run the
# test suite on all supported python versions. To use it, "pip install tox"
# and then run "tox" from this directory.
[tox]
envlist = py37
[testenv]
deps =
numpy
scipy
scapy
pytest
commands =
pytest