diff --git a/.gitignore b/.gitignore index 5d0cc85..c163cea 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,4 @@ venv/ *.ods# *.pcap *.pyc -.idea -.pytest_cache/ -.tox/ +.idea \ No newline at end of file diff --git a/Makefile b/Makefile deleted file mode 100644 index 9c28fc6..0000000 --- a/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -test: - PYTHONPATH=./src pytest - -install: - python setup.py install - -uninstall: - pip uninstall cicflowmeter -y - -clean: - rm -rf *.egg-info/ dist/ build/ diff --git a/requirements.txt b/requirements.txt index 09aff96..23fbfc0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,8 @@ numpy~=1.18 scipy~=1.4.1 scapy~=2.4.3 +#matplotlib==3.1.2 +#scikit-learn==0.22.1 +#Keras==2.3.1 +#tensorflow==2.1.0 +#ijson==3.0 diff --git a/setup.py b/setup.py index 792ff50..4edff24 100644 --- a/setup.py +++ b/setup.py @@ -1,125 +1 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Note: To use the 'upload' functionality of this file, you must: -# $ pipenv install twine --dev - -import io -import os -import sys -from shutil import rmtree - -from setuptools import find_packages, setup, Command - -# Package meta-data. -NAME = "cicflowmeter" -DESCRIPTION = "CICFlowMeter V3 Python Implementation" -URL = "https://github.com/me/myproject" -EMAIL = "hieulw99@gmail.com" -AUTHOR = "Le Hieu" -REQUIRES_PYTHON = ">=3.7.0" -VERSION = "0.1.0" - -# What packages are required for this module to be executed? -REQUIRED = ["numpy", "scapy"] - -# What packages are optional? -EXTRAS = { - # 'fancy feature': ['django'], -} - -# The rest you shouldn't have to touch too much :) -# ------------------------------------------------ -# Except, perhaps the License and Trove Classifiers! -# If you do change the License, remember to change the Trove Classifier for that! - -here = os.path.abspath(os.path.dirname(__file__)) - -# Import the README and use it as the long-description. -# Note: this will only work if 'README.md' is present in your MANIFEST.in file! -try: - with io.open(os.path.join(here, "README.md"), encoding="utf-8") as f: - long_description = "\n" + f.read() -except FileNotFoundError: - long_description = DESCRIPTION - -# Load the package's __version__.py module as a dictionary. -about = {} -if not VERSION: - project_slug = NAME.lower().replace("-", "_").replace(" ", "_") - with open(os.path.join(here, project_slug, "__version__.py")) as f: - exec(f.read(), about) -else: - about["__version__"] = VERSION - - -class UploadCommand(Command): - """Support setup.py upload.""" - - description = "Build and publish the package." - user_options = [] - - @staticmethod - def status(s): - """Prints things in bold.""" - print("\033[1m{0}\033[0m".format(s)) - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - try: - self.status("Removing previous builds…") - rmtree(os.path.join(here, "dist")) - except OSError: - pass - - self.status("Building Source and Wheel (universal) distribution…") - os.system("{0} setup.py sdist bdist_wheel --universal".format(sys.executable)) - - self.status("Uploading the package to PyPI via Twine…") - os.system("twine upload dist/*") - - self.status("Pushing git tags…") - os.system("git tag v{0}".format(about["__version__"])) - os.system("git push --tags") - - sys.exit() - - -# Where the magic happens: -setup( - name=NAME, - version=about["__version__"], - description=DESCRIPTION, - long_description=long_description, - long_description_content_type="text/markdown", - author=AUTHOR, - author_email=EMAIL, - python_requires=REQUIRES_PYTHON, - url=URL, - packages=find_packages("src"), - package_dir={"cicflowmeter": "src/cicflowmeter"}, - entry_points={ - "console_scripts": ["cicflowmeter=cicflowmeter.sniffer:main"], - }, - install_requires=REQUIRED, - extras_require=EXTRAS, - include_package_data=True, - license="MIT", - classifiers=[ - "Topic :: System :: Monitoring", - "Topic :: System :: Networking", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - ], - # $ setup.py publish support. - cmdclass={ - "upload": UploadCommand, - }, -) +from setuptools import setup, find_packages diff --git a/src/cicflowmeter/features/packet_time.py b/src/cicflowmeter/features/packet_time.py index d89bc4d..a2c7213 100644 --- a/src/cicflowmeter/features/packet_time.py +++ b/src/cicflowmeter/features/packet_time.py @@ -2,6 +2,7 @@ from datetime import datetime import numpy from scipy import stats as stat +from utils import get_statistics class PacketTime: diff --git a/src/cicflowmeter/flow.py b/src/cicflowmeter/flow.py index cd035eb..1411340 100644 --- a/src/cicflowmeter/flow.py +++ b/src/cicflowmeter/flow.py @@ -1,16 +1,17 @@ from enum import Enum from typing import Any -from . import constants -from .utils import get_statistics -from .features.context.packet_direction import PacketDirection -from .features.context import packet_flow_key -from .features.flow_bytes import FlowBytes -from .features.flag_count import FlagCount -from .features.packet_count import PacketCount -from .features.packet_length import PacketLength -from .features.packet_time import PacketTime -from .features.response_time import ResponseTime +import constants + +from utils import get_statistics +from features.context.packet_direction import PacketDirection +from features.context import packet_flow_key +from features.flow_bytes import FlowBytes +from features.flag_count import FlagCount +from features.packet_count import PacketCount +from features.packet_length import PacketLength +from features.packet_time import PacketTime +from features.response_time import ResponseTime class Flow: diff --git a/src/cicflowmeter/flow_session.py b/src/cicflowmeter/flow_session.py index fea1935..8a8ada5 100644 --- a/src/cicflowmeter/flow_session.py +++ b/src/cicflowmeter/flow_session.py @@ -1,11 +1,14 @@ import csv +import os from collections import defaultdict +from scapy.layers.tls.record import TLS, TLSApplicationData from scapy.sessions import DefaultSession -from .features.context.packet_direction import PacketDirection -from .features.context.packet_flow_key import get_packet_flow_key -from .flow import Flow +from features.context.packet_direction import PacketDirection +from features.context.packet_flow_key import get_packet_flow_key +from flow import Flow +from time_series.processor import Processor EXPIRED_UPDATE = 40 @@ -41,6 +44,16 @@ class FlowSession(DefaultSession): if "IP" not in packet: return + # if TLS not in packet: + # return + + # if TLSApplicationData not in packet: + # return + + # if len(packet[TLSApplicationData]) < 40: + # # PING frame (len = 34) or other useless frames + # return + self.packets_count += 1 # Creates a key variable to check @@ -106,17 +119,31 @@ class FlowSession(DefaultSession): for k in keys: flow = self.flows.get(k) - if ( - latest_time is None - or latest_time - flow.latest_timestamp > EXPIRED_UPDATE - or flow.duration > 90 - ): - data = flow.get_data() - if self.csv_line == 0: - self.csv_writer.writerow(data.keys()) - self.csv_writer.writerow(data.values()) - self.csv_line += 1 - del self.flows[k] + if self.output_mode == "flow": + if ( + latest_time is None + or latest_time - flow.latest_timestamp > EXPIRED_UPDATE + or flow.duration > 90 + ): + data = flow.get_data() + if self.csv_line == 0: + self.csv_writer.writerow(data.keys()) + self.csv_writer.writerow(data.values()) + self.csv_line += 1 + del self.flows[k] + else: + if ( + latest_time is None + or latest_time - flow.latest_timestamp > EXPIRED_UPDATE + ): + output_dir = os.path.join( + self.output_file, "doh" if flow.is_doh() else "ndoh" + ) + os.makedirs(output_dir, exist_ok=True) + proc = Processor(flow) + flow_clumps = proc.create_flow_clumps_container() + flow_clumps.to_json_file(output_dir) + del self.flows[k] print("Garbage Collection Finished. Flows = {}".format(len(self.flows))) diff --git a/src/cicflowmeter/sniffer.py b/src/cicflowmeter/sniffer.py index 64dfb19..c3302d6 100644 --- a/src/cicflowmeter/sniffer.py +++ b/src/cicflowmeter/sniffer.py @@ -1,8 +1,9 @@ import argparse +from scapy.all import load_layer from scapy.sendrecv import AsyncSniffer -from .flow_session import generate_session_class +from flow_session import generate_session_class def create_sniffer(input_file, input_interface, output_mode, output_file): @@ -33,7 +34,8 @@ def main(): input_group = parser.add_mutually_exclusive_group(required=True) input_group.add_argument( - "-i", + "-n", + "--online", "--interface", action="store", dest="input_interface", @@ -41,6 +43,7 @@ def main(): ) input_group.add_argument( "-f", + "--offline", "--file", action="store", dest="input_file", @@ -54,14 +57,21 @@ def main(): "--flow", action="store_const", const="flow", - default="flow", dest="output_mode", help="output flows as csv", ) + output_group.add_argument( + "-s", + "--json", + "--sequence", + action="store_const", + const="sequence", + dest="output_mode", + help="output flow segments as json", + ) parser.add_argument( - "output", - help="output file name in flow mode", + "output", help="output file name (in flow mode) or directory (in sequence mode)" ) args = parser.parse_args() diff --git a/tests/data/test.pcapfile b/tests/data/test.pcapfile deleted file mode 100644 index 5f18a9d..0000000 Binary files a/tests/data/test.pcapfile and /dev/null differ diff --git a/tests/test_features.py b/tests/test_features.py deleted file mode 100644 index a823135..0000000 --- a/tests/test_features.py +++ /dev/null @@ -1,28 +0,0 @@ -from scapy.all import IP, TCP, UDP, ICMP -from cicflowmeter.features.context.packet_flow_key import get_packet_flow_key -from cicflowmeter.features.context.packet_direction import PacketDirection -import pytest - - -def test_packet_flow_key(): - icmp_packet = IP(src="192.168.1.2", dst="192.168.1.1") / ICMP() - tcp_packet = IP(src="192.168.1.2", dst="192.168.1.1") / TCP() - udp_packet = IP(src="192.168.1.2", dst="192.168.1.1") / UDP() - - with pytest.raises(Exception): - get_packet_flow_key(icmp_packet, PacketDirection.FORWARD) - - """ - get_packet_flow_key return a tuple (dest_ip, src_ip, src_port, dest_port) - """ - tcp_forward = get_packet_flow_key(tcp_packet, PacketDirection.FORWARD) - tcp_backward = get_packet_flow_key(tcp_packet, PacketDirection.REVERSE) - udp_forward = get_packet_flow_key(udp_packet, PacketDirection.FORWARD) - udp_backward = get_packet_flow_key(udp_packet, PacketDirection.REVERSE) - - # Test IP match source and destination - assert tcp_forward[0] == udp_forward[0] - assert tcp_forward[0] == tcp_backward[1] - # Test Port match source and destination - assert udp_forward[2] == udp_backward[3] - assert tcp_forward[2] == tcp_backward[3] diff --git a/tox.ini b/tox.ini deleted file mode 100644 index a2d2b35..0000000 --- a/tox.ini +++ /dev/null @@ -1,16 +0,0 @@ -# tox (https://tox.readthedocs.io/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - -[tox] -envlist = py37 - -[testenv] -deps = - numpy - scipy - scapy - pytest -commands = - pytest