{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# FieldManager\n",
        "\n",
        "This presentations goal it to introduce the features of the `FieldManager` and how to configure it."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### The challenges\n",
        "\n",
        "- I want to move or rename a field.\n",
        "- I want to copy a field.\n",
        "- I want to merge field values to a list.\n",
        "- I want to merge lists from different fields to one list in a new or existing field\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "given preprocessed log entry:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {},
      "outputs": [],
      "source": [
        "document = {\n",
        "    \"client\": {\"ip\": [\"127.0.0.1\", \"fe89::\", \"192.168.5.1\"], \"nat\": {\"ip\": \"223.2.3.2\"}},\n",
        "    \"destination\": {\"ip\": \"8.8.8.8\"},\n",
        "    \"host\": {\"_hostname\": \"customer2\", \"ip\": [\"192.168.5.1\", \"180.22.66.3\"]},\n",
        "    \"observer\": {\"ip\": \"10.10.2.33\"},\n",
        "    \"server\": {\"ip\": \"10.10.2.33\", \"nat\": {\"ip\": \"180.22.66.1\"}},\n",
        "    \"source\": {\"ip\": \"10.10.2.33\"},\n",
        "    \"preexisting\": \"I exists already\"\n",
        "}\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Create rules and processor"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "create the rules:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[filter=\"host._hostname\", FieldManagerRule.Config(description='', regex_fields=[], tests=[], tag_on_failure=['_field_manager_failure'], source_fields=['client.nat.ip', 'source.ip'], target_field='related.ip', delete_source_fields=True, overwrite_target=True, extend_target_list=True)]"
            ]
          },
          "execution_count": 2,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import sys\n",
        "sys.path.append(\"../../../../../\")\n",
        "\n",
        "from logprep.processor.field_manager.rule import FieldManagerRule\n",
        "rules_definitions = [\n",
        "    {\n",
        "        \"filter\": \"host._hostname\",\n",
        "        \"field_manager\": {\n",
        "            \"source_fields\": [\"client.nat.ip\", \"source.ip\"],\n",
        "            \"target_field\": \"related.ip\",\n",
        "            \"overwrite_target\": True,\n",
        "            \"delete_source_fields\": True,\n",
        "            \"extend_target_list\": True\n",
        "        },\n",
        "    }\n",
        "]\n",
        "rules = [FieldManagerRule.create_from_dict(rule_dict) for rule_dict in rules_definitions]\n",
        "rules"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "create the processor config:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [],
      "source": [
        "processor_config = {\n",
        "    \"the_field_manager\": {\n",
        "        \"type\": \"field_manager\",\n",
        "        \"rules\": [\"/dev\"],\n",
        "    }\n",
        "}\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "create the processor with the factory:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "field_manager"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from logging import getLogger\n",
        "from logprep.factory import Factory\n",
        "\n",
        "logger = getLogger()\n",
        "\n",
        "processor = Factory.create(processor_config)\n",
        "processor\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "load rules to processor"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[filter=\"host._hostname\", FieldManagerRule.Config(description='', regex_fields=[], tests=[], tag_on_failure=['_field_manager_failure'], source_fields=['client.nat.ip', 'source.ip'], target_field='related.ip', delete_source_fields=True, overwrite_target=True, extend_target_list=True)]"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "for rule in rules:\n",
        "    processor._rule_tree.add_rule(rule)\n",
        "    \n",
        "processor._rules"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Process event"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [],
      "source": [
        "from copy import deepcopy\n",
        "\n",
        "mydocument = deepcopy(document)\n",
        "processor.process(mydocument)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Check Results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'client': {'ip': ['127.0.0.1', 'fe89::', '192.168.5.1'],\n",
              "  'nat': {'ip': '223.2.3.2'}},\n",
              " 'destination': {'ip': '8.8.8.8'},\n",
              " 'host': {'_hostname': 'customer2', 'ip': ['192.168.5.1', '180.22.66.3']},\n",
              " 'observer': {'ip': '10.10.2.33'},\n",
              " 'server': {'ip': '10.10.2.33', 'nat': {'ip': '180.22.66.1'}},\n",
              " 'source': {'ip': '10.10.2.33'},\n",
              " 'preexisting': 'I exists already'}"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "document"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'client': {'ip': ['127.0.0.1', 'fe89::', '192.168.5.1']},\n",
              " 'destination': {'ip': '8.8.8.8'},\n",
              " 'host': {'_hostname': 'customer2', 'ip': ['192.168.5.1', '180.22.66.3']},\n",
              " 'observer': {'ip': '10.10.2.33'},\n",
              " 'server': {'ip': '10.10.2.33', 'nat': {'ip': '180.22.66.1'}},\n",
              " 'preexisting': 'I exists already',\n",
              " 'related': {'ip': ['10.10.2.33', '223.2.3.2']}}"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "mydocument"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3.11.0 ('.venv': venv)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.0"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "586280540a85d3e21edc698fe7b86af2848b9b02644e6c22463da25c40a3f1be"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}