FieldManager

This presentations goal it to introduce the features of the FieldManager and how to configure it.

The challenges

• I want to move or rename a field.

• I want to copy a field.

• I want to merge field values to a list.

• I want to merge lists from different fields to one list in a new or existing field

given preprocessed log entry:

document = {
    "client": {"ip": ["127.0.0.1", "fe89::", "192.168.5.1"], "nat": {"ip": "223.2.3.2"}},
    "destination": {"ip": "8.8.8.8"},
    "host": {"_hostname": "customer2", "ip": ["192.168.5.1", "180.22.66.3"]},
    "observer": {"ip": "10.10.2.33"},
    "server": {"ip": "10.10.2.33", "nat": {"ip": "180.22.66.1"}},
    "source": {"ip": "10.10.2.33"},
    "preexisting": "I exists already"
}

Create rules and processor

create the rules:

import sys
sys.path.append("../../../../../")

from logprep.processor.field_manager.rule import FieldManagerRule
rules_definitions = [
    {
        "filter": "host._hostname",
        "field_manager": {
            "source_fields": ["client.nat.ip", "source.ip"],
            "target_field": "related.ip",
            "overwrite_target": True,
            "delete_source_fields": True,
            "extend_target_list": True
        },
    }
]
rules = [FieldManagerRule.create_from_dict(rule_dict) for rule_dict in rules_definitions]
rules

[filter="host._hostname", FieldManagerRule.Config(description='', regex_fields=[], tests=[], tag_on_failure=['_field_manager_failure'], source_fields=['client.nat.ip', 'source.ip'], target_field='related.ip', delete_source_fields=True, overwrite_target=True, extend_target_list=True)]

create the processor config:

processor_config = {
    "the_field_manager": {
        "type": "field_manager",
        "rules": ["/dev"],
    }
}

create the processor with the factory:

from logging import getLogger
from logprep.factory import Factory

logger = getLogger()

processor = Factory.create(processor_config)
processor

field_manager

load rules to processor

for rule in rules:
    processor._rule_tree.add_rule(rule)

processor._rules

[filter="host._hostname", FieldManagerRule.Config(description='', regex_fields=[], tests=[], tag_on_failure=['_field_manager_failure'], source_fields=['client.nat.ip', 'source.ip'], target_field='related.ip', delete_source_fields=True, overwrite_target=True, extend_target_list=True)]

Process event

from copy import deepcopy

mydocument = deepcopy(document)
processor.process(mydocument)

Check Results

document

{'client': {'ip': ['127.0.0.1', 'fe89::', '192.168.5.1'],
  'nat': {'ip': '223.2.3.2'}},
 'destination': {'ip': '8.8.8.8'},
 'host': {'_hostname': 'customer2', 'ip': ['192.168.5.1', '180.22.66.3']},
 'observer': {'ip': '10.10.2.33'},
 'server': {'ip': '10.10.2.33', 'nat': {'ip': '180.22.66.1'}},
 'source': {'ip': '10.10.2.33'},
 'preexisting': 'I exists already'}

mydocument

{'client': {'ip': ['127.0.0.1', 'fe89::', '192.168.5.1']},
 'destination': {'ip': '8.8.8.8'},
 'host': {'_hostname': 'customer2', 'ip': ['192.168.5.1', '180.22.66.3']},
 'observer': {'ip': '10.10.2.33'},
 'server': {'ip': '10.10.2.33', 'nat': {'ip': '180.22.66.1'}},
 'preexisting': 'I exists already',
 'related': {'ip': ['10.10.2.33', '223.2.3.2']}}