data.aws.timestamp in Wazuh

[WENWEN H]

Hello Team,

I hope you are doing well.

I would like to know how to convert the format of the "data.aws.timestamp" field in the WAF logs retrieved from AWS S3 to the "data" type. For example, in log.png, the value of "data.aws.timestamp" is 1777391247050.000000. I would like to be able to visually see that its value is 2026-04-28 23:47:27. You may have noticed that there is a field named "timestamp" at the bottom. However, the time in this field differs from the actual log occurrence time by several minutes. Therefore, it is still recommended to convert the field "data.aws.timestamp", as this is the correct occurrence time of the log event.

We sincerely hope to receive your support.

Regards 

WENWEN

[Md. Nazmur Sakib]

Hi WENWEN,

You can achieve this by changing the filebeat pipeline.

Make a backup of the existing pipeline file.

cp /usr/share/filebeat/module/wazuh/alerts/ingest/pipeline.json \

   /usr/share/filebeat/module/wazuh/alerts/ingest/pipeline.json.bak

Edit the
Pipeline file.
/usr/share/filebeat/module/wazuh/alerts/ingest/pipeline.json

And add this

{

  "script": {

    "description": "Convert data.aws.timestamp from milliseconds to seconds",

    "lang": "painless",

    "source": "if (ctx?.data?.aws?.timestamp != null) { ctx.data.aws.timestamp_epoch_sec = (long)(Double.parseDouble(ctx.data.aws.timestamp.toString()) / 1000); }"

  }

},

{

  "date": {

    "description": "Convert data.aws.timestamp epoch seconds to formatted date",

    "field": "data.aws.timestamp_epoch_sec",

    "target_field": "data.aws.timestamp",

    "formats": ["UNIX"],

    "output_format": "yyyy-MM-dd HH:mm:ss",

    "timezone": "UTC"

  }

},

{

  "remove": {

    "field": "data.aws.timestamp_epoch_sec",

    "ignore_missing": true

  }

},

before:

    {

      "date_index_name": {

        "field": "timestamp",

        "date_rounding": "d",

        "index_name_prefix": "{{fields.index_prefix}}",

        "index_name_format": "yyyy.MM.dd",

        "ignore_failure": false

      }

    },

    { "remove": { "field": "message", "ignore_missing": true, "ignore_failure": true } },

Save the configuration and apply the pipeline:

filebeat setup --pipelines

systemctl restart filebeat

Check this to learn more about the filebeat pipeline:
https://www.elastic.co/docs/reference/beats/filebeat/filtering-enhancing-data


Here is a screenshot of the test result.



Let me know if this works for you.

[WENWEN H]

Hello, based on your instructions, I attempted to modify the file "pipeline.json". However, there was a problem with the interruption of the logs. No logs were generated subsequently, and I'm not sure if I made the modifications as per your intention. Below is the complete content of the modified "pipeline.json".
pipeline.json：
{
  "description": "Wazuh alerts pipeline",
  "processors": [
    { "json" : { "field" : "message", "add_to_root": true } },
    {
      "set": {
        "field": "data.aws.region",
        "value": "{{data.aws.awsRegion}}",
        "override": false,
        "ignore_failure": true,
        "ignore_empty_value" : true
      }
    },
    {
      "set": {
        "field": "data.aws.accountId",
        "value": "{{data.aws.aws_account_id}}",
        "override": false,
        "ignore_failure": true,
        "ignore_empty_value" : true
      }
    },
    {
      "geoip": {
        "field": "data.srcip",
        "target_field": "GeoLocation",
        "properties": ["city_name", "country_name", "region_name", "location"],

        "ignore_missing": true,
        "ignore_failure": true
      }
    },

    {
      "geoip": {
        "field": "data.win.eventdata.ipAddress",
        "target_field": "GeoLocation",
        "properties": ["city_name", "country_name", "region_name", "location"],

        "ignore_missing": true,
        "ignore_failure": true
      }
    },

    {
      "geoip": {
        "field": "data.aws.sourceIPAddress",
        "target_field": "GeoLocation",
        "properties": ["city_name", "country_name", "region_name", "location"],

        "ignore_missing": true,
        "ignore_failure": true
      }
    },

    {
      "geoip": {
        "field": "data.aws.client_ip",
        "target_field": "GeoLocation",
        "properties": ["city_name", "country_name", "region_name", "location"],

        "ignore_missing": true,
        "ignore_failure": true
      }
    },

    {
      "geoip": {
        "field": "data.aws.service.action.networkConnectionAction.remoteIpDetails.ipAddressV4",
        "target_field": "GeoLocation",
        "properties": ["city_name", "country_name", "region_name", "location"],

        "ignore_missing": true,
        "ignore_failure": true
      }
    },

    {
      "geoip": {
        "field": "data.aws.httpRequest.clientIp",
        "target_field": "GeoLocation",
        "properties": ["city_name", "country_name", "region_name", "location"],

        "ignore_missing": true,
        "ignore_failure": true
      }
    },

    {
      "geoip": {
        "field": "data.gcp.jsonPayload.sourceIP",
        "target_field": "GeoLocation",
        "properties": ["city_name", "country_name", "region_name", "location"],

        "ignore_missing": true,
        "ignore_failure": true
      }
    },

    {
      "geoip": {
        "field": "data.office365.ClientIP",
        "target_field": "GeoLocation",
        "properties": ["city_name", "country_name", "region_name", "location"],

        "ignore_missing": true,
        "ignore_failure": true
      }
    },

    {
      "date": {
        "field": "timestamp",
        "target_field": "@timestamp",
        "formats": ["ISO8601"],
        "ignore_failure": false
      }
    },

    {
      "script": {
        "description": "Convert data.aws.timestamp from milliseconds to seconds",
        "lang": "painless",
        "source": "if (ctx?.data?.aws?.timestamp != null) { ctx.data.aws.timestamp_epoch_sec = (long)(Double.parseDouble(ctx.data.aws.timestamp.toString()) / 1000); }"
      }
    },
    {
      "date": {
        "description": "Convert data.aws.timestamp epoch seconds to formatted date",
        "field": "data.aws.timestamp_epoch_sec",
        "target_field": "data.aws.timestamp",
        "formats": ["UNIX"],
        "output_format": "yyyy-MM-dd HH:mm:ss",

        "timezone": "Asia/Shanghai"

      }
    },
    {
      "remove": {
        "field": "data.aws.timestamp_epoch_sec",
        "ignore_missing": true
      }
    },

    {
      "date_index_name": {
        "field": "timestamp",
        "date_rounding": "d",
        "index_name_prefix": "{{fields.index_prefix}}",
        "index_name_format": "yyyy.MM.dd",
        "ignore_failure": false
      }
    },
    { "remove": { "field": "message", "ignore_missing": true, "ignore_failure": true } },

    { "remove": { "field": "ecs", "ignore_missing": true, "ignore_failure": true } },
    { "remove": { "field": "beat", "ignore_missing": true, "ignore_failure": true } },
    { "remove": { "field": "input_type", "ignore_missing": true, "ignore_failure": true } },
    { "remove": { "field": "tags", "ignore_missing": true, "ignore_failure": true } },
    { "remove": { "field": "count", "ignore_missing": true, "ignore_failure": true } },
    { "remove": { "field": "@version", "ignore_missing": true, "ignore_failure": true } },
    { "remove": { "field": "log", "ignore_missing": true, "ignore_failure": true } },
    { "remove": { "field": "offset", "ignore_missing": true, "ignore_failure": true } },
    { "remove": { "field": "type", "ignore_missing": true, "ignore_failure": true } },
    { "remove": { "field": "host", "ignore_missing": true, "ignore_failure": true } },
    { "remove": { "field": "fields", "ignore_missing": true, "ignore_failure": true } },
    { "remove": { "field": "event", "ignore_missing": true, "ignore_failure": true } },
    { "remove": { "field": "fileset", "ignore_missing": true, "ignore_failure": true } },
    { "remove": { "field": "service", "ignore_missing": true, "ignore_failure": true } }
  ],
  "on_failure" : [{
    "drop" : { }
  }]