Enormous S3 invoice for loki

[Vyacheslav Voronenko]

Good day.

We are trying to introduce +- long term log storage setup. Via config below, inspired mostly by official docs, we are getting quite enormous S3 services invoice (mostly list bucket/ list objects operations) which potentially would cover few large instances with a big gp3 disk.   Is anyone using loki with S3 backend and what is your cost of the ownership.  Any hints for optimization are highly appreciated.

auth_enabled: false

server:

  http_listen_port: 3100

  grpc_listen_port: 9095

  grpc_server_max_recv_msg_size: 15728640

  grpc_server_max_send_msg_size: 15728640

ingester:

  lifecycler:

    address: 127.0.0.1

    ring:

      kvstore:

        store: inmemory

      replication_factor: 1

  chunk_idle_period: 5m

  chunk_retain_period: 30s

  chunk_target_size: 1536000

  chunk_block_size: 1048576

  max_transfer_retries: 0

  max_chunk_age: 5m

compactor:

  working_directory: /tmp/loki/retention

  shared_store: s3

  compaction_interval: 30m

  retention_enabled: true

  retention_delete_delay: 2h

  retention_delete_worker_count: 150

schema_config:

  configs:

    - from: 2018-04-15

      store: boltdb

      object_store: filesystem

      schema: v11

      index:

        prefix: index_

        period: 168h

    - from: 2021-09-27

      store: boltdb-shipper

      object_store: s3

      schema: v11

      index:

        prefix: loki_index_

        period: 24h

storage_config:

  boltdb:

    directory: /tmp/loki/index

  filesystem:

    directory: /tmp/loki/chunks

  boltdb_shipper:

    active_index_directory: /tmp/loki/index

    shared_store: s3

    cache_location: /tmp/loki/index_cache

    resync_interval: 30m

  aws:

    #bucketnames: CENSORED_BUCKET_NAME

    #region: eu-central-1

    s3: s3://eu-central-1/CENSORED_BUCKET_NAME

    endpoint: s3.eu-central-1.amazonaws.com

    s3forcepathstyle: true

    insecure: false

    signature_version: v4

limits_config:

  enforce_metric_name: false

  reject_old_samples: true

  reject_old_samples_max_age: 168h

chunk_store_config:

  # keep in sync with table_manager->retention_period

  # Loki requires that retention (2520h below) is divisible by the duration of each index table (168h by default)

  max_look_back_period: 2520h

table_manager:

  # keep in sync with chunk_store_config->max_query_lookback

  retention_period: 2520h

  retention_deletes_enabled: true

  poll_interval: 24

Kind Regards,

  Vyacheslav