full_page_writes = off
default_statistics_target = 100
maintenance_work_mem = 1GB
checkpoint_completion_target = 0.9
effective_cache_size = 48GB
work_mem = 64MB
wal_buffers = -1
checkpoint_segments = 128
shared_buffers = 32GB
max_connections = 80
effective_io_concurrency = 3 # Down from 6
# - Background Writer -
#bgwriter_delay = 200ms # 10-10000ms between rounds
#bgwriter_lru_maxpages = 100 # 0-1000 max buffers written/round
#bgwriter_lru_multiplier = 2.0 # 0-10.0 multipler on buffers scanned/round
$ free
total used free shared buffers cached
Mem: 61368192 60988180 380012 0 784 44167172
-/+ buffers/cache: 16820224 44547968
Swap: 0 0 0
$ top -c
top - 21:55:51 up 12 days, 12:41, 4 users, load average: 6.03, 16.10, 24.15
top - 21:55:54 up 12 days, 12:41, 4 users, load average: 6.03, 15.94, 24.06
Tasks: 837 total, 6 running, 831 sleeping, 0 stopped, 0 zombie
Cpu(s): 15.7%us, 1.7%sy, 0.0%ni, 81.6%id, 0.3%wa, 0.0%hi, 0.6%si, 0.0%st
Mem: 61368192k total, 54820988k used, 6547204k free, 9032k buffer
[ec2-user@ip-10-155-231-112 ~]$ sudo iostat
Linux 3.2.39-6.88.amzn1.x86_64 () 04/01/2013 _x86_64_ (32 CPU)
avg-cpu: %user %nice %system %iowait %steal %idle
21.00 0.00 1.10 0.26 0.00 77.63
Device: tps Blk_read/s Blk_wrtn/s Blk_read Blk_wrtn
xvda 0.21 5.00 2.22 5411830 2401368
xvdk 98.32 1774.67 969.86 1919359965 1048932113
xvdj 98.28 1773.68 969.14 1918288697 1048156776
xvdi 98.29 1773.69 969.61 1918300250 1048662470
xvdh 98.24 1773.92 967.54 1918544618 1046419936
xvdg 98.27 1774.15 968.85 1918790636 1047842846
xvdf 98.32 1775.56 968.69 1920316435 1047668172
md127 733.85 10645.68 5813.70 11513598393 6287682313
What bugs me on this is the throughput percentage on the volumes in Cloudwatch is 100% on all volumes.
The problems seem to overlap with checkpoints.
2013-04-01 21:31:35.592 UTC,,,26877,,5159fa5f.68fd,1,,2013-04-01 21:21:35 UTC,,0,LOG,00000,"checkpoint starting: time",,,,,,,,,""
2013-04-01 21:40:35.033 UTC,,,26877,,5159fa5f.68fd,2,,2013-04-01 21:21:35 UTC,,0,LOG,00000,"checkpoint complete: wrote 100635 buffers (2.4%); 0 transaction log file(s) added, 0 removed, 1 recycled; write=539.439 s, sync=0.000 s, total=539.441 s; sync files=0, longest=0.000 s, average=0.000 s",,,,,,,,,""
2013-04-01 21:41:35.093 UTC,,,26877,,5159fa5f.68fd,3,,2013-04-01 21:21:35 UTC,,0,LOG,00000,"checkpoint starting: time",,,,,,,,,""