Logstash config with redis

[Gaurav Agarwal]

While using redis as the in-memory database the logstash config file is not working fine whereas when in place redis if I use the file input plugin it does not fail at all. I am not able to understand what is wrong with my config file where I use redis as the input plugin.

Redis forwader
input {
file {
path => "/logs/app_/.log"
exclude => [
"access*.log*",
"fe*.log*",
"stat*.log*",
"gc*.log*",
"dump.log*"
]
sincedb_path => "/dev/null"
type => "app"
codec => multiline {
pattern => "^%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}"
negate => true
what => "previous"
}
}
}

filter {

grok {
	match => {
		"path" => "/logs/(?<app>[^/]+)/[^/]+/logs/(?<instance>[^/]+)/[^/]+"
	}
	overwrite => [ "host" ]
}

}

output {

stdout { codec => rubydebug }

redis {
	host => "gauapt13"
	data_type => "list"
	key => "logstash-app"
}

}

redis indexer
input {
redis {
host => "gauapt13"
data_type => "list"
key => "logstash-app"
codec => json
threads => 2

codec => multiline {
pattern => "^%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}"
negate => true
what => "previous"
}

}
}
filter {

if [type] == "app" {
	grok {
		match => {
			# "message" => "%{APT_TIMESTAMP:timestamp} \| %{HOSTNAME:hostname} \| %{DATA:application} \| %{APT_LOGLEVEL:loglevel} *\| 1-%{DATA:thread:int} \| %{DATA:class} *\| %{DATA:correlationId} \| %{GREEDYDATA:msg}"
			"message" => "%{APT_TIMESTAMP:timestamp} \| \S+ \| \S+ \| %{APT_LOGLEVEL:loglevel} *\| 1-%{DATA:thread:int} \| %{DATA:class} *\| %{DATA:correlationId} \| %{GREEDYDATA:msg}"
		}
		patterns_dir => "/logs/pattern"
		remove_field => [ "message" ]
	}
}

if "_grokparsefailure" not in [tags] {
    date {
        match => [ "timestamp", "YYYY/MM/dd HH:mm:ss.SSS", "YYYY/MM/dd HH:mm:ss,SSS", "dd/MMM/YYYY:HH:mm:ss +0000", "EEE MMM dd HH:mm:ss YYYY" ]
        timezone => "UTC"
	}
	if [type] == "web_access" or [type] == "web_error" {
        mutate {
	        gsub => [
                "referrer","\"","",
                "agent","\"","",
                "JSESSIONID","\"","",
                "APT_SESSIONID","\"","",
                "correlationId","\"","",
                "transactionOriginator","\"","",
                "customerId","\"",""
            ]
	        remove_field => [ "logline", "timestamp", "BASE10NUM", "INT", "HOSTNAME", "IPV4", "day", "month", "monthday", "time", "year"]
		}
	}

}

}

output {
stdout{ codec => rubydebug}
if [app] == "aaa" or [app] == "amga" or [type] == "fem" or [type] == "sepal" or [type] == "web_access" {
elasticsearch {
hosts => "gauapt13"
index => "apt-%{+YYYY.MM.dd}"

	}
}
else {
	elasticsearch {
		hosts => "gauapt13"
		index => "apt-%{+YYYY.MM.dd}"
        
}

	
        


}

}