I have ES 1.5.2 cluster with Kibana 4.0.1. This consists of 4 data/master nodes along with 4 client nodes. When I'm trying to load kibana sometimes it takes longer time to load or sometimes I need to restart the ES client node which kibana is pointed to. When the issue is there,
I get below console error in Kibana page
Failed to load resource: net::ERR_EMPTY_RESPONSE - http://xx.xx.xx.xx /elasticsearch/_nodes
and below error in ES log
[2016-01-15 07:01:52,629][DEBUG][http.netty] [xxxxxx] Caught exception while handling client http traffic, closing connection [id: 0x8301fd2b, /xx.xx.xx.xx:54237 :> /xx.xx.xx.xx:9200]
All the masters has below config
"index" : {
"number_of_shards" : "3"
},
"bootstrap" : {
"mlockall" : "true"
},
"threadpool" : {
"search" : {
"queue_size" : "2000",
"size" : "20"
}
"indices" : {
"fielddata" : {
"cache" : {
"size" : "50%"
}
}
},
"discovery" : {
"zen" : {
"master_election" : {
"filter_client" : "true"
},
"ping" : {
"multicast" : {
"enabled" : "false"
},
"timeout" : "6s",
"unicast" : {
"hosts" : [ "xxxx", "xxx", "xxxx" ]
}
}
}
},
"name" : "xxxxx",
"http" : {
"enabled" : "true"
},
"action" : {
"disable_delete_all_indices" : "true"
},
"client" : {
"type" : "node"
},
"config" : "/xxxxx/elasticsearch-1.5.2/config/elasticsearch.yml"
},
"os" : {
"refresh_interval_in_millis" : 1000,
"available_processors" : 4,
"cpu" : {
"vendor" : "Intel",
"model" : "Xeon",
"mhz" : 2400,
"total_cores" : 4,
"total_sockets" : 2,
"cores_per_socket" : 2,
"cache_size_in_bytes" : 30720
},
"mem" : {
"total_in_bytes" : 16820760576
},
"swap" : {
"total_in_bytes" : 2145382400
}
},
"process" : {
"refresh_interval_in_millis" : 1000,
"id" : 48090,
"max_file_descriptors" : 65535,
"mlockall" : true
},
"jvm" : {
"pid" : 48090,
"version" : "1.8.0_45",
"vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
"vm_version" : "25.45-b02",
"vm_vendor" : "Oracle Corporation",
"start_time_in_millis" : 1452791955862,
"mem" : {
"heap_init_in_bytes" : 8589934592,
"heap_max_in_bytes" : 8555069440,
"non_heap_init_in_bytes" : 2555904,
"non_heap_max_in_bytes" : 0,
"direct_max_in_bytes" : 8555069440
},
"gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
"memory_pools" : [ "Code Cache", "Metaspace", "Compressed Class Space", "Par Eden Space", "Par Survivor Space", "CMS Old Gen" ]
},
"thread_pool" : {
"percolate" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : "1k"
},
"listener" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : -1
},
"index" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : "200"
},
"refresh" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
},
"suggest" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : "1k"
},
"generic" : {
"type" : "cached",
"keep_alive" : "30s",
"queue_size" : -1
},
"warmer" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
},
"search" : {
"type" : "fixed",
"min" : 20,
"max" : 20,
"queue_size" : "2k"
},
"flush" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
},
"optimize" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : -1
},
"management" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m",
"queue_size" : -1
},
"get" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : "1k"
},
"merge" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
},
"bulk" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : "50"
},
"snapshot" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
}
},
"network" : {
"refresh_interval_in_millis" : 5000,
"primary_interface" : {
"address" : "xxxxxx",
"name" : "eth0",
"mac_address" : "xxxxxxx"
}
},
"transport" : {
"bound_address" : "inet[/xxxxx:9300]",
"publish_address" : "inet[/1xxxx:9300]",
"profiles" : { }
},
"http" : {
"bound_address" : "inet[/1xxxxx:9200]",
"publish_address" : "inet[/xxxxx:9200]",
"max_content_length_in_bytes" : 104857600
},
All the clients has below config
"index" : {
"number_of_shards" : "3"
},
"bootstrap" : {
"mlockall" : "true"
},
"threadpool" : {
"search" : {
"queue_size" : "2000",
"size" : "20"
}
"indices" : {
"fielddata" : {
"cache" : {
"size" : "50%"
}
}
},
"discovery" : {
"zen" : {
"master_election" : {
"filter_client" : "true"
},
"ping" : {
"multicast" : {
"enabled" : "false"
},
"timeout" : "6s",
"unicast" : {
"hosts" : [ "xx.xx.xx.xx", "xx.xx.xx.xx", "xx.xx.xx.xx", "xx.xx.xx.xx" ]
}
}
}
},
"name" : "xxxxxx",
"action" : {
"disable_delete_all_indices" : "true"
},
"client" : {
"type" : "node"
},
"config" : "/xxxx/elasticsearch-1.5.2/config/elasticsearch.yml"
},
"os" : {
"refresh_interval_in_millis" : 1000,
"available_processors" : 2,
"cpu" : {
"vendor" : "Intel",
"model" : "Xeon",
"mhz" : 2400,
"total_cores" : 2,
"total_sockets" : 1,
"cores_per_socket" : 2,
"cache_size_in_bytes" : 30720
},
"mem" : {
"total_in_bytes" : 8365080576
},
"swap" : {
"total_in_bytes" : 2145382400
}
},
"process" : {
"refresh_interval_in_millis" : 1000,
"id" : 34778,
"max_file_descriptors" : 65535,
"mlockall" : true
},
"jvm" : {
"pid" : 34778,
"version" : "1.8.0_45",
"vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
"vm_version" : "25.45-b02",
"vm_vendor" : "Oracle Corporation",
"start_time_in_millis" : 1452792092500,
"mem" : {
"heap_init_in_bytes" : 4294967296,
"heap_max_in_bytes" : 4277534720,
"non_heap_init_in_bytes" : 2555904,
"non_heap_max_in_bytes" : 0,
"direct_max_in_bytes" : 4277534720
},
"gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
"memory_pools" : [ "Code Cache", "Metaspace", "Compressed Class Space", "Par Eden Space", "Par Survivor Space", "CMS Old Gen" ]
},
"thread_pool" : {
"percolate" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : "1k"
},
"listener" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : -1
},
"index" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : "200"
},
"refresh" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"suggest" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : "1k"
},
"generic" : {
"type" : "cached",
"keep_alive" : "30s",
"queue_size" : -1
},
"warmer" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"search" : {
"type" : "fixed",
"min" : 20,
"max" : 20,
"queue_size" : "2k"
},
"flush" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"optimize" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : -1
},
"management" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m",
"queue_size" : -1
},
"get" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : "1k"
},
"merge" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"bulk" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : "50"
},
"snapshot" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
}
},
"network" : {
"refresh_interval_in_millis" : 5000,
"primary_interface" : {
"address" : "xx.xx.xx.xx",
"name" : "eth0",
"mac_address" : "xx.xx.xx.xx"
}
},
"transport" : {
"bound_address" : "inet[/xx.xx.xx.xx:9300]",
"publish_address" : "inet[/xx.xx.xx.xx:9300]",
"profiles" : { }
},
"http" : {
"bound_address" : "inet[/xx.xx.xx.xx:9200]",
"publish_address" : "inet[/xx.xx.xx.xx:9200]",
"max_content_length_in_bytes" : 104857600
},
"plugins" : [ ]
I don't know if you made a typo or what but in the log I see
There's a space generally where a port should be unless you map 80->9200
or if you map 80->92000
Finally found the issue. I had two client nodes in different network. Adjusting tcp keepalive parameters in ubuntu servers fixed the issue.