Upgrade from MySQL 4 to Mysql 5 - same my.cnf - lots of random disconnects

I've recently moved from MySQL4 to MySQL5 (we had apps that required MySQL4 which we'd finally replaced)

Since then, I've been having LOTS of issues similar to:-

connect() failed: Lost connection to MySQL server at 'reading initial communication packet', system error: 0 

Yes, that is System Error 0 - "Success".

I've no idea why this is happening - but want to find out why

Here's my (sanitised) my.cnf

[client]
port        = 3306
socket      = /var/run/mysqld/mysqld.sock

[mysqld_safe]
socket      = /var/run/mysqld/mysqld.sock
nice        = 0

[mysqld]
user        = mysql
pid-file    = /var/run/mysqld/mysqld.pid
socket      = /var/run/mysqld/mysqld.sock
port        = 3306
basedir     = /usr
datadir     = /var/lib/mysql
tmpdir      = /tmp
language    = /usr/share/mysql/english
skip-external-locking

key_buffer      = 1048M
max_allowed_packet  = 16M
thread_stack = 2M
thread_cache_size   = 16384
sort_buffer_size    = 256K
read_buffer_size    = 2M
read_rnd_buffer_size    = 256K
tmp_table_size      = 512M
max_heap_table_size = 512M
query_cache_size    = 512M
thread_concurrency  = 16
set-variable = table_cache=16384
max_connections     = 500

wait_timeout = 3600

innodb_buffer_pool_size = 1024M
innodb_additional_mem_pool_size = 8M
innodb_lock_wait_timeout = 50
innodb_flush_method=O_DIRECT
innodb_log_buffer_size = 8M
innodb_thread_concurrency = 8
innodb_file_per_table=1

query_cache_limit       = 1024M
query_cache_size        = 512M
query_cache_type = 1
server-id       = 9
log_bin         = /var/log/mysql/mysql-bin.log
expire_logs_days    = 10
max_binlog_size         = 100M

expire_logs_days    = 3

skip-bdb

[mysqldump]
quick
quote-names
max_allowed_packet  = 16M

[mysql]
prompt                  = (\u@atlas) [\d]>\\_

[isamchk]
key_buffer      = 16M

!includedir /etc/mysql/conf.d/

EDIT I changed the my.cnf above - I've just realised it was the one from the old server! Also - this is 5.0, not 5.1