Qemu packer ssh failed connection reset by peer

I am trying to get packer to create a qemu/kvm image. I feel like it should be fairly straightforward but my build keeps failing on ssh. Here are the relevant files

template.json:

{
"builders": [
{
  "type": "qemu",
  "iso_url": "/home/username/ubuntu-14.04.4-desktop-amd64.iso",
  "iso_checksum": "807fa1f246b719d28d0b362fd2f31855",
  "iso_checksum_type": "md5",
  "shutdown_command": "/sbin/shutdown -hP now",
  "ssh_username": "ubuntu",
  "ssh_password": "test",
  "ssh_port": 22,
  "ssh_wait_timeout": "5m",
  "ssh_host_port_min": 2222,
  "ssh_host_port_max": 2229,
  "boot_wait": "30s",
  "disk_size": 20480,
  "format": "raw",
  "headless": true,
  "accelerator": "kvm",
  "http_directory": "http",
  "http_port_min": 10082,
  "http_port_max": 10089,
  "vm_name": "ubuntu-test",
  "net_device": "virtio-net",
  "disk_interface": "virtio",
  "boot_command": [
    "<esc><wait>",
    "install ",
    "preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg ",
    "debian-installer=en_US ",
    "auto ",
    "locale=en_US ",
    "kbd-chooser/method=us ",
    "keyboard-configuration/xkb-keymap=us ",
    "fb=false ",
    "debconf/frontend=noninteractive ",
    "console-setup/ask_detect=false ",
    "console-keymaps-at/keymap=us ",
    "<enter><wait>"
  ]
}
]
}

preseed.cfg:

choose-mirror-bin mirror/http/proxy string
d-i debian-installer/framebuffer boolean false
d-i debconf/frontend select noninteractive
d-i base-installer/kernel/override-image string linux-server
d-i clock-setup/utc boolean true
d-i clock-setup/utc-auto boolean true
d-i finish-install/reboot_in_progress note
d-i grub-installer/only_debian boolean true
d-i grub-installer/with_other_os boolean true
d-i netcfg/get_domain string test-domain
d-i netcfg/get_hostname string test-hostname
d-i partman-auto/method string regular
d-i partman/choose_partition select finish
d-i partman/confirm boolean true
d-i partman/confirm_nooverwrite boolean true
d-i partman/confirm_write_new_label boolean true
d-i pkgsel/include string openssh-server
d-i pkgsel/install-language-support boolean false
d-i pkgsel/update-policy select unattended-upgrades
d-i pkgsel/upgrade select none
d-i time/zone string CET
d-i user-setup/allow-password-weak boolean true
d-i user-setup/encrypt-home boolean false
tasksel tasksel/first multiselect none
d-i passwd/user-fullname string default
d-i passwd/username string ubuntu
d-i passwd/user-password password test
d-i passwd/user-password-again password test

Here is the error I get in the packer log:

2016/08/16 11:24:05 packer: 2016/08/16 11:24:05 handshaking with SSH
2016/08/16 11:24:05 packer: 2016/08/16 11:24:05 handshake error: ssh: handshake failed: read tcp 127.0.0.1:40788->127.0.0.1:2227: read: connection reset by peer
2016/08/16 11:24:05 packer: 2016/08/16 11:24:05 [DEBUG] SSH handshake err: ssh: handshake failed: read tcp 127.0.0.1:40788->127.0.0.1:2227: read: connection reset by peer

Any tips or ideas would be welcomed, thanks