diff --git a/Dockerfile.dind b/Dockerfile.dind
index d768cb4..2b7378f 100644
--- a/Dockerfile.dind
+++ b/Dockerfile.dind
@@ -41,7 +41,11 @@ ENV DOCKER_STORAGE_DRIVER=$docker_storage_driver
 # Move to our home
 WORKDIR /root
 
-RUN mkdir -p /var/run/pwd/certs && mkdir -p /var/run/pwd/uploads
+# Setup certs and ssh keys
+RUN mkdir -p /var/run/pwd/certs && mkdir -p /var/run/pwd/uploads \
+    && ssh-keygen -N "" -t rsa -f  /etc/ssh/ssh_host_rsa_key >/dev/null \
+    && mkdir ~/.ssh && ssh-keygen -N "" -t rsa -f ~/.ssh/id_rsa \
+    && cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys
 
 # Remove IPv6 alias for localhost and start docker in the background ...
 CMD cat /etc/hosts >/etc/hosts.bak && \
@@ -53,7 +57,7 @@ CMD cat /etc/hosts >/etc/hosts.bak && \
     sed -i "s/\DOCKER_TLSCERT/$DOCKER_TLSCERT/" /etc/docker/daemon.json && \
     sed -i "s/\DOCKER_TLSKEY/$DOCKER_TLSKEY/" /etc/docker/daemon.json && \
     umount /var/lib/docker && mount -t securityfs none /sys/kernel/security && \
-    echo "root:root" | chpasswd &> /dev/null && ssh-keygen -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key >/dev/null && \
+    echo "root:root" | chpasswd &> /dev/null && \
     /usr/sbin/sshd -o PermitRootLogin=yes -o PrintMotd=no 2>/dev/null && \
     dockerd &>/docker.log & \
     while true ; do script -q -c "/bin/bash -l" /dev/null ; done