Add routable IP

docker/docker.go

@@ -43,7 +43,8 @@ type DockerApi interface {
 	CreateAttachConnection(name string) (net.Conn, error)
 	CopyToContainer(containerName, destination, fileName string, content io.Reader) error
 	DeleteContainer(id string) error
-	CreateContainer(opts CreateContainerOpts) (string, error)
+	CreateContainer(opts CreateContainerOpts) error
+	GetContainerIPs(id string) (map[string]string, error)
 	ExecAttach(instanceName string, command []string, out io.Writer) (int, error)
 	DisconnectNetwork(containerId, networkId string) error
 	DeleteNetwork(id string) error
@@ -221,7 +222,7 @@ type CreateContainerOpts struct {
 	Networks        map[string]string
 }
 
-func (d *docker) CreateContainer(opts CreateContainerOpts) (string, error) {
+func (d *docker) CreateContainer(opts CreateContainerOpts) error {
 	// Make sure directories are available for the new instance container
 	containerDir := "/var/run/pwd"
 	containerCertDir := fmt.Sprintf("%s/certs", containerDir)
@@ -302,38 +303,47 @@ func (d *docker) CreateContainer(opts CreateContainerOpts) (string, error) {
 		if client.IsErrImageNotFound(err) {
 			log.Printf("Unable to find image '%s' locally\n", opts.Image)
 			if err = d.pullImage(context.Background(), opts.Image); err != nil {
-				return "", err
+				return err
 			}
 			container, err = d.c.ContainerCreate(context.Background(), cf, h, networkConf, opts.ContainerName)
 			if err != nil {
-				return "", err
+				return err
 			}
 		} else {
-			return "", err
+			return err
 		}
 	}
 
 	if err := d.copyIfSet(opts.ServerCert, "cert.pem", containerCertDir, opts.ContainerName); err != nil {
-		return "", err
+		return err
 	}
 	if err := d.copyIfSet(opts.ServerKey, "key.pem", containerCertDir, opts.ContainerName); err != nil {
-		return "", err
+		return err
 	}
 	if err := d.copyIfSet(opts.CACert, "ca.pem", containerCertDir, opts.ContainerName); err != nil {
-		return "", err
+		return err
 	}
 
 	err = d.c.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
 	if err != nil {
-		return "", err
+		return err
 	}
 
-	cinfo, err := d.c.ContainerInspect(context.Background(), container.ID)
+	return nil
+}
+
+func (d *docker) GetContainerIPs(id string) (map[string]string, error) {
+	cinfo, err := d.c.ContainerInspect(context.Background(), id)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
-	return cinfo.NetworkSettings.Networks[opts.SessionId].IPAddress, nil
+	ips := map[string]string{}
+	for networkId, conf := range cinfo.NetworkSettings.Networks {
+		ips[networkId] = conf.IPAddress
+	}
+	return ips, nil
+
 }
 
 func (d *docker) pullImage(ctx context.Context, image string) error {

docker/mock.go

@@ -71,10 +71,15 @@ func (m *Mock) DeleteContainer(id string) error {
 	args := m.Called(id)
 	return args.Error(0)
 }
-func (m *Mock) CreateContainer(opts CreateContainerOpts) (string, error) {
+func (m *Mock) CreateContainer(opts CreateContainerOpts) error {
 	args := m.Called(opts)
-	return args.String(0), args.Error(1)
+	return args.Error(0)
 }
+func (m *Mock) GetContainerIPs(id string) (map[string]string, error) {
+	args := m.Called(id)
+	return args.Get(0).(map[string]string), args.Error(1)
+}
+
 func (m *Mock) ExecAttach(instanceName string, command []string, out io.Writer) (int, error) {
 	args := m.Called(instanceName, command, out)
 	return args.Int(0), args.Error(1)

provisioner/dind.go

@@ -71,14 +71,19 @@ func (d *DinD) InstanceNew(session *types.Session, conf types.InstanceConfig) (*
 	if err != nil {
 		return nil, err
 	}
-	ip, err := dockerClient.CreateContainer(opts)
+	if err := dockerClient.CreateContainer(opts); err != nil {
+		return nil, err
+	}
+
+	ips, err := dockerClient.GetContainerIPs(containerName)
 	if err != nil {
 		return nil, err
 	}
 
 	instance := &types.Instance{}
 	instance.Image = opts.Image
-	instance.IP = ip
+	instance.IP = ips[session.Id]
+	instance.RoutableIP = instance.IP
 	instance.SessionId = session.Id
 	instance.Name = containerName
 	instance.Hostname = conf.Hostname
@@ -88,7 +93,7 @@ func (d *DinD) InstanceNew(session *types.Session, conf types.InstanceConfig) (*
 	instance.ServerKey = conf.ServerKey
 	instance.CACert = conf.CACert
 	instance.Session = session
-	instance.ProxyHost = router.EncodeHost(session.Id, ip, router.HostOpts{})
+	instance.ProxyHost = router.EncodeHost(session.Id, instance.RoutableIP, router.HostOpts{})
 	instance.SessionHost = session.Host
 
 	return instance, nil

provisioner/windows.go

@@ -90,8 +90,7 @@ func (d *windows) InstanceNew(session *types.Session, conf types.InstanceConfig)
 		d.releaseInstance(session.Id, winfo.id)
 		return nil, err
 	}
-	_, err = dockerClient.CreateContainer(opts)
+	if err = dockerClient.CreateContainer(opts); err != nil {
-	if err != nil {
 		d.releaseInstance(session.Id, winfo.id)
 		return nil, err
 	}
@@ -100,6 +99,7 @@ func (d *windows) InstanceNew(session *types.Session, conf types.InstanceConfig)
 	instance.Name = containerName
 	instance.Image = opts.Image
 	instance.IP = winfo.privateIP
+	instance.RoutableIP = instance.IP
 	instance.SessionId = session.Id
 	instance.WindowsId = winfo.id
 	instance.Cert = conf.Cert
@@ -109,7 +109,7 @@ func (d *windows) InstanceNew(session *types.Session, conf types.InstanceConfig)
 	instance.ServerKey = conf.ServerKey
 	instance.CACert = conf.CACert
 	instance.Session = session
-	instance.ProxyHost = router.EncodeHost(session.Id, instance.IP, router.HostOpts{})
+	instance.ProxyHost = router.EncodeHost(session.Id, instance.RoutableIP, router.HostOpts{})
 	instance.SessionHost = session.Host
 
 	if cli, err := d.factory.GetForInstance(instance); err != nil {

pwd/instance_test.go

@@ -72,6 +72,7 @@ func TestInstanceNew(t *testing.T) {
 		Name:        fmt.Sprintf("%s_node1", session.Id[:8]),
 		Hostname:    "node1",
 		IP:          "10.0.0.1",
+		RoutableIP:  "10.0.0.1",
 		Image:       config.GetDindImageName(),
 		SessionId:   session.Id,
 		Session:     session,
@@ -91,7 +92,8 @@ func TestInstanceNew(t *testing.T) {
 		HostFQDN:      "something.play-with-docker.com",
 		Networks:      map[string]string{session.Id: expectedInstance.Hostname},
 	}
-	_d.On("CreateContainer", expectedContainerOpts).Return("10.0.0.1", nil)
+	_d.On("CreateContainer", expectedContainerOpts).Return(nil)
+	_d.On("GetContainerIPs", expectedInstance.Name).Return(map[string]string{session.Id: "10.0.0.1"}, nil)
 	_s.On("InstanceCreate", "aaaabbbbcccc", mock.AnythingOfType("*types.Instance")).Return(nil)
 	_e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_node1", "10.0.0.1", "node1", "ip10-0-0-1-aaaabbbbcccc"}).Return()
 
@@ -139,6 +141,7 @@ func TestInstanceNew_WithNotAllowedImage(t *testing.T) {
 		Name:        fmt.Sprintf("%s_node1", session.Id[:8]),
 		Hostname:    "node1",
 		IP:          "10.0.0.1",
+		RoutableIP:  "10.0.0.1",
 		Image:       "redis",
 		SessionId:   session.Id,
 		Session:     session,
@@ -157,7 +160,8 @@ func TestInstanceNew_WithNotAllowedImage(t *testing.T) {
 		Privileged:    true,
 		Networks:      map[string]string{session.Id: expectedInstance.Hostname},
 	}
-	_d.On("CreateContainer", expectedContainerOpts).Return("10.0.0.1", nil)
+	_d.On("CreateContainer", expectedContainerOpts).Return(nil)
+	_d.On("GetContainerIPs", expectedInstance.Name).Return(map[string]string{session.Id: "10.0.0.1"}, nil)
 	_s.On("InstanceCreate", "aaaabbbbcccc", mock.AnythingOfType("*types.Instance")).Return(nil)
 	_e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_node1", "10.0.0.1", "node1", "ip10-0-0-1-aaaabbbbcccc"}).Return()
 
@@ -205,6 +209,7 @@ func TestInstanceNew_WithCustomHostname(t *testing.T) {
 		Name:        fmt.Sprintf("%s_redis-master", session.Id[:8]),
 		Hostname:    "redis-master",
 		IP:          "10.0.0.1",
+		RoutableIP:  "10.0.0.1",
 		Image:       "redis",
 		Session:     session,
 		SessionHost: session.Host,
@@ -224,7 +229,8 @@ func TestInstanceNew_WithCustomHostname(t *testing.T) {
 		Networks:      map[string]string{session.Id: expectedInstance.Hostname},
 	}
 
-	_d.On("CreateContainer", expectedContainerOpts).Return("10.0.0.1", nil)
+	_d.On("CreateContainer", expectedContainerOpts).Return(nil)
+	_d.On("GetContainerIPs", expectedInstance.Name).Return(map[string]string{session.Id: "10.0.0.1"}, nil)
 	_s.On("InstanceCreate", "aaaabbbbcccc", mock.AnythingOfType("*types.Instance")).Return(nil)
 	_e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_redis-master", "10.0.0.1", "redis-master", "ip10-0-0-1-aaaabbbbcccc"}).Return()
 

pwd/session_test.go

@@ -90,27 +90,32 @@ func TestSessionSetup(t *testing.T) {
 	_s.On("SessionCount").Return(1, nil)
 	_s.On("InstanceCount").Return(0, nil)
 
-	_d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_manager1", Hostname: "manager1", Privileged: true, HostFQDN: "localhost", Networks: map[string]string{"aaaabbbbcccc": "manager1"}}).Return("10.0.0.2", nil)
+	_d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_manager1", Hostname: "manager1", Privileged: true, HostFQDN: "localhost", Networks: map[string]string{"aaaabbbbcccc": "manager1"}}).Return(nil)
+	_d.On("GetContainerIPs", "aaaabbbb_manager1").Return(map[string]string{"aaaabbbbcccc": "10.0.0.2"}, nil)
 	_f.On("GetForInstance", mock.AnythingOfType("*types.Instance")).Return(_d, nil)
 	_d.On("SwarmInit").Return(&docker.SwarmTokens{Manager: "managerToken", Worker: "workerToken"}, nil)
 	_e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_manager1", "10.0.0.2", "manager1", "ip10-0-0-2-aaaabbbbcccc"}).Return()
 
-	_d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_manager2", Hostname: "manager2", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "manager2"}}).Return("10.0.0.3", nil)
+	_d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_manager2", Hostname: "manager2", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "manager2"}}).Return(nil)
+	_d.On("GetContainerIPs", "aaaabbbb_manager2").Return(map[string]string{"aaaabbbbcccc": "10.0.0.3"}, nil)
 	_f.On("GetForInstance", mock.AnythingOfType("*types.Instance")).Return(_d, nil)
 	_d.On("SwarmJoin", "10.0.0.2:2377", "managerToken").Return(nil)
 	_e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_manager2", "10.0.0.3", "manager2", "ip10-0-0-3-aaaabbbbcccc"}).Return()
 
-	_d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind:overlay2-dev", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_manager3", Hostname: "manager3", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "manager3"}}).Return("10.0.0.4", nil)
+	_d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind:overlay2-dev", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_manager3", Hostname: "manager3", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "manager3"}}).Return(nil)
+	_d.On("GetContainerIPs", "aaaabbbb_manager3").Return(map[string]string{"aaaabbbbcccc": "10.0.0.4"}, nil)
 	_f.On("GetForInstance", mock.AnythingOfType("*types.Instance")).Return(_d, nil)
 	_d.On("SwarmJoin", "10.0.0.2:2377", "managerToken").Return(nil)
 	_e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_manager3", "10.0.0.4", "manager3", "ip10-0-0-4-aaaabbbbcccc"}).Return()
 
-	_d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_worker1", Hostname: "worker1", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "worker1"}}).Return("10.0.0.5", nil)
+	_d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_worker1", Hostname: "worker1", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "worker1"}}).Return(nil)
+	_d.On("GetContainerIPs", "aaaabbbb_worker1").Return(map[string]string{"aaaabbbbcccc": "10.0.0.5"}, nil)
 	_f.On("GetForInstance", mock.AnythingOfType("*types.Instance")).Return(_d, nil)
 	_d.On("SwarmJoin", "10.0.0.2:2377", "workerToken").Return(nil)
 	_e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_worker1", "10.0.0.5", "worker1", "ip10-0-0-5-aaaabbbbcccc"}).Return()
 
-	_d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_other", Hostname: "other", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "other"}}).Return("10.0.0.6", nil)
+	_d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_other", Hostname: "other", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "other"}}).Return(nil)
+	_d.On("GetContainerIPs", "aaaabbbb_other").Return(map[string]string{"aaaabbbbcccc": "10.0.0.6"}, nil)
 	_e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_other", "10.0.0.6", "other", "ip10-0-0-6-aaaabbbbcccc"}).Return()
 
 	var nilArgs []interface{}

pwd/types/instance.go

@@ -7,6 +7,7 @@ type Instance struct {
 	Name        string          `json:"name" bson:"name"`
 	Hostname    string          `json:"hostname" bson:"hostname"`
 	IP          string          `json:"ip" bson:"ip"`
+	RoutableIP  string          `json:"routable_ip" bson:"routable_id"`
 	ServerCert  []byte          `json:"server_cert" bson:"server_cert"`
 	ServerKey   []byte          `json:"server_key" bson:"server_key"`
 	CACert      []byte          `json:"ca_cert" bson:"ca_cert"`