diff --git a/docker/docker.go b/docker/docker.go index a4c5b36..4bfe735 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -43,7 +43,8 @@ type DockerApi interface { CreateAttachConnection(name string) (net.Conn, error) CopyToContainer(containerName, destination, fileName string, content io.Reader) error DeleteContainer(id string) error - CreateContainer(opts CreateContainerOpts) (string, error) + CreateContainer(opts CreateContainerOpts) error + GetContainerIPs(id string) (map[string]string, error) ExecAttach(instanceName string, command []string, out io.Writer) (int, error) DisconnectNetwork(containerId, networkId string) error DeleteNetwork(id string) error @@ -221,7 +222,7 @@ type CreateContainerOpts struct { Networks map[string]string } -func (d *docker) CreateContainer(opts CreateContainerOpts) (string, error) { +func (d *docker) CreateContainer(opts CreateContainerOpts) error { // Make sure directories are available for the new instance container containerDir := "/var/run/pwd" containerCertDir := fmt.Sprintf("%s/certs", containerDir) @@ -302,38 +303,47 @@ func (d *docker) CreateContainer(opts CreateContainerOpts) (string, error) { if client.IsErrImageNotFound(err) { log.Printf("Unable to find image '%s' locally\n", opts.Image) if err = d.pullImage(context.Background(), opts.Image); err != nil { - return "", err + return err } container, err = d.c.ContainerCreate(context.Background(), cf, h, networkConf, opts.ContainerName) if err != nil { - return "", err + return err } } else { - return "", err + return err } } if err := d.copyIfSet(opts.ServerCert, "cert.pem", containerCertDir, opts.ContainerName); err != nil { - return "", err + return err } if err := d.copyIfSet(opts.ServerKey, "key.pem", containerCertDir, opts.ContainerName); err != nil { - return "", err + return err } if err := d.copyIfSet(opts.CACert, "ca.pem", containerCertDir, opts.ContainerName); err != nil { - return "", err + return err } err = d.c.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{}) if err != nil { - return "", err + return err } - cinfo, err := d.c.ContainerInspect(context.Background(), container.ID) + return nil +} + +func (d *docker) GetContainerIPs(id string) (map[string]string, error) { + cinfo, err := d.c.ContainerInspect(context.Background(), id) if err != nil { - return "", err + return nil, err } - return cinfo.NetworkSettings.Networks[opts.SessionId].IPAddress, nil + ips := map[string]string{} + for networkId, conf := range cinfo.NetworkSettings.Networks { + ips[networkId] = conf.IPAddress + } + return ips, nil + } func (d *docker) pullImage(ctx context.Context, image string) error { diff --git a/docker/mock.go b/docker/mock.go index 9c7112c..b218573 100644 --- a/docker/mock.go +++ b/docker/mock.go @@ -71,10 +71,15 @@ func (m *Mock) DeleteContainer(id string) error { args := m.Called(id) return args.Error(0) } -func (m *Mock) CreateContainer(opts CreateContainerOpts) (string, error) { +func (m *Mock) CreateContainer(opts CreateContainerOpts) error { args := m.Called(opts) - return args.String(0), args.Error(1) + return args.Error(0) } +func (m *Mock) GetContainerIPs(id string) (map[string]string, error) { + args := m.Called(id) + return args.Get(0).(map[string]string), args.Error(1) +} + func (m *Mock) ExecAttach(instanceName string, command []string, out io.Writer) (int, error) { args := m.Called(instanceName, command, out) return args.Int(0), args.Error(1) diff --git a/provisioner/dind.go b/provisioner/dind.go index c35c6e7..477aee0 100644 --- a/provisioner/dind.go +++ b/provisioner/dind.go @@ -71,14 +71,19 @@ func (d *DinD) InstanceNew(session *types.Session, conf types.InstanceConfig) (* if err != nil { return nil, err } - ip, err := dockerClient.CreateContainer(opts) + if err := dockerClient.CreateContainer(opts); err != nil { + return nil, err + } + + ips, err := dockerClient.GetContainerIPs(containerName) if err != nil { return nil, err } instance := &types.Instance{} instance.Image = opts.Image - instance.IP = ip + instance.IP = ips[session.Id] + instance.RoutableIP = instance.IP instance.SessionId = session.Id instance.Name = containerName instance.Hostname = conf.Hostname @@ -88,7 +93,7 @@ func (d *DinD) InstanceNew(session *types.Session, conf types.InstanceConfig) (* instance.ServerKey = conf.ServerKey instance.CACert = conf.CACert instance.Session = session - instance.ProxyHost = router.EncodeHost(session.Id, ip, router.HostOpts{}) + instance.ProxyHost = router.EncodeHost(session.Id, instance.RoutableIP, router.HostOpts{}) instance.SessionHost = session.Host return instance, nil diff --git a/provisioner/windows.go b/provisioner/windows.go index 324f7ff..e99f5e9 100644 --- a/provisioner/windows.go +++ b/provisioner/windows.go @@ -90,8 +90,7 @@ func (d *windows) InstanceNew(session *types.Session, conf types.InstanceConfig) d.releaseInstance(session.Id, winfo.id) return nil, err } - _, err = dockerClient.CreateContainer(opts) - if err != nil { + if err = dockerClient.CreateContainer(opts); err != nil { d.releaseInstance(session.Id, winfo.id) return nil, err } @@ -100,6 +99,7 @@ func (d *windows) InstanceNew(session *types.Session, conf types.InstanceConfig) instance.Name = containerName instance.Image = opts.Image instance.IP = winfo.privateIP + instance.RoutableIP = instance.IP instance.SessionId = session.Id instance.WindowsId = winfo.id instance.Cert = conf.Cert @@ -109,7 +109,7 @@ func (d *windows) InstanceNew(session *types.Session, conf types.InstanceConfig) instance.ServerKey = conf.ServerKey instance.CACert = conf.CACert instance.Session = session - instance.ProxyHost = router.EncodeHost(session.Id, instance.IP, router.HostOpts{}) + instance.ProxyHost = router.EncodeHost(session.Id, instance.RoutableIP, router.HostOpts{}) instance.SessionHost = session.Host if cli, err := d.factory.GetForInstance(instance); err != nil { diff --git a/pwd/instance_test.go b/pwd/instance_test.go index 39c1646..a5dfc9b 100644 --- a/pwd/instance_test.go +++ b/pwd/instance_test.go @@ -72,6 +72,7 @@ func TestInstanceNew(t *testing.T) { Name: fmt.Sprintf("%s_node1", session.Id[:8]), Hostname: "node1", IP: "10.0.0.1", + RoutableIP: "10.0.0.1", Image: config.GetDindImageName(), SessionId: session.Id, Session: session, @@ -91,7 +92,8 @@ func TestInstanceNew(t *testing.T) { HostFQDN: "something.play-with-docker.com", Networks: map[string]string{session.Id: expectedInstance.Hostname}, } - _d.On("CreateContainer", expectedContainerOpts).Return("10.0.0.1", nil) + _d.On("CreateContainer", expectedContainerOpts).Return(nil) + _d.On("GetContainerIPs", expectedInstance.Name).Return(map[string]string{session.Id: "10.0.0.1"}, nil) _s.On("InstanceCreate", "aaaabbbbcccc", mock.AnythingOfType("*types.Instance")).Return(nil) _e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_node1", "10.0.0.1", "node1", "ip10-0-0-1-aaaabbbbcccc"}).Return() @@ -139,6 +141,7 @@ func TestInstanceNew_WithNotAllowedImage(t *testing.T) { Name: fmt.Sprintf("%s_node1", session.Id[:8]), Hostname: "node1", IP: "10.0.0.1", + RoutableIP: "10.0.0.1", Image: "redis", SessionId: session.Id, Session: session, @@ -157,7 +160,8 @@ func TestInstanceNew_WithNotAllowedImage(t *testing.T) { Privileged: true, Networks: map[string]string{session.Id: expectedInstance.Hostname}, } - _d.On("CreateContainer", expectedContainerOpts).Return("10.0.0.1", nil) + _d.On("CreateContainer", expectedContainerOpts).Return(nil) + _d.On("GetContainerIPs", expectedInstance.Name).Return(map[string]string{session.Id: "10.0.0.1"}, nil) _s.On("InstanceCreate", "aaaabbbbcccc", mock.AnythingOfType("*types.Instance")).Return(nil) _e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_node1", "10.0.0.1", "node1", "ip10-0-0-1-aaaabbbbcccc"}).Return() @@ -205,6 +209,7 @@ func TestInstanceNew_WithCustomHostname(t *testing.T) { Name: fmt.Sprintf("%s_redis-master", session.Id[:8]), Hostname: "redis-master", IP: "10.0.0.1", + RoutableIP: "10.0.0.1", Image: "redis", Session: session, SessionHost: session.Host, @@ -224,7 +229,8 @@ func TestInstanceNew_WithCustomHostname(t *testing.T) { Networks: map[string]string{session.Id: expectedInstance.Hostname}, } - _d.On("CreateContainer", expectedContainerOpts).Return("10.0.0.1", nil) + _d.On("CreateContainer", expectedContainerOpts).Return(nil) + _d.On("GetContainerIPs", expectedInstance.Name).Return(map[string]string{session.Id: "10.0.0.1"}, nil) _s.On("InstanceCreate", "aaaabbbbcccc", mock.AnythingOfType("*types.Instance")).Return(nil) _e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_redis-master", "10.0.0.1", "redis-master", "ip10-0-0-1-aaaabbbbcccc"}).Return() diff --git a/pwd/session_test.go b/pwd/session_test.go index a78424d..07e0cec 100644 --- a/pwd/session_test.go +++ b/pwd/session_test.go @@ -90,27 +90,32 @@ func TestSessionSetup(t *testing.T) { _s.On("SessionCount").Return(1, nil) _s.On("InstanceCount").Return(0, nil) - _d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_manager1", Hostname: "manager1", Privileged: true, HostFQDN: "localhost", Networks: map[string]string{"aaaabbbbcccc": "manager1"}}).Return("10.0.0.2", nil) + _d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_manager1", Hostname: "manager1", Privileged: true, HostFQDN: "localhost", Networks: map[string]string{"aaaabbbbcccc": "manager1"}}).Return(nil) + _d.On("GetContainerIPs", "aaaabbbb_manager1").Return(map[string]string{"aaaabbbbcccc": "10.0.0.2"}, nil) _f.On("GetForInstance", mock.AnythingOfType("*types.Instance")).Return(_d, nil) _d.On("SwarmInit").Return(&docker.SwarmTokens{Manager: "managerToken", Worker: "workerToken"}, nil) _e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_manager1", "10.0.0.2", "manager1", "ip10-0-0-2-aaaabbbbcccc"}).Return() - _d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_manager2", Hostname: "manager2", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "manager2"}}).Return("10.0.0.3", nil) + _d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_manager2", Hostname: "manager2", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "manager2"}}).Return(nil) + _d.On("GetContainerIPs", "aaaabbbb_manager2").Return(map[string]string{"aaaabbbbcccc": "10.0.0.3"}, nil) _f.On("GetForInstance", mock.AnythingOfType("*types.Instance")).Return(_d, nil) _d.On("SwarmJoin", "10.0.0.2:2377", "managerToken").Return(nil) _e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_manager2", "10.0.0.3", "manager2", "ip10-0-0-3-aaaabbbbcccc"}).Return() - _d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind:overlay2-dev", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_manager3", Hostname: "manager3", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "manager3"}}).Return("10.0.0.4", nil) + _d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind:overlay2-dev", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_manager3", Hostname: "manager3", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "manager3"}}).Return(nil) + _d.On("GetContainerIPs", "aaaabbbb_manager3").Return(map[string]string{"aaaabbbbcccc": "10.0.0.4"}, nil) _f.On("GetForInstance", mock.AnythingOfType("*types.Instance")).Return(_d, nil) _d.On("SwarmJoin", "10.0.0.2:2377", "managerToken").Return(nil) _e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_manager3", "10.0.0.4", "manager3", "ip10-0-0-4-aaaabbbbcccc"}).Return() - _d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_worker1", Hostname: "worker1", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "worker1"}}).Return("10.0.0.5", nil) + _d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_worker1", Hostname: "worker1", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "worker1"}}).Return(nil) + _d.On("GetContainerIPs", "aaaabbbb_worker1").Return(map[string]string{"aaaabbbbcccc": "10.0.0.5"}, nil) _f.On("GetForInstance", mock.AnythingOfType("*types.Instance")).Return(_d, nil) _d.On("SwarmJoin", "10.0.0.2:2377", "workerToken").Return(nil) _e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_worker1", "10.0.0.5", "worker1", "ip10-0-0-5-aaaabbbbcccc"}).Return() - _d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_other", Hostname: "other", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "other"}}).Return("10.0.0.6", nil) + _d.On("CreateContainer", docker.CreateContainerOpts{Image: "franela/dind", SessionId: "aaaabbbbcccc", PwdIpAddress: "10.0.0.1", ContainerName: "aaaabbbb_other", Hostname: "other", Privileged: true, Networks: map[string]string{"aaaabbbbcccc": "other"}}).Return(nil) + _d.On("GetContainerIPs", "aaaabbbb_other").Return(map[string]string{"aaaabbbbcccc": "10.0.0.6"}, nil) _e.M.On("Emit", event.INSTANCE_NEW, "aaaabbbbcccc", []interface{}{"aaaabbbb_other", "10.0.0.6", "other", "ip10-0-0-6-aaaabbbbcccc"}).Return() var nilArgs []interface{} diff --git a/pwd/types/instance.go b/pwd/types/instance.go index a42b136..18c5e7b 100644 --- a/pwd/types/instance.go +++ b/pwd/types/instance.go @@ -7,6 +7,7 @@ type Instance struct { Name string `json:"name" bson:"name"` Hostname string `json:"hostname" bson:"hostname"` IP string `json:"ip" bson:"ip"` + RoutableIP string `json:"routable_ip" bson:"routable_id"` ServerCert []byte `json:"server_cert" bson:"server_cert"` ServerKey []byte `json:"server_key" bson:"server_key"` CACert []byte `json:"ca_cert" bson:"ca_cert"`