WIP
This commit is contained in:
@@ -2,7 +2,6 @@ package config
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"os"
|
||||
"regexp"
|
||||
"time"
|
||||
@@ -21,7 +20,7 @@ const (
|
||||
var NameFilter = regexp.MustCompile(PWDHostPortGroupRegex)
|
||||
var AliasFilter = regexp.MustCompile(AliasPortGroupRegex)
|
||||
|
||||
var SSLPortNumber, PortNumber, Key, Cert, SessionsFile, PWDContainerName, PWDCName, HashKey, SSHKeyPath string
|
||||
var SSLPortNumber, PortNumber, Key, Cert, SessionsFile, PWDContainerName, L2ContainerName, L2Subdomain, PWDCName, HashKey, SSHKeyPath string
|
||||
var MaxLoadAvg float64
|
||||
|
||||
func ParseFlags() {
|
||||
@@ -31,13 +30,13 @@ func ParseFlags() {
|
||||
flag.StringVar(&Cert, "cert", "./pwd/server.pem", "Give a SSL cert")
|
||||
flag.StringVar(&SessionsFile, "save", "./pwd/sessions", "Tell where to store sessions file")
|
||||
flag.StringVar(&PWDContainerName, "name", "pwd", "Container name used to run PWD (used to be able to connect it to the networks it creates)")
|
||||
flag.StringVar(&PWDCName, "cname", "host1", "CNAME given to this host")
|
||||
flag.StringVar(&L2ContainerName, "l2", "l2", "Container name used to run L2 Router")
|
||||
flag.StringVar(&L2Subdomain, "l2-subdomain", "direct", "Subdomain to the L2 Router")
|
||||
flag.StringVar(&PWDCName, "cname", "", "CNAME given to this host")
|
||||
flag.StringVar(&HashKey, "hash_key", "salmonrosado", "Hash key to use for cookies")
|
||||
flag.Float64Var(&MaxLoadAvg, "maxload", 100, "Maximum allowed load average before failing ping requests")
|
||||
flag.StringVar(&SSHKeyPath, "ssh_key_path", "", "SSH Private Key to use")
|
||||
flag.Parse()
|
||||
|
||||
log.Println("*****************************", SSHKeyPath)
|
||||
}
|
||||
func GetDindImageName() string {
|
||||
dindImage := os.Getenv("DIND_IMAGE")
|
||||
|
||||
@@ -1,21 +1,20 @@
|
||||
version: '3.2'
|
||||
services:
|
||||
haproxy:
|
||||
container_name: proxy
|
||||
container_name: haproxy
|
||||
image: haproxy
|
||||
ports:
|
||||
- "80:8080"
|
||||
- "443:8443"
|
||||
|
||||
volumes:
|
||||
- ./haproxy:/usr/local/etc/haproxy
|
||||
pwd1:
|
||||
|
||||
pwd:
|
||||
# pwd daemon container always needs to be named this way
|
||||
container_name: pwd1
|
||||
container_name: pwd
|
||||
# use the latest golang image
|
||||
image: golang
|
||||
# go to the right place and starts the app
|
||||
command: /bin/sh -c 'ssh-keygen -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key >/dev/null; cd /go/src/github.com/play-with-docker/play-with-docker; go run api.go -save /pwd/sessions1 -name pwd1 -cname host1'
|
||||
command: /bin/sh -c 'ssh-keygen -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key >/dev/null; cd /go/src/github.com/play-with-docker/play-with-docker; go run api.go -save /pwd/sessions -name l2'
|
||||
volumes:
|
||||
# since this app creates networks and launches containers, we need to talk to docker daemon
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
@@ -24,37 +23,20 @@ services:
|
||||
- sessions:/pwd
|
||||
environment:
|
||||
GOOGLE_RECAPTCHA_DISABLED: "true"
|
||||
ports:
|
||||
- "1022:1022"
|
||||
pwd2:
|
||||
# pwd daemon container always needs to be named this way
|
||||
container_name: pwd2
|
||||
l2:
|
||||
container_name: l2
|
||||
# use the latest golang image
|
||||
image: golang
|
||||
# go to the right place and starts the app
|
||||
command: /bin/sh -c 'ssh-keygen -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key >/dev/null; cd /go/src/github.com/play-with-docker/play-with-docker; go run api.go -save /pwd/sessions2 -name pwd2 -cname host2'
|
||||
command: /bin/sh -c 'ssh-keygen -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key >/dev/null; cd /go/src/github.com/play-with-docker/play-with-docker/router/l2; go run l2.go -ssh_key_path /etc/ssh/ssh_host_rsa_key -name l2 -save /pwd/networks'
|
||||
volumes:
|
||||
# since this app creates networks and launches containers, we need to talk to docker daemon
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
# mount the box mounted shared folder to the container
|
||||
- $GOPATH/src:/go/src
|
||||
- sessions:/pwd
|
||||
environment:
|
||||
GOOGLE_RECAPTCHA_DISABLED: "true"
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- networks:/pwd
|
||||
ports:
|
||||
- "1023:1022"
|
||||
prometheus:
|
||||
container_name: prometheus
|
||||
image: prom/prometheus
|
||||
volumes:
|
||||
- ./prometheus.yml:/etc/prometheus/prometheus.yml
|
||||
grafana:
|
||||
container_name: grafana
|
||||
image: grafana/grafana
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- grafana:/var/lib/grafana
|
||||
- "8022:22"
|
||||
- "8053:53"
|
||||
- "443:443"
|
||||
volumes:
|
||||
sessions:
|
||||
grafana:
|
||||
networks:
|
||||
|
||||
@@ -4,11 +4,10 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/docker/docker/client"
|
||||
"github.com/googollee/go-socket.io"
|
||||
"github.com/play-with-docker/play-with-docker/config"
|
||||
"github.com/play-with-docker/play-with-docker/docker"
|
||||
"github.com/play-with-docker/play-with-docker/event"
|
||||
"github.com/play-with-docker/play-with-docker/provider"
|
||||
"github.com/play-with-docker/play-with-docker/pwd"
|
||||
"github.com/play-with-docker/play-with-docker/storage"
|
||||
)
|
||||
@@ -18,23 +17,18 @@ var e event.EventApi
|
||||
var ws *socketio.Server
|
||||
|
||||
func Bootstrap() {
|
||||
c, err := client.NewEnvClient()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
d := docker.NewDocker(c)
|
||||
sp := provider.NewLocalSessionProvider()
|
||||
|
||||
e = event.NewLocalBroker()
|
||||
|
||||
t := pwd.NewScheduler(e, d)
|
||||
t := pwd.NewScheduler(e, sp)
|
||||
|
||||
s, err := storage.NewFileStorage(config.SessionsFile)
|
||||
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
log.Fatal("Error initializing StorageAPI: ", err)
|
||||
}
|
||||
core = pwd.NewPWD(d, t, e, s)
|
||||
core = pwd.NewPWD(sp, t, e, s)
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -49,7 +49,10 @@ func NewSession(rw http.ResponseWriter, req *http.Request) {
|
||||
log.Println(err)
|
||||
//TODO: Return some error code
|
||||
} else {
|
||||
hostname := fmt.Sprintf("%s.%s", config.PWDCName, req.Host)
|
||||
hostname := req.Host
|
||||
if config.PWDCName != "" {
|
||||
hostname = fmt.Sprintf("%s.%s", config.PWDCName, req.Host)
|
||||
}
|
||||
// If request is not a form, return sessionId in the body
|
||||
if req.Header.Get("X-Requested-With") == "XMLHttpRequest" {
|
||||
resp := NewSessionResponse{SessionId: s.Id, Hostname: hostname}
|
||||
|
||||
@@ -8,25 +8,13 @@ frontend http-in
|
||||
bind *:8080
|
||||
|
||||
acl host_localhost hdr(host) localhost
|
||||
acl host_pwd1 hdr_reg(host) -i ^.*\.?host1\.localhost?:?.*$
|
||||
acl host_pwd2 hdr_reg(host) -i ^.*\.?host2\.localhost?:?.*$
|
||||
acl host_direct_localhost hdr_reg(host) -i ^.*\.direct\.localhost?:?.*$
|
||||
|
||||
use_backend all if host_localhost
|
||||
use_backend pwd1 if host_pwd1
|
||||
use_backend pwd2 if host_pwd2
|
||||
use_backend pwd if host_localhost
|
||||
use_backend l2 if host_direct_localhost
|
||||
|
||||
backend all
|
||||
balance roundrobin
|
||||
backend pwd
|
||||
server node1 pwd:3000
|
||||
|
||||
option httpchk GET /ping HTTP/1.0
|
||||
http-check expect rstatus 200
|
||||
default-server inter 3s fall 3 rise 2
|
||||
|
||||
server node1 pwd1:3000 check
|
||||
server node2 pwd2:3000 check
|
||||
|
||||
backend pwd1
|
||||
server node1 pwd1:3000
|
||||
|
||||
backend pwd2
|
||||
server node2 pwd2:3000
|
||||
backend l2
|
||||
server node2 l2:443
|
||||
|
||||
36
provider/local_session_provider.go
Normal file
36
provider/local_session_provider.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/docker/docker/client"
|
||||
"github.com/play-with-docker/play-with-docker/docker"
|
||||
)
|
||||
|
||||
type localSessionProvider struct {
|
||||
rw sync.Mutex
|
||||
|
||||
docker docker.DockerApi
|
||||
}
|
||||
|
||||
func (p *localSessionProvider) GetDocker(sessionId string) (docker.DockerApi, error) {
|
||||
p.rw.Lock()
|
||||
defer p.rw.Unlock()
|
||||
|
||||
if p.docker != nil {
|
||||
return p.docker, nil
|
||||
}
|
||||
|
||||
c, err := client.NewEnvClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := docker.NewDocker(c)
|
||||
|
||||
p.docker = d
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func NewLocalSessionProvider() *localSessionProvider {
|
||||
return &localSessionProvider{}
|
||||
}
|
||||
10
provider/provider.go
Normal file
10
provider/provider.go
Normal file
@@ -0,0 +1,10 @@
|
||||
package provider
|
||||
|
||||
import "github.com/play-with-docker/play-with-docker/docker"
|
||||
|
||||
type InstanceProvider interface {
|
||||
}
|
||||
|
||||
type SessionProvider interface {
|
||||
GetDocker(sessionId string) (docker.DockerApi, error)
|
||||
}
|
||||
@@ -11,12 +11,13 @@ import (
|
||||
)
|
||||
|
||||
func TestClientNew(t *testing.T) {
|
||||
docker := &mockDocker{}
|
||||
d := &mockDocker{}
|
||||
tasks := &mockTasks{}
|
||||
e := event.NewLocalBroker()
|
||||
storage := &mockStorage{}
|
||||
sp := &mockSessionProvider{docker: d}
|
||||
|
||||
p := NewPWD(docker, tasks, e, storage)
|
||||
p := NewPWD(sp, tasks, e, storage)
|
||||
|
||||
session, err := p.SessionNew(time.Hour, "", "", "")
|
||||
assert.Nil(t, err)
|
||||
@@ -27,12 +28,13 @@ func TestClientNew(t *testing.T) {
|
||||
assert.Contains(t, session.Clients, client)
|
||||
}
|
||||
func TestClientCount(t *testing.T) {
|
||||
docker := &mockDocker{}
|
||||
d := &mockDocker{}
|
||||
tasks := &mockTasks{}
|
||||
e := event.NewLocalBroker()
|
||||
storage := &mockStorage{}
|
||||
sp := &mockSessionProvider{docker: d}
|
||||
|
||||
p := NewPWD(docker, tasks, e, storage)
|
||||
p := NewPWD(sp, tasks, e, storage)
|
||||
|
||||
session, err := p.SessionNew(time.Hour, "", "", "")
|
||||
assert.Nil(t, err)
|
||||
@@ -45,9 +47,10 @@ func TestClientCount(t *testing.T) {
|
||||
func TestClientResizeViewPort(t *testing.T) {
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(1)
|
||||
docker := &mockDocker{}
|
||||
d := &mockDocker{}
|
||||
tasks := &mockTasks{}
|
||||
e := event.NewLocalBroker()
|
||||
sp := &mockSessionProvider{docker: d}
|
||||
|
||||
broadcastedSessionId := ""
|
||||
broadcastedArgs := []interface{}{}
|
||||
@@ -60,7 +63,7 @@ func TestClientResizeViewPort(t *testing.T) {
|
||||
|
||||
storage := &mockStorage{}
|
||||
|
||||
p := NewPWD(docker, tasks, e, storage)
|
||||
p := NewPWD(sp, tasks, e, storage)
|
||||
|
||||
session, err := p.SessionNew(time.Hour, "", "", "")
|
||||
assert.Nil(t, err)
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
|
||||
dockerTypes "github.com/docker/docker/api/types"
|
||||
units "github.com/docker/go-units"
|
||||
"github.com/play-with-docker/play-with-docker/docker"
|
||||
"github.com/play-with-docker/play-with-docker/provider"
|
||||
"github.com/play-with-docker/play-with-docker/pwd/types"
|
||||
)
|
||||
|
||||
@@ -20,11 +20,12 @@ type collectStatsTask struct {
|
||||
previousCPU uint64
|
||||
previousSystem uint64
|
||||
|
||||
docker docker.DockerApi
|
||||
sessionProvider provider.SessionProvider
|
||||
}
|
||||
|
||||
func (c collectStatsTask) Run(i *types.Instance) error {
|
||||
reader, err := c.docker.GetContainerStats(i.Name)
|
||||
docker, _ := c.sessionProvider.GetDocker(i.SessionId)
|
||||
reader, err := docker.GetContainerStats(i.Name)
|
||||
if err != nil {
|
||||
log.Println("Error while trying to collect instance stats", err)
|
||||
return err
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/play-with-docker/play-with-docker/docker"
|
||||
"github.com/play-with-docker/play-with-docker/event"
|
||||
"github.com/play-with-docker/play-with-docker/pwd/types"
|
||||
"github.com/play-with-docker/play-with-docker/router"
|
||||
|
||||
"golang.org/x/text/encoding"
|
||||
)
|
||||
@@ -46,7 +47,7 @@ type InstanceConfig struct {
|
||||
|
||||
func (p *pwd) InstanceResizeTerminal(instance *types.Instance, rows, cols uint) error {
|
||||
defer observeAction("InstanceResizeTerminal", time.Now())
|
||||
return p.docker.ContainerResize(instance.Name, rows, cols)
|
||||
return p.docker(instance.SessionId).ContainerResize(instance.Name, rows, cols)
|
||||
}
|
||||
|
||||
func (p *pwd) InstanceAttachTerminal(instance *types.Instance) error {
|
||||
@@ -54,7 +55,7 @@ func (p *pwd) InstanceAttachTerminal(instance *types.Instance) error {
|
||||
if getInstanceTermConn(instance.SessionId, instance.Name) != nil {
|
||||
return nil
|
||||
}
|
||||
conn, err := p.docker.CreateAttachConnection(instance.Name)
|
||||
conn, err := p.docker(instance.SessionId).CreateAttachConnection(instance.Name)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -84,7 +85,7 @@ func (p *pwd) InstanceUploadFromUrl(instance *types.Instance, fileName, dest str
|
||||
return fmt.Errorf("Could not download file [%s]. Status code: %d\n", url, resp.StatusCode)
|
||||
}
|
||||
|
||||
copyErr := p.docker.CopyToContainer(instance.Name, dest, fileName, resp.Body)
|
||||
copyErr := p.docker(instance.SessionId).CopyToContainer(instance.Name, dest, fileName, resp.Body)
|
||||
|
||||
if copyErr != nil {
|
||||
return fmt.Errorf("Error while downloading file [%s]. Error: %s\n", url, copyErr)
|
||||
@@ -96,7 +97,7 @@ func (p *pwd) InstanceUploadFromUrl(instance *types.Instance, fileName, dest str
|
||||
func (p *pwd) getInstanceCWD(instance *types.Instance) (string, error) {
|
||||
b := bytes.NewBufferString("")
|
||||
|
||||
if c, err := p.docker.ExecAttach(instance.Name, []string{"bash", "-c", `pwdx $(</var/run/cwd)`}, b); c > 0 {
|
||||
if c, err := p.docker(instance.SessionId).ExecAttach(instance.Name, []string{"bash", "-c", `pwdx $(</var/run/cwd)`}, b); c > 0 {
|
||||
log.Println(b.String())
|
||||
return "", fmt.Errorf("Error %d trying to get CWD", c)
|
||||
} else if err != nil {
|
||||
@@ -122,7 +123,7 @@ func (p *pwd) InstanceUploadFromReader(instance *types.Instance, fileName, dest
|
||||
}
|
||||
}
|
||||
|
||||
copyErr := p.docker.CopyToContainer(instance.Name, finalDest, fileName, reader)
|
||||
copyErr := p.docker(instance.SessionId).CopyToContainer(instance.Name, finalDest, fileName, reader)
|
||||
|
||||
if copyErr != nil {
|
||||
return fmt.Errorf("Error while uploading file [%s]. Error: %s\n", fileName, copyErr)
|
||||
@@ -171,7 +172,7 @@ func (p *pwd) InstanceDelete(session *types.Session, instance *types.Instance) e
|
||||
if conn != nil {
|
||||
conn.Close()
|
||||
}
|
||||
err := p.docker.DeleteContainer(instance.Name)
|
||||
err := p.docker(session.Id).DeleteContainer(instance.Name)
|
||||
if err != nil && !strings.Contains(err.Error(), "No such container") {
|
||||
log.Println(err)
|
||||
return err
|
||||
@@ -243,7 +244,7 @@ func (p *pwd) InstanceNew(session *types.Session, conf InstanceConfig) (*types.I
|
||||
}
|
||||
}
|
||||
|
||||
ip, err := p.docker.CreateContainer(opts)
|
||||
ip, err := p.docker(session.Id).CreateContainer(opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -261,6 +262,7 @@ func (p *pwd) InstanceNew(session *types.Session, conf InstanceConfig) (*types.I
|
||||
instance.ServerKey = conf.ServerKey
|
||||
instance.CACert = conf.CACert
|
||||
instance.Session = session
|
||||
instance.Proxy = router.EncodeHost(session.Id, ip, router.HostOpts{})
|
||||
// For now this condition holds through. In the future we might need a more complex logic.
|
||||
instance.IsDockerHost = opts.Privileged
|
||||
|
||||
@@ -304,7 +306,7 @@ func (p *pwd) InstanceAllowedImages() []string {
|
||||
|
||||
func (p *pwd) InstanceExec(instance *types.Instance, cmd []string) (int, error) {
|
||||
defer observeAction("InstanceExec", time.Now())
|
||||
return p.docker.Exec(instance.Name, cmd)
|
||||
return p.docker(instance.SessionId).Exec(instance.Name, cmd)
|
||||
}
|
||||
|
||||
func getInstanceTermConn(sessionId, instanceName string) net.Conn {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/play-with-docker/play-with-docker/docker"
|
||||
"github.com/play-with-docker/play-with-docker/event"
|
||||
"github.com/play-with-docker/play-with-docker/pwd/types"
|
||||
"github.com/play-with-docker/play-with-docker/router"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
@@ -28,12 +29,13 @@ func TestInstanceResizeTerminal(t *testing.T) {
|
||||
|
||||
return nil
|
||||
}
|
||||
sp := &mockSessionProvider{docker: docker}
|
||||
|
||||
tasks := &mockTasks{}
|
||||
e := event.NewLocalBroker()
|
||||
storage := &mockStorage{}
|
||||
|
||||
p := NewPWD(docker, tasks, e, storage)
|
||||
p := NewPWD(sp, tasks, e, storage)
|
||||
|
||||
err := p.InstanceResizeTerminal(&types.Instance{Name: "foobar"}, 24, 80)
|
||||
|
||||
@@ -50,12 +52,13 @@ func TestInstanceNew(t *testing.T) {
|
||||
containerOpts = opts
|
||||
return "10.0.0.1", nil
|
||||
}
|
||||
sp := &mockSessionProvider{docker: dock}
|
||||
|
||||
tasks := &mockTasks{}
|
||||
e := event.NewLocalBroker()
|
||||
storage := &mockStorage{}
|
||||
|
||||
p := NewPWD(dock, tasks, e, storage)
|
||||
p := NewPWD(sp, tasks, e, storage)
|
||||
|
||||
session, err := p.SessionNew(time.Hour, "", "", "")
|
||||
|
||||
@@ -74,6 +77,7 @@ func TestInstanceNew(t *testing.T) {
|
||||
IsDockerHost: true,
|
||||
SessionId: session.Id,
|
||||
Session: session,
|
||||
Proxy: router.EncodeHost(session.Id, "10.0.0.1", router.HostOpts{}),
|
||||
}
|
||||
|
||||
assert.Equal(t, expectedInstance, *instance)
|
||||
@@ -101,12 +105,13 @@ func TestInstanceNew_Concurrency(t *testing.T) {
|
||||
i++
|
||||
return fmt.Sprintf("10.0.0.%d", i), nil
|
||||
}
|
||||
sp := &mockSessionProvider{docker: dock}
|
||||
|
||||
tasks := &mockTasks{}
|
||||
e := event.NewLocalBroker()
|
||||
storage := &mockStorage{}
|
||||
|
||||
p := NewPWD(dock, tasks, e, storage)
|
||||
p := NewPWD(sp, tasks, e, storage)
|
||||
|
||||
session, err := p.SessionNew(time.Hour, "", "", "")
|
||||
|
||||
@@ -142,12 +147,13 @@ func TestInstanceNew_WithNotAllowedImage(t *testing.T) {
|
||||
containerOpts = opts
|
||||
return "10.0.0.1", nil
|
||||
}
|
||||
sp := &mockSessionProvider{docker: dock}
|
||||
|
||||
tasks := &mockTasks{}
|
||||
e := event.NewLocalBroker()
|
||||
storage := &mockStorage{}
|
||||
|
||||
p := NewPWD(dock, tasks, e, storage)
|
||||
p := NewPWD(sp, tasks, e, storage)
|
||||
|
||||
session, err := p.SessionNew(time.Hour, "", "", "")
|
||||
|
||||
@@ -166,6 +172,7 @@ func TestInstanceNew_WithNotAllowedImage(t *testing.T) {
|
||||
SessionId: session.Id,
|
||||
IsDockerHost: false,
|
||||
Session: session,
|
||||
Proxy: instance.Proxy,
|
||||
}
|
||||
|
||||
assert.Equal(t, expectedInstance, *instance)
|
||||
@@ -191,12 +198,13 @@ func TestInstanceNew_WithCustomHostname(t *testing.T) {
|
||||
containerOpts = opts
|
||||
return "10.0.0.1", nil
|
||||
}
|
||||
sp := &mockSessionProvider{docker: dock}
|
||||
|
||||
tasks := &mockTasks{}
|
||||
e := event.NewLocalBroker()
|
||||
storage := &mockStorage{}
|
||||
|
||||
p := NewPWD(dock, tasks, e, storage)
|
||||
p := NewPWD(sp, tasks, e, storage)
|
||||
|
||||
session, err := p.SessionNew(time.Hour, "", "", "")
|
||||
|
||||
@@ -215,6 +223,7 @@ func TestInstanceNew_WithCustomHostname(t *testing.T) {
|
||||
IsDockerHost: false,
|
||||
Session: session,
|
||||
SessionId: session.Id,
|
||||
Proxy: instance.Proxy,
|
||||
}
|
||||
|
||||
assert.Equal(t, expectedInstance, *instance)
|
||||
@@ -238,8 +247,9 @@ func TestInstanceAllowedImages(t *testing.T) {
|
||||
tasks := &mockTasks{}
|
||||
e := event.NewLocalBroker()
|
||||
storage := &mockStorage{}
|
||||
sp := &mockSessionProvider{docker: dock}
|
||||
|
||||
p := NewPWD(dock, tasks, e, storage)
|
||||
p := NewPWD(sp, tasks, e, storage)
|
||||
|
||||
expectedImages := []string{config.GetDindImageName(), "franela/dind:overlay2-dev", "franela/ucp:2.4.1"}
|
||||
|
||||
@@ -264,8 +274,9 @@ func TestTermConnAssignment(t *testing.T) {
|
||||
// return error connection to unlock the goroutine
|
||||
return errConn{}, nil
|
||||
}
|
||||
sp := &mockSessionProvider{docker: dock}
|
||||
|
||||
p := NewPWD(dock, tasks, e, storage)
|
||||
p := NewPWD(sp, tasks, e, storage)
|
||||
session, _ := p.SessionNew(time.Hour, "", "", "")
|
||||
mockInstance := &types.Instance{
|
||||
Name: fmt.Sprintf("%s_redis-master", session.Id[:8]),
|
||||
|
||||
23
pwd/pwd.go
23
pwd/pwd.go
@@ -6,6 +6,7 @@ import (
|
||||
|
||||
"github.com/play-with-docker/play-with-docker/docker"
|
||||
"github.com/play-with-docker/play-with-docker/event"
|
||||
"github.com/play-with-docker/play-with-docker/provider"
|
||||
"github.com/play-with-docker/play-with-docker/pwd/types"
|
||||
"github.com/play-with-docker/play-with-docker/storage"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
@@ -44,11 +45,11 @@ func init() {
|
||||
}
|
||||
|
||||
type pwd struct {
|
||||
docker docker.DockerApi
|
||||
tasks SchedulerApi
|
||||
event event.EventApi
|
||||
storage storage.StorageApi
|
||||
clientCount int32
|
||||
sessionProvider provider.SessionProvider
|
||||
tasks SchedulerApi
|
||||
event event.EventApi
|
||||
storage storage.StorageApi
|
||||
clientCount int32
|
||||
}
|
||||
|
||||
type PWDApi interface {
|
||||
@@ -80,8 +81,16 @@ type PWDApi interface {
|
||||
ClientCount() int
|
||||
}
|
||||
|
||||
func NewPWD(d docker.DockerApi, t SchedulerApi, e event.EventApi, s storage.StorageApi) *pwd {
|
||||
return &pwd{docker: d, tasks: t, event: e, storage: s}
|
||||
func NewPWD(sp provider.SessionProvider, t SchedulerApi, e event.EventApi, s storage.StorageApi) *pwd {
|
||||
return &pwd{sessionProvider: sp, tasks: t, event: e, storage: s}
|
||||
}
|
||||
|
||||
func (p *pwd) docker(sessionId string) docker.DockerApi {
|
||||
d, err := p.sessionProvider.GetDocker(sessionId)
|
||||
if err != nil {
|
||||
panic("Should not have got here. Session always need to be validated before calling this.")
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func (p *pwd) setGauges() {
|
||||
|
||||
@@ -62,7 +62,7 @@ func (p *pwd) SessionNew(duration time.Duration, stack, stackName, imageName str
|
||||
|
||||
log.Printf("NewSession id=[%s]\n", s.Id)
|
||||
|
||||
if err := p.docker.CreateNetwork(s.Id); err != nil {
|
||||
if err := p.docker(s.Id).CreateNetwork(s.Id); err != nil {
|
||||
log.Println("ERROR NETWORKING")
|
||||
return nil, err
|
||||
}
|
||||
@@ -101,14 +101,14 @@ func (p *pwd) SessionClose(s *types.Session) error {
|
||||
}
|
||||
}
|
||||
// Disconnect PWD daemon from the network
|
||||
if err := p.docker.DisconnectNetwork(config.PWDContainerName, s.Id); err != nil {
|
||||
if err := p.docker(s.Id).DisconnectNetwork(config.PWDContainerName, s.Id); err != nil {
|
||||
if !strings.Contains(err.Error(), "is not connected to the network") {
|
||||
log.Println("ERROR NETWORKING")
|
||||
return err
|
||||
}
|
||||
}
|
||||
log.Printf("Disconnected pwd from network [%s]\n", s.Id)
|
||||
if err := p.docker.DeleteNetwork(s.Id); err != nil {
|
||||
if err := p.docker(s.Id).DeleteNetwork(s.Id); err != nil {
|
||||
if !strings.Contains(err.Error(), "not found") {
|
||||
log.Println(err)
|
||||
return err
|
||||
@@ -168,7 +168,7 @@ func (p *pwd) SessionDeployStack(s *types.Session) error {
|
||||
cmd := fmt.Sprintf("docker swarm init --advertise-addr eth0 && docker-compose -f %s pull && docker stack deploy -c %s %s", file, file, s.StackName)
|
||||
|
||||
w := sessionBuilderWriter{sessionId: s.Id, event: p.event}
|
||||
code, err := p.docker.ExecAttach(i.Name, []string{"sh", "-c", cmd}, &w)
|
||||
code, err := p.docker(s.Id).ExecAttach(i.Name, []string{"sh", "-c", cmd}, &w)
|
||||
if err != nil {
|
||||
log.Printf("Error executing stack [%s]: %s\n", s.Stack, err)
|
||||
return err
|
||||
@@ -218,7 +218,7 @@ func (p *pwd) SessionSetup(session *types.Session, conf SessionSetupConf) error
|
||||
return err
|
||||
}
|
||||
if i.Docker == nil {
|
||||
dock, err := p.docker.New(i.IP, i.Cert, i.Key)
|
||||
dock, err := p.docker(session.Id).New(i.IP, i.Cert, i.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -254,7 +254,7 @@ func (p *pwd) SessionSetup(session *types.Session, conf SessionSetupConf) error
|
||||
if c.IsSwarmManager || c.IsSwarmWorker {
|
||||
// check if we have connection to the daemon, if not, create it
|
||||
if i.Docker == nil {
|
||||
dock, err := p.docker.New(i.IP, i.Cert, i.Key)
|
||||
dock, err := p.docker(session.Id).New(i.IP, i.Cert, i.Key)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return
|
||||
@@ -334,7 +334,7 @@ func (p *pwd) scheduleSessionClose(s *types.Session) {
|
||||
}
|
||||
|
||||
func (p *pwd) connectToNetwork(s *types.Session) error {
|
||||
ip, err := p.docker.ConnectNetwork(config.PWDContainerName, s.Id, s.PwdIpAddress)
|
||||
ip, err := p.docker(s.Id).ConnectNetwork(config.PWDContainerName, s.Id, s.PwdIpAddress)
|
||||
if err != nil {
|
||||
log.Println("ERROR NETWORKING")
|
||||
return err
|
||||
|
||||
11
pwd/session_provider_mock_test.go
Normal file
11
pwd/session_provider_mock_test.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package pwd
|
||||
|
||||
import "github.com/play-with-docker/play-with-docker/docker"
|
||||
|
||||
type mockSessionProvider struct {
|
||||
docker docker.DockerApi
|
||||
}
|
||||
|
||||
func (p *mockSessionProvider) GetDocker(sessionId string) (docker.DockerApi, error) {
|
||||
return p.docker, nil
|
||||
}
|
||||
@@ -30,6 +30,7 @@ func TestSessionNew(t *testing.T) {
|
||||
connectIP = ip
|
||||
return "10.0.0.1", nil
|
||||
}
|
||||
sp := &mockSessionProvider{docker: docker}
|
||||
|
||||
var scheduledSession *types.Session
|
||||
tasks := &mockTasks{}
|
||||
@@ -44,7 +45,7 @@ func TestSessionNew(t *testing.T) {
|
||||
return nil
|
||||
}
|
||||
|
||||
p := NewPWD(docker, tasks, ev, storage)
|
||||
p := NewPWD(sp, tasks, ev, storage)
|
||||
|
||||
before := time.Now()
|
||||
|
||||
@@ -166,11 +167,12 @@ func TestSessionSetup(t *testing.T) {
|
||||
assert.Fail(t, "Shouldn't have reached here.")
|
||||
return nil, nil
|
||||
}
|
||||
sp := &mockSessionProvider{docker: dock}
|
||||
tasks := &mockTasks{}
|
||||
ev := event.NewLocalBroker()
|
||||
storage := &mockStorage{}
|
||||
|
||||
p := NewPWD(dock, tasks, ev, storage)
|
||||
p := NewPWD(sp, tasks, ev, storage)
|
||||
s, e := p.SessionNew(time.Hour, "", "", "")
|
||||
assert.Nil(t, e)
|
||||
|
||||
@@ -215,6 +217,7 @@ func TestSessionSetup(t *testing.T) {
|
||||
IsDockerHost: true,
|
||||
Session: s,
|
||||
Docker: manager1Received.Docker,
|
||||
Proxy: manager1Received.Proxy,
|
||||
}, manager1Received)
|
||||
|
||||
manager2 := fmt.Sprintf("%s_manager2", s.Id[:8])
|
||||
@@ -229,6 +232,7 @@ func TestSessionSetup(t *testing.T) {
|
||||
SessionId: s.Id,
|
||||
Session: s,
|
||||
Docker: manager2Received.Docker,
|
||||
Proxy: manager2Received.Proxy,
|
||||
}, manager2Received)
|
||||
|
||||
manager3 := fmt.Sprintf("%s_manager3", s.Id[:8])
|
||||
@@ -243,6 +247,7 @@ func TestSessionSetup(t *testing.T) {
|
||||
IsDockerHost: true,
|
||||
Session: s,
|
||||
Docker: manager3Received.Docker,
|
||||
Proxy: manager3Received.Proxy,
|
||||
}, manager3Received)
|
||||
|
||||
worker1 := fmt.Sprintf("%s_worker1", s.Id[:8])
|
||||
@@ -257,6 +262,7 @@ func TestSessionSetup(t *testing.T) {
|
||||
IsDockerHost: true,
|
||||
Session: s,
|
||||
Docker: worker1Received.Docker,
|
||||
Proxy: worker1Received.Proxy,
|
||||
}, worker1Received)
|
||||
|
||||
other := fmt.Sprintf("%s_other", s.Id[:8])
|
||||
@@ -271,6 +277,7 @@ func TestSessionSetup(t *testing.T) {
|
||||
IsDockerHost: true,
|
||||
Session: s,
|
||||
Docker: otherReceived.Docker,
|
||||
Proxy: otherReceived.Proxy,
|
||||
}, otherReceived)
|
||||
|
||||
assert.True(t, swarmInitOnMaster1)
|
||||
@@ -284,8 +291,9 @@ func TestSessionPrepareOnce(t *testing.T) {
|
||||
tasks := &mockTasks{}
|
||||
ev := event.NewLocalBroker()
|
||||
storage := &mockStorage{}
|
||||
sp := &mockSessionProvider{docker: dock}
|
||||
|
||||
p := NewPWD(dock, tasks, ev, storage)
|
||||
p := NewPWD(sp, tasks, ev, storage)
|
||||
session := &types.Session{Id: "1234"}
|
||||
prepared, err := p.prepareSession(session)
|
||||
assert.True(t, preparedSessions[session.Id])
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/docker/go-connections/tlsconfig"
|
||||
"github.com/play-with-docker/play-with-docker/docker"
|
||||
"github.com/play-with-docker/play-with-docker/event"
|
||||
"github.com/play-with-docker/play-with-docker/provider"
|
||||
"github.com/play-with-docker/play-with-docker/pwd/types"
|
||||
)
|
||||
|
||||
@@ -112,8 +113,8 @@ func (sch *scheduler) Schedule(s *types.Session) {
|
||||
func (sch *scheduler) Unschedule(s *types.Session) {
|
||||
}
|
||||
|
||||
func NewScheduler(e event.EventApi, d docker.DockerApi) *scheduler {
|
||||
func NewScheduler(e event.EventApi, sp provider.SessionProvider) *scheduler {
|
||||
s := &scheduler{event: e}
|
||||
s.periodicTasks = []periodicTask{&collectStatsTask{docker: d}, &checkSwarmStatusTask{}, &checkUsedPortsTask{}, &checkSwarmUsedPortsTask{}}
|
||||
s.periodicTasks = []periodicTask{&collectStatsTask{sessionProvider: sp}, &checkSwarmStatusTask{}, &checkUsedPortsTask{}, &checkSwarmUsedPortsTask{}}
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ type Instance struct {
|
||||
IsDockerHost bool `json:"is_docker_host" bson:"is_docker_host"`
|
||||
SessionId string `json:"session_id" bson:"session_id"`
|
||||
SessionPrefix string `json:"session_prefix" bson:"session_prefix"`
|
||||
Proxy string `json:"proxy" bson:"proxy"`
|
||||
Docker docker.DockerApi `json:"-"`
|
||||
Session *Session `json:"-" bson:"-"`
|
||||
ctx context.Context `json:"-" bson:"-"`
|
||||
|
||||
71
router/host.go
Normal file
71
router/host.go
Normal file
@@ -0,0 +1,71 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const hostPattern = "^.*ip([0-9]{1,3}-[0-9]{1,3}-[0-9]{1,3}-[0-9]{1,3})-([0-9|a-z]+)(?:-?([0-9]{1,5}))?(?:\\.([a-z|A-Z|0-9|_|\\-\\.]+))?(?:\\:([0-9]{1,5}))?$"
|
||||
|
||||
var hostRegex *regexp.Regexp
|
||||
|
||||
func init() {
|
||||
hostRegex = regexp.MustCompile(hostPattern)
|
||||
}
|
||||
|
||||
type HostOpts struct {
|
||||
TLD string
|
||||
EncodedPort int
|
||||
Port int
|
||||
}
|
||||
|
||||
type HostInfo struct {
|
||||
SessionId string
|
||||
InstanceIP string
|
||||
TLD string
|
||||
EncodedPort int
|
||||
Port int
|
||||
}
|
||||
|
||||
func EncodeHost(sessionId, instanceIP string, opts HostOpts) string {
|
||||
encodedIP := strings.Replace(instanceIP, ".", "-", -1)
|
||||
|
||||
sub := fmt.Sprintf("ip%s-%s", encodedIP, sessionId)
|
||||
if opts.EncodedPort > 0 {
|
||||
sub = fmt.Sprintf("%s-%d", sub, opts.EncodedPort)
|
||||
}
|
||||
if opts.TLD != "" {
|
||||
sub = fmt.Sprintf("%s.%s", sub, opts.TLD)
|
||||
}
|
||||
if opts.Port > 0 {
|
||||
sub = fmt.Sprintf("%s:%d", sub, opts.Port)
|
||||
}
|
||||
|
||||
return sub
|
||||
}
|
||||
|
||||
func DecodeHost(host string) (HostInfo, error) {
|
||||
info := HostInfo{}
|
||||
|
||||
matches := hostRegex.FindStringSubmatch(host)
|
||||
if len(matches) != 6 {
|
||||
return HostInfo{}, fmt.Errorf("Couldn't find host in string")
|
||||
}
|
||||
|
||||
info.InstanceIP = strings.Replace(matches[1], "-", ".", -1)
|
||||
info.SessionId = matches[2]
|
||||
info.TLD = matches[4]
|
||||
|
||||
if matches[3] != "" {
|
||||
i, _ := strconv.Atoi(matches[3])
|
||||
info.EncodedPort = i
|
||||
}
|
||||
if matches[5] != "" {
|
||||
i, _ := strconv.Atoi(matches[5])
|
||||
info.Port = i
|
||||
}
|
||||
|
||||
return info, nil
|
||||
}
|
||||
45
router/host_test.go
Normal file
45
router/host_test.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestEncodeHostInfo(t *testing.T) {
|
||||
host := EncodeHost("aaabbbcccddd", "10.0.0.1", HostOpts{})
|
||||
assert.Equal(t, "ip10-0-0-1-aaabbbcccddd", host)
|
||||
|
||||
opts := HostOpts{EncodedPort: 8080}
|
||||
host = EncodeHost("aaabbbcccddd", "10.0.0.1", opts)
|
||||
assert.Equal(t, "ip10-0-0-1-aaabbbcccddd-8080", host)
|
||||
|
||||
opts = HostOpts{TLD: "foo.bar"}
|
||||
host = EncodeHost("aaabbbcccddd", "10.0.0.1", opts)
|
||||
assert.Equal(t, "ip10-0-0-1-aaabbbcccddd.foo.bar", host)
|
||||
|
||||
opts = HostOpts{TLD: "foo.bar", EncodedPort: 8080, Port: 443}
|
||||
host = EncodeHost("aaabbbcccddd", "10.0.0.1", opts)
|
||||
assert.Equal(t, "ip10-0-0-1-aaabbbcccddd-8080.foo.bar:443", host)
|
||||
}
|
||||
|
||||
func TestDecodeHostInfo(t *testing.T) {
|
||||
info, err := DecodeHost("ip10-0-0-1-aaabbbcccddd")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, HostInfo{InstanceIP: "10.0.0.1", SessionId: "aaabbbcccddd"}, info)
|
||||
|
||||
info, err = DecodeHost("ip10-0-0-1-aaabbbcccddd-8080")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, HostInfo{InstanceIP: "10.0.0.1", SessionId: "aaabbbcccddd", EncodedPort: 8080}, info)
|
||||
|
||||
info, err = DecodeHost("ip10-0-0-1-aaabbbcccddd-8080.foo.bar")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, HostInfo{InstanceIP: "10.0.0.1", SessionId: "aaabbbcccddd", EncodedPort: 8080, TLD: "foo.bar"}, info)
|
||||
|
||||
info, err = DecodeHost("ip10-0-0-1-aaabbbcccddd-8080.foo.bar:443")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, HostInfo{InstanceIP: "10.0.0.1", SessionId: "aaabbbcccddd", EncodedPort: 8080, TLD: "foo.bar", Port: 443}, info)
|
||||
|
||||
_, err = DecodeHost("ip10-0-0-1")
|
||||
assert.NotNil(t, err)
|
||||
}
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/filters"
|
||||
@@ -18,31 +17,23 @@ import (
|
||||
)
|
||||
|
||||
func director(host string) (*net.TCPAddr, error) {
|
||||
chunks := strings.Split(host, ":")
|
||||
matches := config.NameFilter.FindStringSubmatch(chunks[0])
|
||||
|
||||
var rawHost, port string
|
||||
|
||||
if len(matches) == 3 {
|
||||
rawHost = matches[1]
|
||||
port = matches[2]
|
||||
} else if len(matches) == 2 {
|
||||
rawHost = matches[1]
|
||||
} else {
|
||||
return nil, fmt.Errorf("Couldn't find host in string")
|
||||
info, err := router.DecodeHost(host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if port == "" {
|
||||
if len(chunks) == 2 {
|
||||
port = chunks[1]
|
||||
} else {
|
||||
port = "80"
|
||||
}
|
||||
port := info.Port
|
||||
|
||||
if info.EncodedPort > 0 {
|
||||
port = info.EncodedPort
|
||||
}
|
||||
|
||||
dstHost := strings.Replace(rawHost, "-", ".", -1)
|
||||
if port == 0 {
|
||||
// TODO: Should default depending on the protocol
|
||||
port = 80
|
||||
}
|
||||
|
||||
t, err := net.ResolveTCPAddr("tcp4", fmt.Sprintf("%s:%s", dstHost, port))
|
||||
t, err := net.ResolveTCPAddr("tcp4", fmt.Sprintf("%s:%d", info.InstanceIP, port))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -7,41 +7,38 @@ import (
|
||||
)
|
||||
|
||||
func TestDirector(t *testing.T) {
|
||||
addr, err := director("ip10-0-0-1-8080.foo.bar")
|
||||
addr, err := director("ip10-0-0-1-aabb-8080.foo.bar")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "10.0.0.1:8080", addr.String())
|
||||
|
||||
addr, err = director("ip10-0-0-1.foo.bar")
|
||||
addr, err = director("ip10-0-0-1-aabb.foo.bar")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "10.0.0.1:80", addr.String())
|
||||
|
||||
addr, err = director("ip10-0-0-1.foo.bar:9090")
|
||||
addr, err = director("ip10-0-0-1-aabb.foo.bar:9090")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "10.0.0.1:9090", addr.String())
|
||||
|
||||
addr, err = director("ip10-0-0-1-2222.foo.bar:9090")
|
||||
addr, err = director("ip10-0-0-1-aabb-2222.foo.bar:9090")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "10.0.0.1:2222", addr.String())
|
||||
|
||||
addr, err = director("lala.ip10-0-0-1-2222.foo.bar")
|
||||
addr, err = director("lala.ip10-0-0-1-aabb-2222.foo.bar")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "10.0.0.1:2222", addr.String())
|
||||
|
||||
addr, err = director("lala.ip10-0-0-1-2222")
|
||||
addr, err = director("lala.ip10-0-0-1-aabb-2222")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "10.0.0.1:2222", addr.String())
|
||||
|
||||
addr, err = director("ip10-0-0-1-2222")
|
||||
addr, err = director("ip10-0-0-1-aabb-2222")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "10.0.0.1:2222", addr.String())
|
||||
|
||||
addr, err = director("ip10-0-0-1")
|
||||
addr, err = director("ip10-0-0-1-aabb")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "10.0.0.1:80", addr.String())
|
||||
|
||||
_, err = director("lala10-0-0-1.foo.bar")
|
||||
assert.NotNil(t, err)
|
||||
|
||||
_, err = director("ip10-0-0-1-10-20")
|
||||
_, err = director("lala10-0-0-1-aabb.foo.bar")
|
||||
assert.NotNil(t, err)
|
||||
}
|
||||
|
||||
@@ -241,7 +241,7 @@
|
||||
}
|
||||
|
||||
$scope.getProxyUrl = function(instance, port) {
|
||||
var url = window.location.protocol + '//pwd' + instance.ip.replace(/\./g, '-') + '-' + port + '.' + window.location.host;
|
||||
var url = window.location.protocol + '//' + instance.proxy + '-' + port + '.' + window.location.host;
|
||||
|
||||
return url;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user