rancherOS/dfs/scratch.go

package dfs

import (
	"bufio"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"syscall"

	"github.com/docker/libnetwork/resolvconf"
	"github.com/rancher/os/log"
	"github.com/rancher/os/netconf"
	"github.com/rancher/os/selinux"
	"github.com/rancher/os/util"
)

const (
	defaultPrefix = "/usr"
	iptables      = "/sbin/iptables"
	modprobe      = "/sbin/modprobe"
	distSuffix    = ".dist"
)

var (
	mounts = [][]string{
		{"devtmpfs", "/dev", "devtmpfs", ""},
		{"none", "/dev/pts", "devpts", ""},
		{"shm", "/dev/shm", "tmpfs", "rw,nosuid,nodev,noexec,relatime,size=65536k"},
		{"mqueue", "/dev/mqueue", "mqueue", "rw,nosuid,nodev,noexec,relatime"},
		{"none", "/proc", "proc", ""},
		{"none", "/run", "tmpfs", ""},
		{"none", "/sys", "sysfs", ""},
		{"none", "/sys/fs/cgroup", "tmpfs", ""},
	}
	optionalMounts = [][]string{
		{"none", "/sys/fs/selinux", "selinuxfs", "ro"},
	}
)

type Config struct {
	Fork            bool
	PidOne          bool
	CommandName     string
	DNSConfig       netconf.DNSConfig
	BridgeName      string
	BridgeAddress   string
	BridgeMtu       int
	CgroupHierarchy map[string]string
	LogFile         string
	NoLog           bool
	NoFiles         uint64
	Environment     []string
	GraphDirectory  string
	DaemonConfig    string
}

func createMounts(mounts ...[]string) error {
	for _, mount := range mounts {
		log.Debugf("Mounting %s %s %s %s", mount[0], mount[1], mount[2], mount[3])
		err := util.Mount(mount[0], mount[1], mount[2], mount[3])
		if err != nil {
			return err
		}
	}

	return nil
}

func createOptionalMounts(mounts ...[]string) {
	for _, mount := range mounts {
		log.Debugf("Mounting %s %s %s %s", mount[0], mount[1], mount[2], mount[3])
		err := util.Mount(mount[0], mount[1], mount[2], mount[3])
		if err != nil {
			log.Debugf("Unable to mount %s %s %s %s: %s", mount[0], mount[1], mount[2], mount[3], err)
		}
	}
}

func createDirs(dirs ...string) error {
	for _, dir := range dirs {
		if _, err := os.Stat(dir); os.IsNotExist(err) {
			log.Debugf("Creating %s", dir)
			err = os.MkdirAll(dir, 0755)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func mountCgroups(hierarchyConfig map[string]string) error {
	f, err := os.Open("/proc/cgroups")
	if err != nil {
		return err
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)

	hierarchies := make(map[string][]string)

	for scanner.Scan() {
		text := scanner.Text()
		log.Debugf("/proc/cgroups: %s", text)
		fields := strings.Split(text, "\t")
		cgroup := fields[0]
		if cgroup == "" || cgroup[0] == '#' || (len(fields) > 3 && fields[3] == "0") {
			continue
		}

		hierarchy := hierarchyConfig[cgroup]
		if hierarchy == "" {
			hierarchy = fields[1]
		}

		if hierarchy == "0" {
			hierarchy = cgroup
		}

		hierarchies[hierarchy] = append(hierarchies[hierarchy], cgroup)
	}

	for _, hierarchy := range hierarchies {
		if err := mountCgroup(strings.Join(hierarchy, ",")); err != nil {
			return err
		}
	}

	if err = scanner.Err(); err != nil {
		return err
	}

	log.Debug("Done mouting cgroupfs")
	return nil
}

func CreateSymlinks(pathSets [][]string) error {
	for _, paths := range pathSets {
		if err := CreateSymlink(paths[0], paths[1]); err != nil {
			return err
		}
	}

	return nil
}

func CreateSymlink(src, dest string) error {
	if _, err := os.Lstat(dest); os.IsNotExist(err) {
		log.Debugf("Symlinking %s => %s", dest, src)
		if err = os.Symlink(src, dest); err != nil {
			return err
		}
	}

	return nil
}

func mountCgroup(cgroup string) error {
	if err := createDirs("/sys/fs/cgroup/" + cgroup); err != nil {
		return err
	}

	if err := createMounts([][]string{{"none", "/sys/fs/cgroup/" + cgroup, "cgroup", cgroup}}...); err != nil {
		return err
	}

	parts := strings.Split(cgroup, ",")
	if len(parts) > 1 {
		for _, part := range parts {
			if err := CreateSymlink("/sys/fs/cgroup/"+cgroup, "/sys/fs/cgroup/"+part); err != nil {
				return err
			}
		}
	}

	return nil
}

func execDocker(config *Config, docker, cmd string, args []string) (*exec.Cmd, error) {
	if len(args) > 0 && args[0] == "docker" {
		args = args[1:]
	}
	log.Debugf("Launching Docker %s %s %v", docker, cmd, args)

	env := os.Environ()
	if len(config.Environment) != 0 {
		env = append(env, config.Environment...)
	}

	if config.Fork {
		cmd := exec.Command(docker, args...)
		if !config.NoLog {
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
		}
		cmd.Env = env
		err := cmd.Start()
		if err != nil {
			return cmd, err
		}
		if config.PidOne {
			PidOne()
		}
		return cmd, err
	}

	return nil, syscall.Exec(expand(docker), append([]string{cmd}, args...), env)
}

func copyDefault(folder, name string) error {
	defaultFile := path.Join(defaultPrefix, folder, name)
	return CopyFile(defaultFile, folder, name)
}

func copyDefaultFolder(folder string) error {
	log.Debugf("Copying folder %s", folder)
	defaultFolder := path.Join(defaultPrefix, folder)
	files, _ := ioutil.ReadDir(defaultFolder)
	for _, file := range files {
		var err error
		if file.IsDir() {
			err = copyDefaultFolder(path.Join(folder, file.Name()))
		} else {
			err = copyDefault(folder, file.Name())
		}
		if err != nil {
			return err
		}
	}

	return nil
}

func defaultFiles(files ...string) error {
	for _, file := range files {
		dir := path.Dir(file)
		name := path.Base(file)
		if err := copyDefault(dir, name); err != nil {
			return err
		}
	}

	return nil
}

func defaultFolders(folders ...string) error {
	for _, folder := range folders {
		if err := copyDefaultFolder(folder); err != nil {
			return err
		}
	}

	return nil
}

func CopyFile(src, folder, name string) error {
	return CopyFileOverwrite(src, folder, name, false)
}

func CopyFileOverwrite(src, folder, name string, overwrite bool) error {
	if _, err := os.Lstat(src); os.IsNotExist(err) {
		log.Debugf("Not copying %s, does not exists", src)
		return nil
	}

	dst := path.Join(folder, name)
	if !overwrite {
		if _, err := os.Lstat(dst); err == nil {
			log.Debugf("Not copying %s => %s already exists", src, dst)
			return nil
		}
	}

	if err := createDirs(folder); err != nil {
		return err
	}

	stat, err := os.Lstat(src)
	if err != nil {
		return err
	}

	if stat.Mode()&os.ModeSymlink != 0 {
		symDst, err := os.Readlink(src)
		if err != nil {
			log.Errorf("Failed to readlink: %v", err)
			return err
		}
		// file is a symlink
		log.Debugf("Symlinking %s => %s", dst, symDst)
		return os.Symlink(symDst, dst)
	}

	srcFile, err := os.Open(src)
	if err != nil {
		return err
	}
	defer srcFile.Close()

	dstFile, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer dstFile.Close()

	log.Debugf("Copying %s => %s", src, dst)
	_, err = io.Copy(dstFile, srcFile)
	return err
}

func tryCreateFile(name, content string) error {
	if _, err := os.Stat(name); err == nil {
		return nil
	}

	if err := createDirs(path.Dir(name)); err != nil {
		return err
	}

	return ioutil.WriteFile(name, []byte(content), 0644)
}

func createPasswd() error {
	return tryCreateFile("/etc/passwd", "root:x:0:0:root:/root:/bin/sh\n")
}

func createGroup() error {
	return tryCreateFile("/etc/group", "root:x:0:\n")
}

func setupNetworking(cfg *Config) error {
	if cfg == nil {
		return nil
	}

	hostname, err := os.Hostname()
	if err != nil {
		return err
	}
	tryCreateFile("/etc/hosts", `127.0.0.1    localhost
::1    localhost ip6-localhost ip6-loopback
fe00::0    ip6-localnet
ff00::0    ip6-mcastprefix
ff02::1    ip6-allnodes
ff02::2    ip6-allrouters

127.0.1.1       `+hostname)

	if len(cfg.DNSConfig.Nameservers) != 0 {
		log.Infof("Writing resolv.conf (%v) %v", cfg.DNSConfig.Nameservers, cfg.DNSConfig.Search)
		if _, err := resolvconf.Build("/etc/resolv.conf", cfg.DNSConfig.Nameservers, cfg.DNSConfig.Search, nil); err != nil {
			return err
		}
	}

	if cfg.BridgeName != "" && cfg.BridgeName != "none" {
		log.Debugf("Creating bridge %s (%s)", cfg.BridgeName, cfg.BridgeAddress)
		if err := netconf.ApplyNetworkConfigs(&netconf.NetworkConfig{
			Interfaces: map[string]netconf.InterfaceConfig{
				cfg.BridgeName: {
					Address: cfg.BridgeAddress,
					MTU:     cfg.BridgeMtu,
					Bridge:  "true",
				},
			},
		}, false, false); err != nil {
			return err
		}
	}

	return nil
}

func GetValue(index int, args []string) string {
	val := args[index]
	parts := strings.SplitN(val, "=", 2)
	if len(parts) == 1 {
		if len(args) > index+1 {
			return args[index+1]
		}
		return ""
	}
	return parts[1]
}

func ParseConfig(config *Config, args ...string) []string {
	for i, arg := range args {
		if strings.HasPrefix(arg, "--bip") {
			config.BridgeAddress = GetValue(i, args)
		} else if strings.HasPrefix(arg, "--fixed-cidr") {
			config.BridgeAddress = GetValue(i, args)
		} else if strings.HasPrefix(arg, "-b") || strings.HasPrefix(arg, "--bridge") {
			config.BridgeName = GetValue(i, args)
		} else if strings.HasPrefix(arg, "--config-file") {
			config.DaemonConfig = GetValue(i, args)
		} else if strings.HasPrefix(arg, "--mtu") {
			mtu, err := strconv.Atoi(GetValue(i, args))
			if err != nil {
				config.BridgeMtu = mtu
			}
		} else if strings.HasPrefix(arg, "-g") || strings.HasPrefix(arg, "--graph") {
			config.GraphDirectory = GetValue(i, args)
		}
	}

	if config.BridgeName != "" && config.BridgeAddress != "" {
		newArgs := []string{}
		skip := false
		for _, arg := range args {
			if skip {
				skip = false
				continue
			}

			if arg == "--bip" {
				skip = true
				continue
			} else if strings.HasPrefix(arg, "--bip=") {
				continue
			}

			newArgs = append(newArgs, arg)
		}

		args = newArgs
	}

	return args
}

func PrepareFs(config *Config) error {
	if err := createMounts(mounts...); err != nil {
		return err
	}

	createOptionalMounts(optionalMounts...)

	if err := mountCgroups(config.CgroupHierarchy); err != nil {
		return err
	}

	if err := createLayout(config); err != nil {
		return err
	}

	return firstPrepare()
}

func touchSocket(path string) error {
	if err := syscall.Unlink(path); err != nil && !os.IsNotExist(err) {
		return err
	}
	return ioutil.WriteFile(path, []byte{}, 0700)
}

func touchSockets(args ...string) error {
	touched := false

	for i, arg := range args {
		if strings.HasPrefix(arg, "-H") {
			val := GetValue(i, args)
			if strings.HasPrefix(val, "unix://") {
				val = val[len("unix://"):]
				log.Debugf("Creating temp file at %s", val)
				if err := touchSocket(val); err != nil {
					return err
				}
				touched = true
			}
		}
	}

	if !touched {
		return touchSocket("/var/run/docker.sock")
	}

	return nil
}

func createDaemonConfig(config *Config) error {
	if config.DaemonConfig == "" {
		return nil
	}

	if _, err := os.Stat(config.DaemonConfig); os.IsNotExist(err) {
		if err := os.MkdirAll(path.Dir(config.DaemonConfig), 0755); err != nil {
			return err
		}

		return ioutil.WriteFile(config.DaemonConfig, []byte("{}"), 0600)
	}

	return nil
}

func cleanupFiles(graphDirectory string) {
	zeroFiles := []string{
		"/etc/docker/key.json",
		"/etc/docker/daemon.json",
		"/etc/docker/system-daemon.json",
		path.Join(graphDirectory, "image/overlay/repositories.json"),
	}

	for _, file := range zeroFiles {
		if stat, err := os.Stat(file); err == nil {
			if stat.Size() < 2 {
				log.Warnf("Deleting invalid json file: %s", file)
				os.Remove(file)
			}
		}
	}
}

func createLayout(config *Config) error {
	if err := createDirs("/tmp", "/root/.ssh", "/var", "/usr/lib"); err != nil {
		return err
	}

	graphDirectory := config.GraphDirectory

	if config.GraphDirectory == "" {
		graphDirectory = "/var/lib/docker"
	}

	if err := createDirs(graphDirectory); err != nil {
		return err
	}

	if err := createDaemonConfig(config); err != nil {
		return err
	}

	cleanupFiles(graphDirectory)

	selinux.SetFileContext(graphDirectory, "system_u:object_r:var_lib_t:s0")

	return CreateSymlinks([][]string{
		{"usr/lib", "/lib"},
		{"usr/sbin", "/sbin"},
		{"../run", "/var/run"},
	})
}

func firstPrepare() error {
	os.Setenv("PATH", "/sbin:/usr/sbin:/usr/bin")

	if err := defaultFiles(
		"/etc/ssl/certs/ca-certificates.crt",
		"/etc/passwd",
		"/etc/group",
	); err != nil {
		return err
	}

	if err := defaultFolders(
		"/etc/docker",
		"/etc/selinux",
		"/etc/selinux/ros",
		"/etc/selinux/ros/policy",
		"/etc/selinux/ros/contexts",
		"/var/lib/cni",
	); err != nil {
		return err
	}

	if err := createPasswd(); err != nil {
		return err
	}

	return createGroup()
}

func secondPrepare(config *Config, docker string, args ...string) error {

	if err := setupNetworking(config); err != nil {
		return err
	}

	if err := touchSockets(args...); err != nil {
		return err
	}

	if err := setupLogging(config); err != nil {
		return err
	}

	for _, i := range []string{docker, iptables, modprobe} {
		if err := setupBin(config, i); err != nil {
			return err
		}
	}

	if err := setUlimit(config); err != nil {
		return err
	}

	ioutil.WriteFile("/proc/sys/net/ipv4/ip_forward", []byte("1"), 0655)

	return nil
}

func expand(bin string) string {
	expanded, err := exec.LookPath(bin)
	if err == nil {
		return expanded
	}
	return bin
}

func setupBin(config *Config, bin string) error {
	expanded, err := exec.LookPath(bin)
	if err == nil {
		return nil
	}

	expanded, err = exec.LookPath(bin + distSuffix)
	if err != nil {
		// Purposely not returning error
		return nil
	}

	return CreateSymlink(expanded, expanded[:len(expanded)-len(distSuffix)])
}

func setupLogging(config *Config) error {
	if config.LogFile == "" {
		return nil
	}

	if err := createDirs(path.Dir(config.LogFile)); err != nil {
		return err
	}

	output, err := os.OpenFile(config.LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		return err
	}

	syscall.Dup3(int(output.Fd()), int(os.Stdout.Fd()), 0)
	syscall.Dup3(int(output.Fd()), int(os.Stderr.Fd()), 0)

	return nil
}

func setUlimit(cfg *Config) error {
	var rLimit syscall.Rlimit
	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
		return err
	}
	if cfg.NoFiles == 0 {
		rLimit.Max = 1000000
	} else {
		rLimit.Max = cfg.NoFiles
	}
	rLimit.Cur = rLimit.Max
	return syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
}

func runOrExec(config *Config, docker string, args ...string) (*exec.Cmd, error) {
	if err := secondPrepare(config, docker, args...); err != nil {
		return nil, err
	}

	cmd := path.Base(docker)
	if config != nil && config.CommandName != "" {
		cmd = config.CommandName
	}

	if cmd == "dockerd" && len(args) > 1 && args[0] == "daemon" {
		args = args[1:]
	}

	return execDocker(config, docker, cmd, args)
}

func LaunchDocker(config *Config, docker string, args ...string) (*exec.Cmd, error) {
	if err := PrepareFs(config); err != nil {
		return nil, err
	}

	return runOrExec(config, docker, args...)
}

func Main() {
	log.InitLogger()
	if os.Getenv("DOCKER_LAUNCH_DEBUG") == "true" {
		log.SetLevel(log.DebugLevel)
	}

	if len(os.Args) < 2 {
		log.Fatalf("Usage Example: %s /usr/bin/docker -d -D", os.Args[0])
	}

	args := []string{}
	if len(os.Args) > 1 {
		args = os.Args[2:]
	}

	var config Config
	args = ParseConfig(&config, args...)

	if os.Getenv("DOCKER_LAUNCH_REAP") == "true" {
		config.Fork = true
		config.PidOne = true
	}

	log.Debugf("Launch config %#v", config)

	_, err := LaunchDocker(&config, os.Args[1], args...)
	if err != nil {
		log.Fatal(err)
	}
}