pagebot/main.go

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"

	"github.com/NYTimes/gziphandler"

	"github.com/gorilla/websocket"
	"golang.org/x/net/proxy"

	"github.com/davecgh/go-spew/spew"
	"github.com/go-yaml/yaml"
	"github.com/ogier/pflag"
	"github.com/superp00t/etc"
	"github.com/superp00t/etc/yo"
)

var (
	gitService = pflag.StringP("gitservice", "g", "https://git.ikrypto.club/", "the default Git service to map hosts to")
	subService = pflag.StringP("subservice", "s", "https://*.pg.ikrypto.club/", "the default web service that provides pages")
	addr       = pflag.StringP("address", "a", ":3000", "the default address to listen on")
	cfg        = pflag.StringP("cfg", "c", "config.yaml", "the default config file")
)

type coreMatch struct {
	Type    string    `yaml:"type"`
	Repo    string    `yaml:"repo"`
	ID      string    `yaml:"id"`
	SubPath string    `yaml:"subpath"`
	Updated time.Time `yaml:"updated"`
}

type ConfigFile struct {
	Sites map[string]*coreMatch `yaml:"sites"`
}

func die(err error) {
	fmt.Println(err)
	os.Exit(0)
}

func dies(err string) {
	fmt.Println(err)
	os.Exit(0)
}

func loadOrDie() ConfigFile {
	f, err := ioutil.ReadFile(*cfg)
	if err != nil {
		die(err)
	}

	var cf ConfigFile
	err = yaml.Unmarshal(f, &cf)
	if err != nil {
		die(err)
	}

	return cf
}

func isSubSystem(host string) (string, string, bool) {
	u, err := url.Parse(host)
	if err != nil {
		fmt.Println("Encountered error", err)
		return "", "", false
	}

	us, err := url.Parse(*subService)
	if err != nil {
		panic(err)
	}

	s := strings.Split(us.Host, "*")
	if len(s) != 2 {
		return "", u.Hostname(), false
	}

	if strings.HasSuffix(u.Hostname(), s[1]) {
		base := strings.Split(u.Hostname(), ".")[0]
		return base, u.Hostname(), true
	}

	return "", u.Hostname(), false
}

func noSiteFound(rw http.ResponseWriter, r *http.Request) {
	fmt.Fprint(rw, "No site found.")
}

func cloneSite(host, repo string) {
	l := loadLock()
	if l.Sites[host] != nil {
		if time.Since(l.Sites[host].Updated) > (10 * time.Second) {
			errBuf := etc.NewBuffer()
			fmt.Println("Pulling", host)
			c := exec.Command("git", "-C", "./git_home/"+l.Sites[host].ID, "pull")
			c.Stderr = errBuf
			c.Run()
			l.Sites[host].Updated = time.Now()
			writeLock(l)
			return
		}
	} else {
		fmt.Println("Cloning", host)
		l.Sites[host] = &coreMatch{
			Type:    "static",
			Repo:    repo,
			ID:      host,
			SubPath: loadOrDie().Sites[host].SubPath,
			Updated: time.Now(),
		}
		writeLock(l)

		therepo := ""
		branch := "master"
		repoel := strings.Split(repo, "#")
		if len(repoel) == 2 {
			branch = repoel[1]
			therepo = repoel[0]
		} else {
			therepo = repoel[0]
		}

		cmd := exec.Command("git", "clone", "-b", branch, therepo, "./git_home/"+l.Sites[host].ID)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Run()
	}
}

func exists(url string) bool {
	h, err := http.Get(url)
	if err != nil {
		return false
	}

	return h.StatusCode == 200
}

func isRemote(_url string) bool {
	u, err := url.Parse(_url)
	if err != nil {
		return false
	}

	return !(u.Scheme == "" && u.Opaque == "" && u.Host == "")
}

func pushLocal(href string, as string, rw http.ResponseWriter, r *http.Request, slice *[]string) {
	pusher, ok := rw.(http.Pusher)

	if isRemote(href) == false {
		if strings.HasPrefix(href, "/") == false {
			href = "/" + href
		}

		if ok {
			err := pusher.Push(href, &http.PushOptions{
				Header: http.Header{
					"Accept-Encoding": r.Header["Accept-Encoding"],
				},
			})
			yo.Println("Pushing asset", href, err)
		} else {
			sli := *slice
			sli = append(sli, fmt.Sprintf("<%s>; as=%s; rel=preload", href, as))
			*slice = sli
			yo.Println("Pushing asset via preload", href)
		}

	} else {
		yo.Warn("non-remote asset", href)
	}
}

func serveHost(rw http.ResponseWriter, r *http.Request, host string) {
	lck := loadLock().Sites[host]
	id := lck.ID

	srcDir := "./git_home/" + id + lck.SubPath
	yo.Ok("Source directory: ", srcDir)

	var preloads = []string{}

	// Push static assets for acceleration
	if r.URL.Path == "/" {
		yo.Printf("%s %s [%s] (Attempting acceleration)\n", r.Method, r.URL, host)
		f, err := os.Open(srcDir + "/index.html")
		if err == nil {
			gq, err := goquery.NewDocumentFromReader(f)
			if err == nil {
				// Push CSS
				gq.Find("link").Each(func(i int, s *goquery.Selection) {
					if val, exists := s.Attr("rel"); exists {
						if val == "stylesheet" {
							if href, exists := s.Attr("href"); exists {
								pushLocal(href, "style", rw, r, &preloads)
							}
						}
					}
				})

				// Push JS
				gq.Find("script").Each(func(i int, s *goquery.Selection) {
					if src, exists := s.Attr("src"); exists {
						pushLocal(src, "script", rw, r, &preloads)
					}
				})
			}
		}
	}

	if len(preloads) > 0 {
		rw.Header().Set("Link", strings.Join(preloads, ","))
	}

	hnd := gziphandler.GzipHandler(http.FileServer(http.Dir(srcDir)))
	hnd.ServeHTTP(rw, r)
}

func proxyWs(rw http.ResponseWriter, r *http.Request, remote string, dFn func(network, addr string) (net.Conn, error)) {
	yo.Println("Attempting to proxy", remote)

	u, err := url.Parse(remote)
	if err != nil {
		yo.Fatal(err)
	}

	u.RawPath = r.URL.RawPath
	u.RawQuery = r.URL.Query().Encode()

	var upgrader = websocket.Upgrader{
		ReadBufferSize:  1024,
		WriteBufferSize: 1024,
		CheckOrigin: func(r *http.Request) bool {
			return true
		},
	}

	prx := &websocket.Dialer{
		NetDial: dFn,
	}

	if u.Scheme == "http" {
		u.Scheme = "ws"
	} else {
		u.Scheme = "wss"
	}

	remotecn, err := upgrader.Upgrade(rw, r, nil)
	if err != nil {
		yo.Println("Error upgrading", remotecn)
		return
	}

	yo.Ok("upgraded successfully")

	localcn, _, err := prx.Dial(u.String(), nil)
	if err != nil {
		http.Error(rw, "error connecting to backend websocket", 500)
		return
	}

	defer remotecn.Close()
	defer localcn.Close()

	go func() {
		for {
			mt, b, err := remotecn.ReadMessage()
			if err != nil {
				return
			}

			err = localcn.WriteMessage(mt, b)
			if err != nil {
				return
			}
		}
	}()

	for {
		mt, b, err := localcn.ReadMessage()
		if err != nil {
			return
		}

		err = remotecn.WriteMessage(mt, b)
		if err != nil {
			return
		}
	}
}

func serveProxy(rw http.ResponseWriter, r *http.Request, remoteURL string) {
	yo.Println("Serving proxy", remoteURL)

	cl := http.Client{}
	path := r.URL.Path

	if strings.HasSuffix(path, ".git") {
		http.Error(rw, "forbidden", http.StatusForbidden)
		return
	}

	nur, err := url.Parse(remoteURL)
	if err != nil {
		panic(err)
		return
	}

	var dialr func(network, address string) (net.Conn, error)

	// Proxy tor requests
	if strings.HasSuffix(nur.Host, ".onion") {
		yo.Println("Tor proxy", nur.Host)
		dialer, err := proxy.SOCKS5("tcp", "127.0.0.1:9050", nil, proxy.Direct)
		if err != nil {
			yo.Fatal(err)
		}
		dialr = dialer.Dial
	} else {
		yo.Println("Not a Tor proxy", nur.Host)
		dialr = func(network, addr string) (net.Conn, error) {
			return net.Dial(network, addr)
		}
	}

	cl.Transport = &http.Transport{Dial: dialr}

	if r.Header.Get("Upgrade") == "websocket" {
		proxyWs(rw, r, remoteURL, dialr)
		return
	}

	// handle websockets

	yo.Println(r.Method, r.URL.RawPath)

	nur.Path = path
	nur.RawQuery = r.URL.Query().Encode()

	nr, err := http.NewRequest(r.Method, nur.String(), r.Body)
	if err != nil {
		yo.Warn(err)
		return
	}

	nr.Header.Set("Origin", nur.String())

	for k := range r.Header {
		nr.Header.Set(k, r.Header.Get(k))
	}

	nr.Host = r.Host

	r.URL.RawPath = path

	resp, err := cl.Do(nr)
	if err != nil {
		yo.Println("Proxy error", err)
		rw.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(rw, "Backend request error")
		return
	}

	for k := range resp.Header {
		rw.Header().Set(k, resp.Header.Get(k))
	}

	rw.WriteHeader(resp.StatusCode)
	io.Copy(rw, resp.Body)
}

func router(rw http.ResponseWriter, r *http.Request) {
	h := r.Host
	user, rur, is := isSubSystem("https://" + h)
	yo.Println("Opened ", rur, "with user")
	if !is {
		l := loadOrDie()
		if l.Sites[rur] != nil {
			if l.Sites[rur].Type == "proxy" {
				serveProxy(rw, r, l.Sites[rur].Repo)
				return
			}

			cloneSite(rur, l.Sites[rur].Repo)
			serveHost(rw, r, rur)
			return
		}
		noSiteFound(rw, r)
	} else {
		c := loadOrDie()
		if c.Sites[rur] != nil {
			if c.Sites[rur].Type == "proxy" {
				serveProxy(rw, r, c.Sites[rur].Repo)
				return
			}
			cloneSite(rur, c.Sites[rur].Repo)
			serveHost(rw, r, rur)
			return
		}

		l := loadLock()
		yo.Println("Attempting to open domain", rur)
		if l.Sites[rur] != nil {
			cloneSite(rur, l.Sites[rur].Repo)
			serveHost(rw, r, rur)
			return
		} else {
			gURL := *gitService + url.QueryEscape(user) + "/homepage.git"
			ex := exists(gURL)
			yo.Println("Queryied user repo. ", gURL, exists(gURL))
			if ex {
				cloneSite(rur, gURL)
				serveHost(rw, r, rur)
			} else {
				noSiteFound(rw, r)
			}
		}
	}
}

func main() {
	pflag.Parse()

	yo.Println(spew.Sdump(loadOrDie()))

	r := http.NewServeMux()
	r.HandleFunc("/", router)

	log.Fatal(http.ListenAndServe(*addr, r))
}