From 0f0b1bee47be23c37e6298308c86d61d9bce33e0 Mon Sep 17 00:00:00 2001 From: mischief Date: Wed, 20 Jan 2016 17:36:13 -0800 Subject: [PATCH] switch to godocs vfs wrapper. unfortunately remote zip support is gone now. --- Makefile | 7 +- fs.go | 61 ++++++++++++++++ main.go | 88 ++++++++++++---------- store.go | 198 -------------------------------------------------- store_test.go | 22 ------ util.go | 29 ++++++++ 6 files changed, 142 insertions(+), 263 deletions(-) create mode 100644 fs.go delete mode 100644 store.go delete mode 100644 store_test.go create mode 100644 util.go diff --git a/Makefile b/Makefile index 135f794..57ecd30 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ P=gowerc SRC=bitbucket.org USER?=mischief +GO_VERSION=1.5.3 all: $(USER)/$(P) @@ -8,9 +9,9 @@ $(USER)/$(P): bin/$(P) docker build -t "$(USER)/$(P):latest" . bin/$(P): *.go - docker run --rm -v ${PWD}:/go/src/$(SRC)/$(USER)/$(P) golang:1.4 /bin/bash -c "go list -f '{{range .Imports}}{{printf \"%s\n\" .}}{{end}}' $(SRC)/$(USER)/$(P) | xargs go get -d; CGO_ENABLED=0 go build -v -installsuffix cgo -o /go/src/$(SRC)/$(USER)/$(P)/bin/$(P) $(SRC)/$(USER)/$(P)" + docker run --rm -v ${PWD}:/go/src/$(SRC)/$(USER)/$(P) golang:${GO_VERSION} /bin/bash -c "go list -f '{{range .Imports}}{{printf \"%s\n\" .}}{{end}}' $(SRC)/$(USER)/$(P) | xargs go get -d; CGO_ENABLED=0 go build -v -installsuffix cgo -o /go/src/$(SRC)/$(USER)/$(P)/bin/$(P) $(SRC)/$(USER)/$(P)" clean: - docker run --rm -v ${PWD}:/opt busybox rm /opt/bin/$(P) - docker rmi "$(USER)/$(P):latest" + docker run --rm -v ${PWD}:/opt busybox rm -f /opt/bin/$(P) + docker rmi -f "$(USER)/$(P):latest" diff --git a/fs.go b/fs.go new file mode 100644 index 0000000..a656a11 --- /dev/null +++ b/fs.go @@ -0,0 +1,61 @@ +package main + +import ( + "archive/zip" + "io" + "net/http" + "os" + "strings" + + "golang.org/x/tools/godoc/vfs/httpfs" + "golang.org/x/tools/godoc/vfs/zipfs" +) + +// Make a few FS. +// Supported types are: +// - local directory (foo/bar/) +// - local zip file (foo/bar.zip) +func NewFS(uri string) (*FS, error) { + if strings.HasSuffix(uri, ".zip") { + r, err := zip.OpenReader(uri) + if err != nil { + return nil, err + } + + zfs := zipfs.New(r, uri) + hfs := httpfs.New(zfs) + fs := &FS{ + FileSystem: hfs, + closeme: r, + } + + return fs, nil + } + + fi, err := os.Stat(uri) + if err != nil { + return nil, err + } + + if !fi.IsDir() { + return nil, os.ErrInvalid + } + + fs := &FS{ + FileSystem: http.Dir(uri), + } + + return fs, nil +} + +type FS struct { + http.FileSystem + closeme io.Closer +} + +func (f *FS) Close() error { + if f.closeme != nil { + return f.Close() + } + return nil +} diff --git a/main.go b/main.go index b2f1bb3..8e5ffd0 100644 --- a/main.go +++ b/main.go @@ -2,20 +2,18 @@ package main import ( "bytes" - "fmt" - "strings" - "time" - - "flag" - "log" - "os" - "encoding/json" + "flag" + "fmt" "html/template" - + "io" + "log" "net" "net/http" + "os" "path/filepath" + "strings" + "time" "github.com/russross/blackfriday" ) @@ -48,10 +46,11 @@ type MenuEntry struct { } type Werc struct { - root string - conf WercConfig - tmpl *template.Template - store Store + root string + conf WercConfig + tmpl *template.Template + + fs *FS } func New(root string) *Werc { @@ -60,22 +59,19 @@ func New(root string) *Werc { w.tmpl = template.New("root") var err error - if strings.HasSuffix(root, ".zip") { - w.store, err = openZipStore(root) - } else { - w.store = &fileStore{root} - err = nil - } + + w.fs, err = NewFS(root) if err != nil { log.Printf("can't open root: %v", err) return nil } - b, err := w.store.ReadFile("etc/config.json") + b, err := readfile(w.fs, "etc/config.json") if err != nil { log.Printf("error loading config.json: %v", err) return nil } + err = json.Unmarshal(b, &w.conf) if err != nil { log.Printf("%s: %s", root+"/config.json", err) @@ -85,7 +81,7 @@ func New(root string) *Werc { // load templates tmpls := []string{"base", "directory", "footer", "menu", "text", "topbar"} for _, tn := range tmpls { - b, err := w.store.ReadFile(fmt.Sprintf("lib/%s.html", tn)) + b, err := readfile(w.fs, fmt.Sprintf("lib/%s.html", tn)) if err != nil { panic(err) } @@ -160,7 +156,7 @@ func (werc *Werc) genmenu(site, dir string) []MenuEntry { for i := range dirs { var sub []MenuEntry b := filepath.Join(base, dirs[i]) - fi, _ := werc.store.ReadDir(b) + fi, _ := readdir(werc.fs, b) for _, f := range fi { newname, ok := okmenu(b, f) if !ok { @@ -204,7 +200,7 @@ func (werc *Werc) WercCommon(w http.ResponseWriter, r *http.Request, site string page.Menu = werc.genmenu(site, path) conf := "sites/" + site + "/_werc/config.json" - b, err := werc.store.ReadFile(conf) + b, err := readfile(werc.fs, conf) if err != nil { log.Printf("%s: %s", conf, err) } else { @@ -261,7 +257,7 @@ func (werc *Werc) WercDir(w http.ResponseWriter, r *http.Request, site, dir stri data.Title = r.URL.Path buf := new(bytes.Buffer) - fi, err := werc.store.ReadDir(dir) + fi, err := readdir(werc.fs, dir) if err != nil { http.Error(w, fmt.Sprintf("%s", err), 500) return @@ -278,7 +274,7 @@ func (werc *Werc) WercDir(w http.ResponseWriter, r *http.Request, site, dir stri } func (werc *Werc) WercMd(w http.ResponseWriter, r *http.Request, site, path string) { - b, err := werc.store.ReadFile(path) + b, err := readfile(werc.fs, path) if err != nil { http.Error(w, fmt.Sprintf("%s", err), 404) return @@ -288,7 +284,7 @@ func (werc *Werc) WercMd(w http.ResponseWriter, r *http.Request, site, path stri } func (werc *Werc) WercHTML(w http.ResponseWriter, r *http.Request, site, path string) { - b, err := werc.store.ReadFile(path) + b, err := readfile(werc.fs, path) if err != nil { http.Error(w, fmt.Sprintf("%s", err), 404) return @@ -297,7 +293,7 @@ func (werc *Werc) WercHTML(w http.ResponseWriter, r *http.Request, site, path st } func (werc *Werc) WercTXT(w http.ResponseWriter, r *http.Request, site, path string) { - b, err := werc.store.ReadFile(path) + b, err := readfile(werc.fs, path) if err != nil { http.Error(w, fmt.Sprintf("%s", err), 404) return @@ -313,7 +309,7 @@ func (werc *Werc) Pub(w http.ResponseWriter, r *http.Request, path string) { path = path[1:] } - b, err := werc.store.ReadFile(path) + b, err := readfile(werc.fs, path) if err != nil { log.Printf("Pub: %v", err) http.Error(w, err.Error(), 404) @@ -349,8 +345,11 @@ again: } if !strings.HasSuffix(path, "/") { - if st, err := werc.store.Stat(base + path); err == nil { - if st.Mode().IsDir() { + f, err := werc.fs.Open(base + path) + if err == nil { + defer f.Close() + fi, err := f.Stat() + if err != nil && fi.IsDir() { http.Redirect(w, r, path+"/", http.StatusMovedPermanently) return } @@ -377,26 +376,35 @@ again: } for _, f := range tryfiles { - if _, err := werc.store.Stat(f); err == nil { - log.Printf("%s %s", suf, f) - handler(w, r, site, f) - return + fh, err := werc.fs.Open(f) + if err != nil { + continue } + + defer fh.Close() + + log.Printf("%s %s", suf, f) + handler(w, r, site, f) + return } } - if st, err := werc.store.Stat(base + path); err == nil { + if f, err := werc.fs.Open(base + path); err == nil { + defer f.Close() + + st, _ := f.Stat() if st.Mode().IsDir() { // directory handling log.Printf("d %s", base+path) werc.WercDir(w, r, site, base+path) return - } else { - // plain file handling - log.Printf("f %s", base+path) - http.ServeFile(w, r, base+path) - return } + + // plain file handling + log.Printf("f %s", base+path) + + io.Copy(w, f) + return } if site != werc.conf.MasterSite { diff --git a/store.go b/store.go deleted file mode 100644 index 33f8a43..0000000 --- a/store.go +++ /dev/null @@ -1,198 +0,0 @@ -package main - -import ( - "archive/zip" - "io/ioutil" - "log" - "net/url" - "os" - "path/filepath" - "strings" - - "github.com/mischief/httpreader" -) - -type Store interface { - ReadFile(filename string) ([]byte, error) - ReadDir(dirname string) ([]os.FileInfo, error) - Stat(name string) (os.FileInfo, error) - Close() error -} - -type fileStore struct { - base string -} - -func (f *fileStore) ReadFile(filename string) ([]byte, error) { - path := filepath.Join(f.base, filename) - return ioutil.ReadFile(path) -} - -func (f *fileStore) ReadDir(dirname string) ([]os.FileInfo, error) { - path := filepath.Join(f.base, dirname) - // resolve symlinks - fis, err := ioutil.ReadDir(path) - if err != nil { - return nil, err - } - - for i, fi := range fis { - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - sympath := filepath.Join(path, fi.Name()) - newfi, err := os.Stat(sympath) - if err != nil { - log.Printf("broken symlink %q: %s", sympath, err) - continue - } - - fis[i] = newfi - } - } - - return fis, nil -} - -func (f *fileStore) Stat(name string) (os.FileInfo, error) { - path := filepath.Join(f.base, name) - return os.Stat(path) -} - -func (f *fileStore) Close() error { - return nil -} - -type zipStore struct { - archive *zip.Reader - closer func() error - files map[string]*zip.File - dirs map[string][]*zip.File -} - -func openZipStore(file string) (*zipStore, error) { - url, err := url.Parse(file) - if err != nil { - return nil, err - } - - zs := &zipStore{ - files: make(map[string]*zip.File), - dirs: make(map[string][]*zip.File), - } - - switch url.Scheme { - case "", "file": - r, err := zip.OpenReader(file) - if err != nil { - return nil, err - } - zs.closer = r.Close - zs.archive = &r.Reader - case "http", "https": - ra, err := httpreader.NewReader(file) - if err != nil { - return nil, err - } - sz, err := ra.Size() - if err != nil { - return nil, err - } - r, err := zip.NewReader(ra, sz) - if err != nil { - return nil, err - } - zs.archive = r - } - - log.Printf("zip %q loading files...", file) - for _, file := range zs.archive.File { - zs.files[file.Name] = file - log.Printf("%v...", file.Name) - } - - log.Printf("zip %q loading directories...", file) - for _, dir := range zs.archive.File { - if !strings.HasSuffix(dir.Name, "/") { - continue - } - - var fi []*zip.File - for _, file := range zs.files { - name := file.Name - if name == dir.Name { - continue - } - - if strings.HasSuffix(name, "/") { - name = name[:len(name)-1] - } - - d, _ := filepath.Split(name) - - if d != dir.Name { - continue - } - - fi = append(fi, file) - - if strings.HasPrefix(file.Name, dir.Name) { - log.Printf("%v in dir %v", file.Name, dir.Name) - } - } - - zs.dirs[dir.Name] = fi - log.Printf("%v...", dir.Name) - } - - return zs, nil -} - -func (z *zipStore) ReadFile(filename string) ([]byte, error) { - log.Printf("read of %v", filename) - zf, ok := z.files[filename] - if !ok { - return nil, os.ErrNotExist - } - - zr, err := zf.Open() - if err != nil { - return nil, err - } - defer zr.Close() - zc, err := ioutil.ReadAll(zr) - if err != nil { - return nil, err - } - - return zc, nil -} - -func (z *zipStore) ReadDir(dirname string) ([]os.FileInfo, error) { - var fi []os.FileInfo - - dir, ok := z.dirs[dirname] - if !ok { - return nil, os.ErrNotExist - } - - for _, d := range dir { - fi = append(fi, d.FileInfo()) - } - - return fi, nil -} - -func (z *zipStore) Stat(name string) (os.FileInfo, error) { - log.Printf("stat %v", name) - if f, ok := z.files[name]; ok { - return f.FileInfo(), nil - } - return nil, os.ErrNotExist -} - -func (z *zipStore) Close() error { - if z.closer != nil { - return z.closer() - } - - return nil -} diff --git a/store_test.go b/store_test.go deleted file mode 100644 index 0d41612..0000000 --- a/store_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import ( - "testing" -) - -func TestZipStore(t *testing.T) { - file := "fixtures/site.zip" - zs, err := openZipStore(file) - if err != nil { - t.Error(err) - return - } - - ls, err := zs.ReadDir("site/foo/") - if err != nil { - t.Error(err) - return - } - - t.Logf("file %v", ls[0].Name()) -} diff --git a/util.go b/util.go new file mode 100644 index 0000000..df8a352 --- /dev/null +++ b/util.go @@ -0,0 +1,29 @@ +package main + +import ( + "io/ioutil" + "net/http" + "os" +) + +func readfile(fs http.FileSystem, path string) ([]byte, error) { + f, err := fs.Open(path) + if err != nil { + return nil, err + } + + defer f.Close() + + return ioutil.ReadAll(f) +} + +func readdir(fs http.FileSystem, path string) ([]os.FileInfo, error) { + f, err := fs.Open(path) + if err != nil { + return nil, err + } + + defer f.Close() + + return f.Readdir(-1) +}