cmd/internal/assets: package for embedding resources (#2374)

.gitignore

@@ -30,3 +30,5 @@ debug
 *.svg
 /bin
 resource.syso
+
+*.resource.go

cmd/internal/asset/asset.go

@@ -0,0 +1,121 @@
+// Copyright (C) 2019 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+// Package asset implements asset embedding via implementing http.FileSystem interface.
+//
+// To use the package you would define:
+//
+//     //go:generate go run ../internal/asset/generate/main.go -pkg main -dir ../../web/bootstrap -var embeddedAssets -out console.resource.go
+//     var embeddedAssets http.FileSystem
+//
+// This will generate a new "console.resource.go" which contains the content of "../../web/bootstrap".
+//
+// In the program initialization you can select based on whether the embedded resources exist or not:
+//
+//     var assets http.FileSystem
+//     if *staticAssetDirectory != "" {
+//         assets = http.Dir(*staticAssetDirectory)
+//     } else if embeddedAssets == nil {
+//         assets = embeddedAssets
+//     } else {
+//         assets = http.Dir(defaultAssetLocation)
+//     }
+//
+// Then write the service in terms of http.FileSystem, which hides the actual thing used for loading.
+//
+package asset
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+)
+
+// Asset describes a tree of asset files and directories.
+type Asset struct {
+	Name     string
+	Mode     os.FileMode
+	ModTime  time.Time
+	Data     []byte
+	Children []*Asset
+}
+
+// ReadDir loads an asset directory from filesystem.
+func ReadDir(path string) (*Asset, error) {
+	abspath, err := filepath.Abs(path)
+	if err != nil {
+		return nil, err
+	}
+
+	asset, err := ReadFile(abspath)
+	if err != nil {
+		return nil, err
+	}
+	asset.Name = ""
+	return asset, nil
+}
+
+// ReadFile loads an asset from filesystem.
+func ReadFile(path string) (*Asset, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		closeErr := file.Close()
+		if closeErr != nil {
+			if err == nil {
+				err = closeErr
+			} else {
+				err = fmt.Errorf("%v: %v", err, closeErr)
+			}
+		}
+	}()
+
+	stat, err := file.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	asset := &Asset{
+		Name:    stat.Name(),
+		Mode:    stat.Mode(),
+		ModTime: stat.ModTime(),
+	}
+
+	if stat.IsDir() {
+		children, err := file.Readdir(-1)
+		if err != nil {
+			return nil, err
+		}
+		err = asset.readFiles(path, children)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		asset.Data, err = ioutil.ReadAll(file)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return asset, nil
+}
+
+// readFiles adds all nested files to asset
+func (asset *Asset) readFiles(dir string, infos []os.FileInfo) error {
+	for _, info := range infos {
+		child, err := ReadFile(filepath.Join(dir, info.Name()))
+		if err != nil {
+			return err
+		}
+		asset.Children = append(asset.Children, child)
+	}
+	sort.Slice(asset.Children, func(i, k int) bool {
+		return asset.Children[i].Name < asset.Children[k].Name
+	})
+	return nil
+}

cmd/internal/asset/asset_test.go

@@ -0,0 +1,66 @@
+// Copyright (C) 2019 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package asset_test
+
+import (
+	"io/ioutil"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"storj.io/storj/cmd/internal/asset"
+	"storj.io/storj/internal/testcontext"
+)
+
+func TestAssets(t *testing.T) {
+	ctx := testcontext.New(t)
+	defer ctx.Cleanup()
+
+	require.NoError(t, ioutil.WriteFile(ctx.File("a", "example.css"), []byte("/a/example.css"), 0644))
+	require.NoError(t, ioutil.WriteFile(ctx.File("a", "example.js"), []byte("/a/example.js"), 0644))
+	require.NoError(t, ioutil.WriteFile(ctx.File("alpha.css"), []byte("/alpha.css"), 0644))
+	require.NoError(t, ioutil.WriteFile(ctx.File("x", "beta.css"), []byte("/x/beta.css"), 0644))
+	require.NoError(t, ioutil.WriteFile(ctx.File("x", "y", "gamma.js"), []byte("/x/y/gamma.js"), 0644))
+
+	root, err := asset.ReadDir(ctx.Dir())
+	require.NotNil(t, root)
+	require.NoError(t, err)
+
+	// sparse check on the content
+	require.Equal(t, root.Name, "")
+	require.Equal(t, len(root.Children), 3)
+
+	require.Equal(t, root.Children[0].Name, "a")
+
+	require.Equal(t, root.Children[1].Name, "alpha.css")
+	require.Equal(t, root.Children[1].Data, []byte("/alpha.css"))
+
+	require.Equal(t, root.Children[2].Name, "x")
+	require.Equal(t, root.Children[2].Children[1].Children[0].Name, "gamma.js")
+
+	var walk func(prefix string, node *asset.Asset)
+	walk = func(prefix string, node *asset.Asset) {
+		if !node.Mode.IsDir() {
+			assert.Equal(t, string(node.Data), path.Join(prefix, node.Name))
+		} else {
+			assert.Equal(t, string(node.Data), "")
+		}
+
+		for _, child := range node.Children {
+			walk(path.Join(prefix, node.Name), child)
+		}
+	}
+	walk("/", root)
+
+	inmemory := asset.Inmemory(root)
+	for path, node := range inmemory.Index {
+		if !node.Mode.IsDir() {
+			assert.Equal(t, string(node.Data), path)
+		} else {
+			assert.Equal(t, string(node.Data), "")
+		}
+	}
+}

cmd/internal/asset/code.go

@@ -0,0 +1,80 @@
+// Copyright (C) 2019 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package asset
+
+import (
+	"bytes"
+	"fmt"
+)
+
+// InmemoryCode generates a function closure []byte that can be assigned to a variable.
+func (asset *Asset) InmemoryCode() []byte {
+	var source bytes.Buffer
+	fmt.Fprintf(&source, "func() *asset.InmemoryFileSystem {\n")
+
+	blob := []byte{}
+	blobMapping := map[*Asset][2]int{}
+
+	var writeBlob func(asset *Asset)
+	writeBlob = func(asset *Asset) {
+		if !asset.Mode.IsDir() {
+			start := len(blob)
+			blob = append(blob, asset.Data...)
+			finish := len(blob)
+			blobMapping[asset] = [2]int{start, finish}
+			return
+		}
+
+		for _, child := range asset.Children {
+			writeBlob(child)
+		}
+	}
+	writeBlob(asset)
+
+	fmt.Fprintf(&source, "const blob = ")
+
+	const lineLength = 120
+	for len(blob) > 0 {
+		if lineLength < len(blob) {
+			fmt.Fprintf(&source, "\t%q +\n", string(blob[:lineLength]))
+			blob = blob[lineLength:]
+			continue
+		}
+		fmt.Fprintf(&source, "\t%q\n", string(blob))
+		break
+	}
+
+	var writeAsset func(asset *Asset)
+	writeAsset = func(asset *Asset) {
+		fmt.Fprintf(&source, "{")
+		defer fmt.Fprintf(&source, "}")
+		if asset.Mode.IsDir() {
+			fmt.Fprintf(&source, "\n")
+		}
+		fmt.Fprintf(&source, "Name: %q,", asset.Name)
+		fmt.Fprintf(&source, "Mode: 0%o,", asset.Mode)
+		fmt.Fprintf(&source, "ModTime: time.Unix(%d, 0),", asset.ModTime.Unix())
+
+		if !asset.Mode.IsDir() {
+			r := blobMapping[asset]
+			fmt.Fprintf(&source, "Data: []byte(blob[%d:%d])", r[0], r[1])
+			return
+		}
+
+		fmt.Fprintf(&source, "\nChildren: []*asset.Asset{\n")
+		for _, child := range asset.Children {
+			writeAsset(child)
+			fmt.Fprintf(&source, ",\n")
+		}
+		fmt.Fprintf(&source, "},\n")
+	}
+
+	fmt.Fprintf(&source, "\n")
+	fmt.Fprintf(&source, "return asset.Inmemory(&asset.Asset")
+	writeAsset(asset)
+	fmt.Fprintf(&source, ")\n")
+
+	fmt.Fprintf(&source, "}()\n")
+	return source.Bytes()
+}

cmd/internal/asset/generate/main.go

@@ -0,0 +1,58 @@
+// Copyright (C) 2019 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"go/format"
+	"io/ioutil"
+	"log"
+	"os"
+
+	"storj.io/storj/cmd/internal/asset"
+)
+
+func main() {
+	packageName := flag.String("pkg", "", "package name")
+	variableName := flag.String("var", "", "variable name to assign to")
+	dir := flag.String("dir", "", "directory")
+	out := flag.String("out", "", "output file")
+
+	flag.Parse()
+
+	asset, err := asset.ReadDir(*dir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var code bytes.Buffer
+	fmt.Fprintf(&code, "// DO NOT COMMIT\n\n")
+	fmt.Fprintf(&code, "package %s\n\n", *packageName)
+
+	fmt.Fprintf(&code, "import (\n")
+	fmt.Fprintf(&code, "\t\t\"storj.io/cmd/internal/asset\"\n")
+	fmt.Fprintf(&code, ")\n\n")
+
+	fmt.Fprintf(&code, "func init() {\n")
+	fmt.Fprintf(&code, "%s = ", *variableName)
+	code.Write(asset.InmemoryCode())
+	fmt.Fprintf(&code, "}\n")
+
+	formatted, err := format.Source(code.Bytes())
+	if err != nil {
+		fmt.Fprintln(os.Stderr, code.String())
+		log.Fatal(err)
+	}
+
+	if *out == "" {
+		fmt.Println(string(formatted))
+	} else {
+		err := ioutil.WriteFile(*out, formatted, 0644)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+}

cmd/internal/asset/inmemory.go

@@ -0,0 +1,117 @@
+// Copyright (C) 2019 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package asset
+
+import (
+	"bytes"
+	"errors"
+	"net/http"
+	"os"
+	"path"
+	"time"
+)
+
+var _ http.FileSystem = (*InmemoryFileSystem)(nil)
+
+// InmemoryFileSystem defines an inmemory http.FileSystem
+type InmemoryFileSystem struct {
+	Root  *Asset
+	Index map[string]*Asset
+}
+
+// Inmemory creates an InmemoryFileSystem from
+func Inmemory(root *Asset) *InmemoryFileSystem {
+	fs := &InmemoryFileSystem{}
+	fs.Root = root
+	fs.Index = map[string]*Asset{}
+	fs.reindex("/", "", root)
+	return fs
+}
+
+// reindex inserts a node to the index
+func (fs *InmemoryFileSystem) reindex(prefix, name string, file *Asset) {
+	fs.Index[path.Join(prefix, name)] = file
+	for _, child := range file.Children {
+		fs.reindex(path.Join(prefix, name), child.Name, child)
+	}
+}
+
+// Open opens the file at the specified path.
+func (fs *InmemoryFileSystem) Open(path string) (http.File, error) {
+	asset, ok := fs.Index[path]
+	if !ok {
+		return nil, os.ErrNotExist
+	}
+	return asset.File(), nil
+}
+
+// File opens the particular asset as a file.
+func (asset *Asset) File() *File {
+	return &File{*bytes.NewReader(asset.Data), asset}
+}
+
+// File defines a readable file
+type File struct {
+	bytes.Reader
+	*Asset
+}
+
+// Readdir reads all file infos from the directory.
+func (file *File) Readdir(count int) ([]os.FileInfo, error) {
+	if !file.Mode.IsDir() {
+		return nil, errors.New("not a directory")
+	}
+
+	if count > len(file.Children) {
+		count = len(file.Children)
+	}
+
+	infos := make([]os.FileInfo, 0, count)
+	for _, child := range file.Children {
+		infos = append(infos, child.stat())
+	}
+
+	return infos, nil
+}
+
+func (asset *Asset) stat() FileInfo {
+	return FileInfo{
+		name:    asset.Name,
+		size:    int64(len(asset.Data)),
+		mode:    asset.Mode,
+		modTime: asset.ModTime,
+	}
+}
+
+// Stat returns stats about the file.
+func (file *File) Stat() (os.FileInfo, error) { return file.stat(), nil }
+
+// Close closes the file.
+func (file *File) Close() error { return nil }
+
+// FileInfo implements file info.
+type FileInfo struct {
+	name    string
+	size    int64
+	mode    os.FileMode
+	modTime time.Time
+}
+
+// Name implements os.FileInfo
+func (info FileInfo) Name() string { return info.name }
+
+// Size implements os.FileInfo
+func (info FileInfo) Size() int64 { return info.size }
+
+// Mode implements os.FileInfo
+func (info FileInfo) Mode() os.FileMode { return info.mode }
+
+// ModTime implements os.FileInfo
+func (info FileInfo) ModTime() time.Time { return info.modTime }
+
+// IsDir implements os.FileInfo
+func (info FileInfo) IsDir() bool { return info.mode.IsDir() }
+
+// Sys implements os.FileInfo
+func (info FileInfo) Sys() interface{} { return nil }