diff --git a/.gitignore b/.gitignore index 257a6c9b0..6f2857807 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ debug *.svg /bin resource.syso + +*.resource.go \ No newline at end of file diff --git a/cmd/internal/asset/asset.go b/cmd/internal/asset/asset.go new file mode 100644 index 000000000..1f641c3ee --- /dev/null +++ b/cmd/internal/asset/asset.go @@ -0,0 +1,121 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +// Package asset implements asset embedding via implementing http.FileSystem interface. +// +// To use the package you would define: +// +// //go:generate go run ../internal/asset/generate/main.go -pkg main -dir ../../web/bootstrap -var embeddedAssets -out console.resource.go +// var embeddedAssets http.FileSystem +// +// This will generate a new "console.resource.go" which contains the content of "../../web/bootstrap". +// +// In the program initialization you can select based on whether the embedded resources exist or not: +// +// var assets http.FileSystem +// if *staticAssetDirectory != "" { +// assets = http.Dir(*staticAssetDirectory) +// } else if embeddedAssets == nil { +// assets = embeddedAssets +// } else { +// assets = http.Dir(defaultAssetLocation) +// } +// +// Then write the service in terms of http.FileSystem, which hides the actual thing used for loading. +// +package asset + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "time" +) + +// Asset describes a tree of asset files and directories. +type Asset struct { + Name string + Mode os.FileMode + ModTime time.Time + Data []byte + Children []*Asset +} + +// ReadDir loads an asset directory from filesystem. +func ReadDir(path string) (*Asset, error) { + abspath, err := filepath.Abs(path) + if err != nil { + return nil, err + } + + asset, err := ReadFile(abspath) + if err != nil { + return nil, err + } + asset.Name = "" + return asset, nil +} + +// ReadFile loads an asset from filesystem. +func ReadFile(path string) (*Asset, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { + closeErr := file.Close() + if closeErr != nil { + if err == nil { + err = closeErr + } else { + err = fmt.Errorf("%v: %v", err, closeErr) + } + } + }() + + stat, err := file.Stat() + if err != nil { + return nil, err + } + + asset := &Asset{ + Name: stat.Name(), + Mode: stat.Mode(), + ModTime: stat.ModTime(), + } + + if stat.IsDir() { + children, err := file.Readdir(-1) + if err != nil { + return nil, err + } + err = asset.readFiles(path, children) + if err != nil { + return nil, err + } + } else { + asset.Data, err = ioutil.ReadAll(file) + if err != nil { + return nil, err + } + } + + return asset, nil +} + +// readFiles adds all nested files to asset +func (asset *Asset) readFiles(dir string, infos []os.FileInfo) error { + for _, info := range infos { + child, err := ReadFile(filepath.Join(dir, info.Name())) + if err != nil { + return err + } + asset.Children = append(asset.Children, child) + } + sort.Slice(asset.Children, func(i, k int) bool { + return asset.Children[i].Name < asset.Children[k].Name + }) + return nil +} diff --git a/cmd/internal/asset/asset_test.go b/cmd/internal/asset/asset_test.go new file mode 100644 index 000000000..9d890c33a --- /dev/null +++ b/cmd/internal/asset/asset_test.go @@ -0,0 +1,66 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package asset_test + +import ( + "io/ioutil" + "path" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "storj.io/storj/cmd/internal/asset" + "storj.io/storj/internal/testcontext" +) + +func TestAssets(t *testing.T) { + ctx := testcontext.New(t) + defer ctx.Cleanup() + + require.NoError(t, ioutil.WriteFile(ctx.File("a", "example.css"), []byte("/a/example.css"), 0644)) + require.NoError(t, ioutil.WriteFile(ctx.File("a", "example.js"), []byte("/a/example.js"), 0644)) + require.NoError(t, ioutil.WriteFile(ctx.File("alpha.css"), []byte("/alpha.css"), 0644)) + require.NoError(t, ioutil.WriteFile(ctx.File("x", "beta.css"), []byte("/x/beta.css"), 0644)) + require.NoError(t, ioutil.WriteFile(ctx.File("x", "y", "gamma.js"), []byte("/x/y/gamma.js"), 0644)) + + root, err := asset.ReadDir(ctx.Dir()) + require.NotNil(t, root) + require.NoError(t, err) + + // sparse check on the content + require.Equal(t, root.Name, "") + require.Equal(t, len(root.Children), 3) + + require.Equal(t, root.Children[0].Name, "a") + + require.Equal(t, root.Children[1].Name, "alpha.css") + require.Equal(t, root.Children[1].Data, []byte("/alpha.css")) + + require.Equal(t, root.Children[2].Name, "x") + require.Equal(t, root.Children[2].Children[1].Children[0].Name, "gamma.js") + + var walk func(prefix string, node *asset.Asset) + walk = func(prefix string, node *asset.Asset) { + if !node.Mode.IsDir() { + assert.Equal(t, string(node.Data), path.Join(prefix, node.Name)) + } else { + assert.Equal(t, string(node.Data), "") + } + + for _, child := range node.Children { + walk(path.Join(prefix, node.Name), child) + } + } + walk("/", root) + + inmemory := asset.Inmemory(root) + for path, node := range inmemory.Index { + if !node.Mode.IsDir() { + assert.Equal(t, string(node.Data), path) + } else { + assert.Equal(t, string(node.Data), "") + } + } +} diff --git a/cmd/internal/asset/code.go b/cmd/internal/asset/code.go new file mode 100644 index 000000000..e32ec3db8 --- /dev/null +++ b/cmd/internal/asset/code.go @@ -0,0 +1,80 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package asset + +import ( + "bytes" + "fmt" +) + +// InmemoryCode generates a function closure []byte that can be assigned to a variable. +func (asset *Asset) InmemoryCode() []byte { + var source bytes.Buffer + fmt.Fprintf(&source, "func() *asset.InmemoryFileSystem {\n") + + blob := []byte{} + blobMapping := map[*Asset][2]int{} + + var writeBlob func(asset *Asset) + writeBlob = func(asset *Asset) { + if !asset.Mode.IsDir() { + start := len(blob) + blob = append(blob, asset.Data...) + finish := len(blob) + blobMapping[asset] = [2]int{start, finish} + return + } + + for _, child := range asset.Children { + writeBlob(child) + } + } + writeBlob(asset) + + fmt.Fprintf(&source, "const blob = ") + + const lineLength = 120 + for len(blob) > 0 { + if lineLength < len(blob) { + fmt.Fprintf(&source, "\t%q +\n", string(blob[:lineLength])) + blob = blob[lineLength:] + continue + } + fmt.Fprintf(&source, "\t%q\n", string(blob)) + break + } + + var writeAsset func(asset *Asset) + writeAsset = func(asset *Asset) { + fmt.Fprintf(&source, "{") + defer fmt.Fprintf(&source, "}") + if asset.Mode.IsDir() { + fmt.Fprintf(&source, "\n") + } + fmt.Fprintf(&source, "Name: %q,", asset.Name) + fmt.Fprintf(&source, "Mode: 0%o,", asset.Mode) + fmt.Fprintf(&source, "ModTime: time.Unix(%d, 0),", asset.ModTime.Unix()) + + if !asset.Mode.IsDir() { + r := blobMapping[asset] + fmt.Fprintf(&source, "Data: []byte(blob[%d:%d])", r[0], r[1]) + return + } + + fmt.Fprintf(&source, "\nChildren: []*asset.Asset{\n") + for _, child := range asset.Children { + writeAsset(child) + fmt.Fprintf(&source, ",\n") + } + fmt.Fprintf(&source, "},\n") + } + + fmt.Fprintf(&source, "\n") + fmt.Fprintf(&source, "return asset.Inmemory(&asset.Asset") + writeAsset(asset) + fmt.Fprintf(&source, ")\n") + + fmt.Fprintf(&source, "}()\n") + return source.Bytes() +} diff --git a/cmd/internal/asset/generate/main.go b/cmd/internal/asset/generate/main.go new file mode 100644 index 000000000..066acf43f --- /dev/null +++ b/cmd/internal/asset/generate/main.go @@ -0,0 +1,58 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package main + +import ( + "bytes" + "flag" + "fmt" + "go/format" + "io/ioutil" + "log" + "os" + + "storj.io/storj/cmd/internal/asset" +) + +func main() { + packageName := flag.String("pkg", "", "package name") + variableName := flag.String("var", "", "variable name to assign to") + dir := flag.String("dir", "", "directory") + out := flag.String("out", "", "output file") + + flag.Parse() + + asset, err := asset.ReadDir(*dir) + if err != nil { + log.Fatal(err) + } + + var code bytes.Buffer + fmt.Fprintf(&code, "// DO NOT COMMIT\n\n") + fmt.Fprintf(&code, "package %s\n\n", *packageName) + + fmt.Fprintf(&code, "import (\n") + fmt.Fprintf(&code, "\t\t\"storj.io/cmd/internal/asset\"\n") + fmt.Fprintf(&code, ")\n\n") + + fmt.Fprintf(&code, "func init() {\n") + fmt.Fprintf(&code, "%s = ", *variableName) + code.Write(asset.InmemoryCode()) + fmt.Fprintf(&code, "}\n") + + formatted, err := format.Source(code.Bytes()) + if err != nil { + fmt.Fprintln(os.Stderr, code.String()) + log.Fatal(err) + } + + if *out == "" { + fmt.Println(string(formatted)) + } else { + err := ioutil.WriteFile(*out, formatted, 0644) + if err != nil { + log.Fatal(err) + } + } +} diff --git a/cmd/internal/asset/inmemory.go b/cmd/internal/asset/inmemory.go new file mode 100644 index 000000000..b3c437a9f --- /dev/null +++ b/cmd/internal/asset/inmemory.go @@ -0,0 +1,117 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package asset + +import ( + "bytes" + "errors" + "net/http" + "os" + "path" + "time" +) + +var _ http.FileSystem = (*InmemoryFileSystem)(nil) + +// InmemoryFileSystem defines an inmemory http.FileSystem +type InmemoryFileSystem struct { + Root *Asset + Index map[string]*Asset +} + +// Inmemory creates an InmemoryFileSystem from +func Inmemory(root *Asset) *InmemoryFileSystem { + fs := &InmemoryFileSystem{} + fs.Root = root + fs.Index = map[string]*Asset{} + fs.reindex("/", "", root) + return fs +} + +// reindex inserts a node to the index +func (fs *InmemoryFileSystem) reindex(prefix, name string, file *Asset) { + fs.Index[path.Join(prefix, name)] = file + for _, child := range file.Children { + fs.reindex(path.Join(prefix, name), child.Name, child) + } +} + +// Open opens the file at the specified path. +func (fs *InmemoryFileSystem) Open(path string) (http.File, error) { + asset, ok := fs.Index[path] + if !ok { + return nil, os.ErrNotExist + } + return asset.File(), nil +} + +// File opens the particular asset as a file. +func (asset *Asset) File() *File { + return &File{*bytes.NewReader(asset.Data), asset} +} + +// File defines a readable file +type File struct { + bytes.Reader + *Asset +} + +// Readdir reads all file infos from the directory. +func (file *File) Readdir(count int) ([]os.FileInfo, error) { + if !file.Mode.IsDir() { + return nil, errors.New("not a directory") + } + + if count > len(file.Children) { + count = len(file.Children) + } + + infos := make([]os.FileInfo, 0, count) + for _, child := range file.Children { + infos = append(infos, child.stat()) + } + + return infos, nil +} + +func (asset *Asset) stat() FileInfo { + return FileInfo{ + name: asset.Name, + size: int64(len(asset.Data)), + mode: asset.Mode, + modTime: asset.ModTime, + } +} + +// Stat returns stats about the file. +func (file *File) Stat() (os.FileInfo, error) { return file.stat(), nil } + +// Close closes the file. +func (file *File) Close() error { return nil } + +// FileInfo implements file info. +type FileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time +} + +// Name implements os.FileInfo +func (info FileInfo) Name() string { return info.name } + +// Size implements os.FileInfo +func (info FileInfo) Size() int64 { return info.size } + +// Mode implements os.FileInfo +func (info FileInfo) Mode() os.FileMode { return info.mode } + +// ModTime implements os.FileInfo +func (info FileInfo) ModTime() time.Time { return info.modTime } + +// IsDir implements os.FileInfo +func (info FileInfo) IsDir() bool { return info.mode.IsDir() } + +// Sys implements os.FileInfo +func (info FileInfo) Sys() interface{} { return nil }