JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cmd/uplink: import imports 'access' into existing configuration

Browse Source 
https://storjlabs.atlassian.net/browse/V3-3491

Change-Id: I9c5f649ded314bb3a2235588c746913a3ec2d203
This commit is contained in:
Michal Niewrzal 2020-01-10 16:51:28 +01:00
parent a57ce18f58
commit c8ccd26e04
5 changed files with 170 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
cmd/uplink/cmd/config.go
View File

@ -85,15 +85,19 @@ func (a AccessConfig) GetAccess() (_ *libuplink.Scope, err error) {
// fallback to scope if access not found
if a.Access == "" {
if data, ok := a.Scopes[a.Scope]; ok && a.Scope != "" {
return libuplink.ParseScope(data)
}
a.Access = a.Scope
}
// if a access exists for that name, try to load it.
if data, ok := a.Accesses[a.Access]; ok && a.Access != "" {
return libuplink.ParseScope(data)
if a.Access == "" {
return nil, errs.New("must specify access")
}
access, err := a.GetNamedAccess(a.Access)
if err != nil {
return nil, err
}
if access != nil {
return access, nil
}
// Otherwise, try to load the access name as a serialized access.
@ -144,6 +148,20 @@ func (a AccessConfig) GetAccess() (_ *libuplink.Scope, err error) {
}, nil
}
// GetNamedAccess returns named access if exists.
func (a AccessConfig) GetNamedAccess(name string) (_ *libuplink.Scope, err error) {
// if an access exists for that name, try to load it.
if data, ok := a.Accesses[name]; ok {
return libuplink.ParseScope(data)
}
// fallback to scopes
if data, ok := a.Scopes[name]; ok {
return libuplink.ParseScope(data)
}
return nil, nil
}
// GetRedundancyScheme returns the configured redundancy scheme for new uploads
func (c Config) GetRedundancyScheme() storj.RedundancyScheme {
return storj.RedundancyScheme{

168
cmd/uplink/cmd/import.go
View File

@ -26,10 +26,11 @@ var importCfg struct {
func init() {
importCmd := &cobra.Command{
Use: "import NAME PATH",
Short: "Imports an access under the given name from the supplied path",
Args: cobra.ExactArgs(2),
RunE: importMain,
Use: "import [NAME] (ACCESS | FILE)",
Short: "Imports an access into configuration. Configuration will be created if doesn't exists.",
Args: cobra.RangeArgs(1, 2),
RunE: importMain,
Annotations: map[string]string{"type": "setup"},
}
RootCmd.AddCommand(importCmd)
@ -43,55 +44,109 @@ func init() {
// importMain is the function executed when importCmd is called
func importMain(cmd *cobra.Command, args []string) (err error) {
name := args[0]
path := args[1]
// This is a little hacky but viper deserializes accesses into a map[string]interface{}
// and complains if we try and override with map[string]string{}.
accesses := map[string]interface{}{}
for k, v := range importCfg.Accesses {
accesses[k] = v
}
overwritten := false
if _, ok := accesses[name]; ok {
if !importCfg.Overwrite {
return fmt.Errorf("access %q already exists", name)
saveConfig := func(saveConfigOption process.SaveConfigOption) error {
path := filepath.Join(confDir, process.DefaultCfgFilename)
exists, err := fileExists(path)
if err != nil {
return Error.Wrap(err)
}
overwritten = true
if !exists {
if err := createConfigFile(path); err != nil {
return err
}
}
return process.SaveConfig(cmd, path,
saveConfigOption,
process.SaveConfigRemovingDeprecated())
}
accessData, err := readFirstUncommentedLine(path)
if err != nil {
return Error.Wrap(err)
}
// one argument means we are importing into main 'access' field without name
if len(args) == 1 {
overwritten := false
if importCfg.Access != "" {
if !importCfg.Overwrite {
return Error.New("%s", "default access already exists")
}
overwritten = true
}
// Parse the scope data to ensure it is well formed
if _, err := libuplink.ParseScope(accessData); err != nil {
return Error.Wrap(err)
}
access, err := findAccess(args[0])
if err != nil {
return Error.Wrap(err)
}
accesses[name] = accessData
if err := saveConfig(process.SaveConfigWithOverride("access", access)); err != nil {
return err
}
// There is no easy way currently to save off a "hidden" configurable into
// the config file without a larger refactoring. For now, just do a manual
// override of the accesses.
// TODO: revisit when the configuration/flag code makes it easy
err = process.SaveConfig(cmd, filepath.Join(confDir, process.DefaultCfgFilename),
process.SaveConfigWithOverride("accesses", accesses),
process.SaveConfigRemovingDeprecated())
if err != nil {
return Error.Wrap(err)
}
if overwritten {
fmt.Printf("access %q overwritten.\n", name)
if overwritten {
fmt.Printf("default access overwritten.\n")
} else {
fmt.Printf("default access imported.\n")
}
} else {
fmt.Printf("access %q imported.\n", name)
name := args[0]
// This is a little hacky but viper deserializes accesses into a map[string]interface{}
// and complains if we try and override with map[string]string{}.
accesses := map[string]interface{}{}
for k, v := range importCfg.Accesses {
accesses[k] = v
}
overwritten := false
if _, ok := accesses[name]; ok {
if !importCfg.Overwrite {
return fmt.Errorf("access %q already exists", name)
}
overwritten = true
}
accessData, err := findAccess(args[1])
if err != nil {
return Error.Wrap(err)
}
accesses[name] = accessData
// There is no easy way currently to save off a "hidden" configurable into
// the config file without a larger refactoring. For now, just do a manual
// override of the accesses.
// TODO: revisit when the configuration/flag code makes it easy
if err := saveConfig(process.SaveConfigWithOverride("accesses", accesses)); err != nil {
return err
}
if overwritten {
fmt.Printf("access %q overwritten.\n", name)
} else {
fmt.Printf("access %q imported.\n", name)
}
}
return nil
}
func findAccess(input string) (access string, err error) {
// check if parameter is a valid access, otherwise try to read it from file
if _, err := libuplink.ParseScope(input); err == nil {
access = input
} else {
path := input
access, err = readFirstUncommentedLine(path)
if err != nil {
return "", err
}
// Parse the access data to ensure it is well formed
if _, err := libuplink.ParseScope(access); err != nil {
return "", err
}
}
return access, nil
}
func readFirstUncommentedLine(path string) (_ string, err error) {
f, err := os.Open(path)
if err != nil {
@ -117,3 +172,32 @@ func readFirstUncommentedLine(path string) (_ string, err error) {
return "", Error.New("no data found")
}
func createConfigFile(path string) error {
setupDir, err := filepath.Abs(confDir)
if err != nil {
return err
}
err = os.MkdirAll(setupDir, 0700)
if err != nil {
return err
}
_, err = os.Create(path)
if err != nil {
return err
}
return nil
}
func fileExists(path string) (bool, error) {
stat, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
return !stat.IsDir(), nil
}

7
cmd/uplink/main.go
View File

@ -14,8 +14,11 @@ import (
func main() {
process.ExecWithCustomConfig(cmd.RootCmd, func(cmd *cobra.Command, vip *viper.Viper) error {
accessFlag := cmd.Flags().Lookup("access")
if accessFlag == nil || accessFlag.Value.String() == "" {
return process.LoadConfig(cmd, vip)
// try to load configuration because we may still need 'accesses' (for named access)
// field but error only if 'access' flag is not set
err := process.LoadConfig(cmd, vip)
if err != nil && (accessFlag == nil || accessFlag.Value.String() == "") {
return err
}
return nil
})

5
pkg/process/exec_conf.go
View File

@ -149,9 +149,10 @@ func LoadConfig(cmd *cobra.Command, vip *viper.Viper) error {
cfgFlag := cmd.Flags().Lookup("config-dir")
if cfgFlag != nil && cfgFlag.Value.String() != "" {
path := filepath.Join(os.ExpandEnv(cfgFlag.Value.String()), DefaultCfgFilename)
if cmd.Annotations["type"] != "setup" || fileExists(path) {
if fileExists(path) {
setupCommand := cmd.Annotations["type"] == "setup"
vip.SetConfigFile(path)
if err := vip.ReadInConfig(); err != nil {
if err := vip.ReadInConfig(); err != nil && !setupCommand {
return err
}
}

13
scripts/test-uplink.sh
View File

@ -13,6 +13,7 @@ trap cleanup EXIT
BUCKET=bucket-123
SRC_DIR=$TMPDIR/source
DST_DIR=$TMPDIR/dst
UPLINK_DIR=$TMPDIR/uplink
mkdir -p "$SRC_DIR" "$DST_DIR"
@ -36,12 +37,22 @@ uplink --access "$GATEWAY_0_ACCESS" --progress=false --debug.addr "$UPLINK_DEBUG
uplink --access "$GATEWAY_0_ACCESS" --progress=false --debug.addr "$UPLINK_DEBUG_ADDR" cp "$SRC_DIR/multisegment-upload-testfile" "sj://$BUCKET/"
uplink --access "$GATEWAY_0_ACCESS" --progress=false --debug.addr "$UPLINK_DEBUG_ADDR" cp "$SRC_DIR/diff-size-segments" "sj://$BUCKET/"
uplink --config-dir "$UPLINK_DIR" import named-access $GATEWAY_0_ACCESS
FILES=$(uplink --config-dir "$UPLINK_DIR" --access named-access ls "sj://$BUCKET" | wc -l)
EXPECTED_FILES="4"
if [ "$FILES" == $EXPECTED_FILES ]
then
echo "listing returns $FILES files"
else
echo "listing returns $FILES files but want $EXPECTED_FILES"
exit 1
fi
uplink --access "$GATEWAY_0_ACCESS" --progress=false --debug.addr "$UPLINK_DEBUG_ADDR" cp "sj://$BUCKET/small-upload-testfile" "$DST_DIR"
uplink --access "$GATEWAY_0_ACCESS" --progress=false --debug.addr "$UPLINK_DEBUG_ADDR" cp "sj://$BUCKET/big-upload-testfile" "$DST_DIR"
uplink --access "$GATEWAY_0_ACCESS" --progress=false --debug.addr "$UPLINK_DEBUG_ADDR" cp "sj://$BUCKET/multisegment-upload-testfile" "$DST_DIR"
uplink --access "$GATEWAY_0_ACCESS" --progress=false --debug.addr "$UPLINK_DEBUG_ADDR" cp "sj://$BUCKET/diff-size-segments" "$DST_DIR"
uplink --access "$GATEWAY_0_ACCESS" --debug.addr "$UPLINK_DEBUG_ADDR" rm "sj://$BUCKET/small-upload-testfile"
uplink --access "$GATEWAY_0_ACCESS" --debug.addr "$UPLINK_DEBUG_ADDR" rm "sj://$BUCKET/big-upload-testfile"
uplink --access "$GATEWAY_0_ACCESS" --debug.addr "$UPLINK_DEBUG_ADDR" rm "sj://$BUCKET/multisegment-upload-testfile"