diff --git a/internal/fpath/temp_data.go b/internal/fpath/temp_data.go new file mode 100644 index 000000000..19c4b6086 --- /dev/null +++ b/internal/fpath/temp_data.go @@ -0,0 +1,38 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +// TODO maybe there is better place for this + +package fpath + +import "context" + +// The key type is unexported to prevent collisions with context keys defined in +// other packages. +type key int + +// temp is the context key for temp struct +const tempKey key = 0 + +type temp struct { + inmemory bool + directory string +} + +// WithTempData creates context with information how store temporary data, in memory or on disk +func WithTempData(ctx context.Context, directory string, inmemory bool) context.Context { + temp := &temp{ + inmemory: inmemory, + directory: directory, + } + return context.WithValue(ctx, tempKey, temp) +} + +// GetTempData returns if temporary data should be stored in memory or on disk +func GetTempData(ctx context.Context) (string, bool, bool) { + tempValue, ok := ctx.Value(tempKey).(temp) + if !ok { + return "", false, false + } + return tempValue.directory, tempValue.inmemory, ok +} diff --git a/internal/sync2/tee.go b/internal/sync2/tee.go index f47527364..badef0d9b 100644 --- a/internal/sync2/tee.go +++ b/internal/sync2/tee.go @@ -42,6 +42,13 @@ func NewTeeFile(readers int, tempdir string) ([]PipeReader, PipeWriter, error) { return newTee(buffer, readers, &handles) } +// NewTeeInmemory returns a tee that uses inmemory +func NewTeeInmemory(readers int, allocMemory int64) ([]PipeReader, PipeWriter, error) { + handles := int64(readers + 1) // +1 for the writer + memory := memory(make([]byte, allocMemory)) + return newTee(memory, readers, &handles) +} + func newTee(buffer ReadAtWriteAtCloser, readers int, open *int64) ([]PipeReader, PipeWriter, error) { tee := &tee{ buffer: buffer, diff --git a/internal/sync2/tee_test.go b/internal/sync2/tee_test.go index 66c1ffd0a..3e780a1db 100644 --- a/internal/sync2/tee_test.go +++ b/internal/sync2/tee_test.go @@ -12,44 +12,51 @@ import ( "github.com/stretchr/testify/assert" "golang.org/x/sync/errgroup" + "storj.io/storj/internal/memory" "storj.io/storj/internal/sync2" ) -func TestTee_Basic(t *testing.T) { - testTees(t, func(t *testing.T, readers []sync2.PipeReader, writer sync2.PipeWriter) { - var group errgroup.Group +func TestTee_Basic_On_Disk(t *testing.T) { + testTees(t, false, testBasic) +} + +func TestTee_Basic_In_Memory(t *testing.T) { + testTees(t, true, testBasic) +} + +func testBasic(t *testing.T, readers []sync2.PipeReader, writer sync2.PipeWriter) { + var group errgroup.Group + group.Go(func() error { + n, err := writer.Write([]byte{1, 2, 3}) + assert.Equal(t, n, 3) + assert.NoError(t, err) + + n, err = writer.Write([]byte{1, 2, 3}) + assert.Equal(t, n, 3) + assert.NoError(t, err) + + assert.NoError(t, writer.Close()) + return nil + }) + + for i := 0; i < len(readers); i++ { + i := i group.Go(func() error { - n, err := writer.Write([]byte{1, 2, 3}) - assert.Equal(t, n, 3) - assert.NoError(t, err) - - n, err = writer.Write([]byte{1, 2, 3}) - assert.Equal(t, n, 3) - assert.NoError(t, err) - - assert.NoError(t, writer.Close()) + data, err := ioutil.ReadAll(readers[i]) + assert.Equal(t, []byte{1, 2, 3, 1, 2, 3}, data) + if err != nil { + assert.Equal(t, io.EOF, err) + } + assert.NoError(t, readers[i].Close()) return nil }) + } - for i := 0; i < len(readers); i++ { - i := i - group.Go(func() error { - data, err := ioutil.ReadAll(readers[i]) - assert.Equal(t, []byte{1, 2, 3, 1, 2, 3}, data) - if err != nil { - assert.Equal(t, io.EOF, err) - } - assert.NoError(t, readers[i].Close()) - return nil - }) - } - - assert.NoError(t, group.Wait()) - }) + assert.NoError(t, group.Wait()) } func TestTee_CloseWithError(t *testing.T) { - testTees(t, func(t *testing.T, readers []sync2.PipeReader, writer sync2.PipeWriter) { + testTees(t, false, func(t *testing.T, readers []sync2.PipeReader, writer sync2.PipeWriter) { var failure = errors.New("write failure") var group errgroup.Group @@ -80,11 +87,19 @@ func TestTee_CloseWithError(t *testing.T) { }) } -func testTees(t *testing.T, test func(t *testing.T, readers []sync2.PipeReader, writer sync2.PipeWriter)) { +func testTees(t *testing.T, inmemory bool, test func(t *testing.T, readers []sync2.PipeReader, writer sync2.PipeWriter)) { t.Parallel() t.Run("File", func(t *testing.T) { t.Parallel() - readers, writer, err := sync2.NewTeeFile(2, "") + + var err error + var readers []sync2.PipeReader + var writer sync2.PipeWriter + if inmemory { + readers, writer, err = sync2.NewTeeInmemory(2, memory.MiB.Int64()) + } else { + readers, writer, err = sync2.NewTeeFile(2, "") + } if err != nil { t.Fatal(err) } diff --git a/lib/uplink/bucket.go b/lib/uplink/bucket.go index 3a378bdbc..26f5fe183 100644 --- a/lib/uplink/bucket.go +++ b/lib/uplink/bucket.go @@ -96,6 +96,38 @@ type UploadOptions struct { func (b *Bucket) UploadObject(ctx context.Context, path storj.Path, data io.Reader, opts *UploadOptions) (err error) { defer mon.Task()(&ctx)(&err) + upload, err := b.NewWriter(ctx, path, opts) + if err != nil { + return err + } + + _, err = io.Copy(upload, data) + + return errs.Combine(err, upload.Close()) +} + +// DeleteObject removes an object, if authorized. +func (b *Bucket) DeleteObject(ctx context.Context, path storj.Path) (err error) { + defer mon.Task()(&ctx)(&err) + return b.metainfo.DeleteObject(ctx, b.bucket.Name, path) +} + +// ListOptions controls options for the ListObjects() call. +type ListOptions = storj.ListOptions + +// ListObjects lists objects a user is authorized to see. +func (b *Bucket) ListObjects(ctx context.Context, cfg *ListOptions) (list storj.ObjectList, err error) { + defer mon.Task()(&ctx)(&err) + if cfg == nil { + cfg = &storj.ListOptions{} + } + return b.metainfo.ListObjects(ctx, b.bucket.Name, *cfg) +} + +// NewWriter creates a writer which uploads the object. +func (b *Bucket) NewWriter(ctx context.Context, path storj.Path, opts *UploadOptions) (_ io.WriteCloser, err error) { + defer mon.Task()(&ctx)(&err) + if opts == nil { opts = &UploadOptions{} } @@ -134,37 +166,35 @@ func (b *Bucket) UploadObject(ctx context.Context, path storj.Path, data io.Read obj, err := b.metainfo.CreateObject(ctx, b.Name, path, &createInfo) if err != nil { - return err + return nil, err } mutableStream, err := obj.CreateStream(ctx) if err != nil { - return err + return nil, err } upload := stream.NewUpload(ctx, mutableStream, b.streams) - - _, err = io.Copy(upload, data) - - return errs.Combine(err, upload.Close()) + return upload, nil } -// DeleteObject removes an object, if authorized. -func (b *Bucket) DeleteObject(ctx context.Context, path storj.Path) (err error) { - defer mon.Task()(&ctx)(&err) - return b.metainfo.DeleteObject(ctx, b.bucket.Name, path) +// ReadSeekCloser combines interfaces io.Reader, io.Seeker, io.Closer +type ReadSeekCloser interface { + io.Reader + io.Seeker + io.Closer } -// ListOptions controls options for the ListObjects() call. -type ListOptions = storj.ListOptions - -// ListObjects lists objects a user is authorized to see. -func (b *Bucket) ListObjects(ctx context.Context, cfg *ListOptions) (list storj.ObjectList, err error) { +// NewReader creates a new reader that downloads the object data. +func (b *Bucket) NewReader(ctx context.Context, path storj.Path) (_ ReadSeekCloser, err error) { defer mon.Task()(&ctx)(&err) - if cfg == nil { - cfg = &storj.ListOptions{} + + segmentStream, err := b.metainfo.GetObjectStream(ctx, b.Name, path) + if err != nil { + return nil, err } - return b.metainfo.ListObjects(ctx, b.bucket.Name, *cfg) + + return stream.NewDownload(ctx, segmentStream, b.streams), nil } // Close closes the Bucket session. diff --git a/mobile/.gitignore b/mobile/.gitignore new file mode 100644 index 000000000..7498b75d0 --- /dev/null +++ b/mobile/.gitignore @@ -0,0 +1,3 @@ +.gradle +libuplink_android/app/libs/*.jar +libuplink_android/app/libs/*.aar \ No newline at end of file diff --git a/mobile/bucket.go b/mobile/bucket.go new file mode 100644 index 000000000..5d5515efd --- /dev/null +++ b/mobile/bucket.go @@ -0,0 +1,376 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package mobile + +import ( + "fmt" + "io" + "time" + + libuplink "storj.io/storj/lib/uplink" + "storj.io/storj/pkg/storj" +) + +const ( + // CipherSuiteEncUnspecified indicates no encryption suite has been selected. + CipherSuiteEncUnspecified = byte(storj.EncUnspecified) + // CipherSuiteEncNull indicates use of the NULL cipher; that is, no encryption is + // done. The ciphertext is equal to the plaintext. + CipherSuiteEncNull = byte(storj.EncNull) + // CipherSuiteEncAESGCM indicates use of AES128-GCM encryption. + CipherSuiteEncAESGCM = byte(storj.EncAESGCM) + // CipherSuiteEncSecretBox indicates use of XSalsa20-Poly1305 encryption, as provided + // by the NaCl cryptography library under the name "Secretbox". + CipherSuiteEncSecretBox = byte(storj.EncSecretBox) + + // DirectionAfter lists forwards from cursor, without cursor + DirectionAfter = int(storj.After) + // DirectionForward lists forwards from cursor, including cursor + DirectionForward = int(storj.Forward) + // DirectionBackward lists backwards from cursor, including cursor + DirectionBackward = int(storj.Backward) + // DirectionBefore lists backwards from cursor, without cursor + DirectionBefore = int(storj.Before) +) + +// Bucket represents operations you can perform on a bucket +type Bucket struct { + Name string + + scope + lib *libuplink.Bucket +} + +// BucketAccess defines access to bucket +type BucketAccess struct { + PathEncryptionKey []byte + EncryptedPathPrefix storj.Path +} + +// BucketInfo bucket meta struct +type BucketInfo struct { + Name string + Created int64 + PathCipher byte + SegmentsSize int64 + RedundancyScheme *RedundancyScheme + EncryptionParameters *EncryptionParameters +} + +func newBucketInfo(bucket storj.Bucket) *BucketInfo { + return &BucketInfo{ + Name: bucket.Name, + Created: bucket.Created.UTC().UnixNano() / int64(time.Millisecond), + PathCipher: byte(bucket.PathCipher), + SegmentsSize: bucket.SegmentsSize, + RedundancyScheme: &RedundancyScheme{ + Algorithm: byte(bucket.RedundancyScheme.Algorithm), + ShareSize: bucket.RedundancyScheme.ShareSize, + RequiredShares: bucket.RedundancyScheme.RequiredShares, + RepairShares: bucket.RedundancyScheme.RepairShares, + OptimalShares: bucket.RedundancyScheme.OptimalShares, + TotalShares: bucket.RedundancyScheme.TotalShares, + }, + EncryptionParameters: &EncryptionParameters{ + CipherSuite: byte(bucket.EncryptionParameters.CipherSuite), + BlockSize: bucket.EncryptionParameters.BlockSize, + }, + } +} + +// BucketConfig bucket configuration +type BucketConfig struct { + // PathCipher indicates which cipher suite is to be used for path + // encryption within the new Bucket. If not set, AES-GCM encryption + // will be used. + PathCipher byte + + // EncryptionParameters specifies the default encryption parameters to + // be used for data encryption of new Objects in this bucket. + EncryptionParameters *EncryptionParameters + + // RedundancyScheme defines the default Reed-Solomon and/or + // Forward Error Correction encoding parameters to be used by + // objects in this Bucket. + RedundancyScheme *RedundancyScheme + // SegmentsSize is the default segment size to use for new + // objects in this Bucket. + SegmentsSize int64 +} + +// BucketList is a list of buckets +type BucketList struct { + list storj.BucketList +} + +// More returns true if list request was not able to return all results +func (bl *BucketList) More() bool { + return bl.list.More +} + +// Length returns number of returned items +func (bl *BucketList) Length() int { + return len(bl.list.Items) +} + +// Item gets item from specific index +func (bl *BucketList) Item(index int) (*BucketInfo, error) { + if index < 0 && index >= len(bl.list.Items) { + return nil, fmt.Errorf("index out of range") + } + return newBucketInfo(bl.list.Items[index]), nil +} + +// RedundancyScheme specifies the parameters and the algorithm for redundancy +type RedundancyScheme struct { + // Algorithm determines the algorithm to be used for redundancy. + Algorithm byte + + // ShareSize is the size to use for new redundancy shares. + ShareSize int32 + + // RequiredShares is the minimum number of shares required to recover a + // segment. + RequiredShares int16 + // RepairShares is the minimum number of safe shares that can remain + // before a repair is triggered. + RepairShares int16 + // OptimalShares is the desired total number of shares for a segment. + OptimalShares int16 + // TotalShares is the number of shares to encode. If it is larger than + // OptimalShares, slower uploads of the excess shares will be aborted in + // order to improve performance. + TotalShares int16 +} + +func newStorjRedundancyScheme(scheme *RedundancyScheme) storj.RedundancyScheme { + if scheme == nil { + return storj.RedundancyScheme{} + } + return storj.RedundancyScheme{ + Algorithm: storj.RedundancyAlgorithm(scheme.Algorithm), + ShareSize: scheme.ShareSize, + RequiredShares: scheme.RequiredShares, + RepairShares: scheme.RepairShares, + OptimalShares: scheme.OptimalShares, + TotalShares: scheme.TotalShares, + } +} + +// EncryptionParameters is the cipher suite and parameters used for encryption +// It is like EncryptionScheme, but uses the CipherSuite type instead of Cipher. +// EncryptionParameters is preferred for new uses. +type EncryptionParameters struct { + // CipherSuite specifies the cipher suite to be used for encryption. + CipherSuite byte + // BlockSize determines the unit size at which encryption is performed. + // It is important to distinguish this from the block size used by the + // cipher suite (probably 128 bits). There is some small overhead for + // each encryption unit, so BlockSize should not be too small, but + // smaller sizes yield shorter first-byte latency and better seek times. + // Note that BlockSize itself is the size of data blocks _after_ they + // have been encrypted and the authentication overhead has been added. + // It is _not_ the size of the data blocks to _be_ encrypted. + BlockSize int32 +} + +func newStorjEncryptionParameters(ec *EncryptionParameters) storj.EncryptionParameters { + if ec == nil { + return storj.EncryptionParameters{} + } + return storj.EncryptionParameters{ + CipherSuite: storj.CipherSuite(ec.CipherSuite), + BlockSize: ec.BlockSize, + } +} + +// ListOptions options for listing objects +type ListOptions struct { + Prefix string + Cursor string // Cursor is relative to Prefix, full path is Prefix + Cursor + Delimiter int32 + Recursive bool + Direction int + Limit int +} + +// ListObjects list objects in bucket, if authorized. +func (bucket *Bucket) ListObjects(options *ListOptions) (*ObjectList, error) { + scope := bucket.scope.child() + + opts := &storj.ListOptions{} + if options != nil { + opts.Prefix = options.Prefix + opts.Cursor = options.Cursor + opts.Direction = storj.ListDirection(options.Direction) + opts.Delimiter = options.Delimiter + opts.Recursive = options.Recursive + opts.Limit = options.Limit + } + + list, err := bucket.lib.ListObjects(scope.ctx, opts) + if err != nil { + return nil, safeError(err) + } + return &ObjectList{list}, nil +} + +// OpenObject returns an Object handle, if authorized. +func (bucket *Bucket) OpenObject(objectPath string) (*ObjectInfo, error) { + scope := bucket.scope.child() + object, err := bucket.lib.OpenObject(scope.ctx, objectPath) + if err != nil { + return nil, safeError(err) + } + return newObjectInfoFromObjectMeta(object.Meta), nil +} + +// DeleteObject removes an object, if authorized. +func (bucket *Bucket) DeleteObject(objectPath string) error { + scope := bucket.scope.child() + return safeError(bucket.lib.DeleteObject(scope.ctx, objectPath)) +} + +// Close closes the Bucket session. +func (bucket *Bucket) Close() error { + defer bucket.cancel() + return safeError(bucket.lib.Close()) +} + +// WriterOptions controls options about writing a new Object +type WriterOptions struct { + // ContentType, if set, gives a MIME content-type for the Object. + ContentType string + // Metadata contains additional information about an Object. It can + // hold arbitrary textual fields and can be retrieved together with the + // Object. Field names can be at most 1024 bytes long. Field values are + // not individually limited in size, but the total of all metadata + // (fields and values) can not exceed 4 kiB. + Metadata map[string]string + // Expires is the time at which the new Object can expire (be deleted + // automatically from storage nodes). + Expires int + + // EncryptionParameters determines the cipher suite to use for + // the Object's data encryption. If not set, the Bucket's + // defaults will be used. + EncryptionParameters *EncryptionParameters + + // RedundancyScheme determines the Reed-Solomon and/or Forward + // Error Correction encoding parameters to be used for this + // Object. + RedundancyScheme *RedundancyScheme +} + +// NewWriterOptions creates writer options +func NewWriterOptions() *WriterOptions { + return &WriterOptions{} +} + +// Writer writes data into object +type Writer struct { + scope + writer io.WriteCloser +} + +// NewWriter creates instance of Writer +func (bucket *Bucket) NewWriter(path storj.Path, options *WriterOptions) (*Writer, error) { + scope := bucket.scope.child() + + opts := &libuplink.UploadOptions{} + if options != nil { + opts.ContentType = options.ContentType + opts.Metadata = options.Metadata + if options.Expires != 0 { + opts.Expires = time.Unix(int64(options.Expires), 0) + } + opts.Volatile.EncryptionParameters = newStorjEncryptionParameters(options.EncryptionParameters) + opts.Volatile.RedundancyScheme = newStorjRedundancyScheme(options.RedundancyScheme) + } + + writer, err := bucket.lib.NewWriter(scope.ctx, path, opts) + if err != nil { + return nil, safeError(err) + } + return &Writer{scope, writer}, nil +} + +// Write writes data.length bytes from data to the underlying data stream. +func (w *Writer) Write(data []byte, offset, length int32) (int32, error) { + // in Java byte array size is max int32 + n, err := w.writer.Write(data[offset:length]) + return int32(n), safeError(err) +} + +// Cancel cancels writing operation +func (w *Writer) Cancel() { + w.cancel() +} + +// Close closes writer +func (w *Writer) Close() error { + defer w.cancel() + return safeError(w.writer.Close()) +} + +// ReaderOptions options for reading +type ReaderOptions struct { +} + +// Reader reader for downloading object +type Reader struct { + scope + readError error + reader interface { + io.Reader + io.Seeker + io.Closer + } +} + +// NewReader returns new reader for downloading object +func (bucket *Bucket) NewReader(path storj.Path, options *ReaderOptions) (*Reader, error) { + scope := bucket.scope.child() + + reader, err := bucket.lib.NewReader(scope.ctx, path) + if err != nil { + return nil, safeError(err) + } + return &Reader{ + scope: scope, + reader: reader, + }, nil +} + +// Read reads data into byte array +func (r *Reader) Read(data []byte) (n int32, err error) { + if r.readError != nil { + err = r.readError + } else { + var read int + read, err = r.reader.Read(data) + n = int32(read) + } + + if n > 0 && err != nil { + r.readError = err + err = nil + } + + if err == io.EOF { + return -1, nil + } + return n, safeError(err) +} + +// Cancel cancels read operation +func (r *Reader) Cancel() { + r.cancel() +} + +// Close closes reader +func (r *Reader) Close() error { + defer r.cancel() + return safeError(r.reader.Close()) +} diff --git a/mobile/build.sh b/mobile/build.sh new file mode 100755 index 000000000..5d504f120 --- /dev/null +++ b/mobile/build.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# This script will build libuplink-android.aar library from scratch +# Required: +# * ANDROID_HOME set with NDK available +# * go +# * gospace + +if [ -z "$ANDROID_HOME" ] +then + echo "\$ANDROID_HOME is not set" + exit 1 +fi + +OUTPUT_DIR=${1:-$PWD} +OUTPUT_AAR="libuplink-android.aar" +OUTPUT_JAVA_PACKAGE="io.storj.libuplink" + +STORJ_PATH=~/storj-for-android + +# set go modules to default behavior +export GO111MODULE=auto + +# go knows where our gopath is +export GOPATH=$STORJ_PATH + +# gospace knows where our gopath is (this is to avoid accidental damage to existing GOPATH) +# you should not use default GOPATH here +export GOSPACE_ROOT=$STORJ_PATH + +# set the github repository that this GOSPACE manages +export GOSPACE_PKG=storj.io/storj + +# set the where the repository is located +export GOSPACE_REPO=git@github.com:storj/storj.git + +gospace setup + +export PATH=$PATH:$GOPATH/bin + +# step can be removed after merge to master +cd $GOPATH/src/storj.io/storj +git checkout -q mn/java-bindings + +cd $GOPATH + +go get golang.org/x/mobile/cmd/gomobile + +gomobile init + +echo -e "\nbuilding aar" +gomobile bind -target android -o $OUTPUT_DIR/libuplink-android.aar -javapkg $OUTPUT_JAVA_PACKAGE storj.io/storj/mobile +echo "output aar: $OUTPUT_DIR/libuplink-android.aar" diff --git a/mobile/doc.go b/mobile/doc.go new file mode 100644 index 000000000..e55a4214b --- /dev/null +++ b/mobile/doc.go @@ -0,0 +1,25 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +// Package mobile contains the simplified mobile APIs to Storj Network. +// +// For API limitations see https://github.com/ethereum/go-ethereum/blob/461291882edce0ac4a28f64c4e8725b7f57cbeae/mobile/doc.go#L23 +// +// +// build loop for development +// watchrun gobind -lang=java -outdir=../mobile-out storj.io/storj/mobile == pt skipped ../mobile-out +// +// gomobile bind -target android +// +// To use: +// gomobile bind -target android +// +// Create a new project in AndroidStudio +// +// Copy mobile-source.jar and mobile.aar into `AndroidStudioProjects\MyApplication\app\libs\` +// +// Modify build.gradle to also find *.aar files: +// implementation fileTree(dir: 'libs', include: ['*.jar', '*.aar']) +// +// See example Java file +package mobile diff --git a/mobile/libuplink_android/.gitignore b/mobile/libuplink_android/.gitignore new file mode 100644 index 000000000..2b75303ac --- /dev/null +++ b/mobile/libuplink_android/.gitignore @@ -0,0 +1,13 @@ +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild diff --git a/mobile/libuplink_android/app/.gitignore b/mobile/libuplink_android/app/.gitignore new file mode 100644 index 000000000..796b96d1c --- /dev/null +++ b/mobile/libuplink_android/app/.gitignore @@ -0,0 +1 @@ +/build diff --git a/mobile/libuplink_android/app/build.gradle b/mobile/libuplink_android/app/build.gradle new file mode 100644 index 000000000..7ee3659be --- /dev/null +++ b/mobile/libuplink_android/app/build.gradle @@ -0,0 +1,27 @@ +apply plugin: 'com.android.application' + +android { + compileSdkVersion 28 + defaultConfig { + applicationId "io.storj.mobile.libuplink" + minSdkVersion 15 + targetSdkVersion 28 + versionCode 1 + versionName "1.0" + testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" + } + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } +} + +dependencies { + implementation fileTree(dir: 'libs', include: ['*.jar', '*.aar']) + implementation 'com.android.support:appcompat-v7:28.0.0' + testImplementation 'junit:junit:4.12' + androidTestImplementation 'com.android.support.test:runner:1.0.2' + androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2' +} diff --git a/mobile/libuplink_android/app/proguard-rules.pro b/mobile/libuplink_android/app/proguard-rules.pro new file mode 100644 index 000000000..f1b424510 --- /dev/null +++ b/mobile/libuplink_android/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/mobile/libuplink_android/app/src/androidTest/java/io/storj/mobile/libuplink/LibuplinkInstrumentedTest.java b/mobile/libuplink_android/app/src/androidTest/java/io/storj/mobile/libuplink/LibuplinkInstrumentedTest.java new file mode 100644 index 000000000..0d9e10b91 --- /dev/null +++ b/mobile/libuplink_android/app/src/androidTest/java/io/storj/mobile/libuplink/LibuplinkInstrumentedTest.java @@ -0,0 +1,338 @@ +package io.storj.mobile.libuplink; + +import android.support.test.InstrumentationRegistry; +import android.support.test.runner.AndroidJUnit4; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.ByteArrayOutputStream; +import java.util.HashSet; +import java.util.Random; +import java.util.Set; + +import io.storj.libuplink.mobile.*; + +import static org.junit.Assert.*; + +@RunWith(AndroidJUnit4.class) +public class LibuplinkInstrumentedTest { + + public static final String VALID_SATELLITE_ADDRESS = InstrumentationRegistry.getArguments().getString("storj.sim.host", "192.168.8.134:10000"); + public static final String VALID_API_KEY = InstrumentationRegistry.getArguments().getString("api.key", "GBK6TEMIPJQUOVVN99C2QO9USKTU26QB6C4VNM0="); + + String filesDir; + + @Before + public void setUp() { + filesDir = InstrumentationRegistry.getTargetContext().getFilesDir().getAbsolutePath(); + } + + @Test + public void testOpenProjectFail() throws Exception { + Config config = new Config(); + + Uplink uplink = new Uplink(config, filesDir); + try { + ProjectOptions options = new ProjectOptions(); + options.setEncryptionKey("TestEncryptionKey".getBytes()); + + Project project = null; + try { + // 10.0.2.2 refers to not existing satellite + project = uplink.openProject("10.0.2.2:1", VALID_API_KEY, options); + fail("exception expected"); + } catch (Exception e) { + // skip + } finally { + if (project != null) { + project.close(); + } + } + } finally { + uplink.close(); + } + } + + @Test + public void testBasicBucket() throws Exception { + Config config = new Config(); + + Uplink uplink = new Uplink(config, filesDir); + try { + ProjectOptions options = new ProjectOptions(); + options.setEncryptionKey("TestEncryptionKey".getBytes()); + + Project project = null; + try { + project = uplink.openProject(VALID_SATELLITE_ADDRESS, VALID_API_KEY, options); + + String expectedBucket = "testBucket"; + project.createBucket(expectedBucket, new BucketConfig()); + BucketInfo bucketInfo = project.getBucketInfo(expectedBucket); + Assert.assertEquals(expectedBucket, bucketInfo.getName()); + + project.deleteBucket(expectedBucket); + + try { + project.getBucketInfo(expectedBucket); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().startsWith("bucket not found")); + } + } finally { + if (project != null) { + project.close(); + } + } + } finally { + uplink.close(); + } + } + + @Test + public void testListBuckets() throws Exception { + Config config = new Config(); + + Uplink uplink = new Uplink(config, filesDir); + try { + ProjectOptions options = new ProjectOptions(); + options.setEncryptionKey("TestEncryptionKey".getBytes()); + + Project project = uplink.openProject(VALID_SATELLITE_ADDRESS, VALID_API_KEY, options); + try { + BucketConfig bucketConfig = new BucketConfig(); + Set expectedBuckets = new HashSet<>(); + for (int i = 0; i < 10; i++) { + String expectedBucket = "testBucket" + i; + project.createBucket(expectedBucket, bucketConfig); + expectedBuckets.add(expectedBucket); + } + + BucketList bucketList = project.listBuckets("", 1, 100); + assertEquals(false, bucketList.more()); + String aa = ""; + for (int i = 0; i < bucketList.length(); i++) { + aa += bucketList.item(i).getName() + "|"; + } + + assertEquals(aa, expectedBuckets.size(), bucketList.length()); + + for (String bucket : expectedBuckets) { + project.deleteBucket(bucket); + } + + bucketList = project.listBuckets("", 1, 100); + assertEquals(false, bucketList.more()); + assertEquals(0, bucketList.length()); + } finally { + project.close(); + } + } finally { + uplink.close(); + } + } + + @Test + public void testUploadDownloadInline() throws Exception { + Config config = new Config(); + + Uplink uplink = new Uplink(config, filesDir); + try { + ProjectOptions options = new ProjectOptions(); + options.setEncryptionKey("TestEncryptionKey".getBytes()); + + Project project = uplink.openProject(VALID_SATELLITE_ADDRESS, VALID_API_KEY, options); + try { + BucketAccess access = new BucketAccess(); + access.setPathEncryptionKey("TestEncryptionKey".getBytes()); + + RedundancyScheme scheme = new RedundancyScheme(); + scheme.setRequiredShares((short) 2); + scheme.setRepairShares((short) 4); + scheme.setOptimalShares((short) 6); + scheme.setTotalShares((short) 8); + + BucketConfig bucketConfig = new BucketConfig(); + bucketConfig.setRedundancyScheme(scheme); + + project.createBucket("test", bucketConfig); + + Bucket bucket = project.openBucket("test", access); + + byte[] expectedData = new byte[1024]; + Random random = new Random(); + random.nextBytes(expectedData); + + { + Writer writer = bucket.newWriter("object/path", new WriterOptions()); + try { + writer.write(expectedData,0, expectedData.length); + } finally { + writer.close(); + } + } + + { + Reader reader = bucket.newReader("object/path", new ReaderOptions()); + try { + ByteArrayOutputStream writer = new ByteArrayOutputStream(); + byte[] buf = new byte[256]; + int read = 0; + while ((read = reader.read(buf)) != -1) { + writer.write(buf, 0, read); + } + assertArrayEquals(writer.toByteArray(), expectedData); + } finally { + reader.close(); + } + } + + bucket.close(); + + project.deleteBucket("test"); + } finally { + project.close(); + } + } finally { + uplink.close(); + } + } + + + @Test + public void testUploadDownloadDeleteRemote() throws Exception { + Config config = new Config(); + + Uplink uplink = new Uplink(config, filesDir); + try { + ProjectOptions options = new ProjectOptions(); + options.setEncryptionKey("TestEncryptionKey".getBytes()); + + Project project = uplink.openProject(VALID_SATELLITE_ADDRESS, VALID_API_KEY, options); + try { + BucketAccess access = new BucketAccess(); + access.setPathEncryptionKey("TestEncryptionKey".getBytes()); + + RedundancyScheme scheme = new RedundancyScheme(); + scheme.setRequiredShares((short) 2); + scheme.setRepairShares((short) 4); + scheme.setOptimalShares((short) 6); + scheme.setTotalShares((short) 8); + + BucketConfig bucketConfig = new BucketConfig(); + bucketConfig.setRedundancyScheme(scheme); + + project.createBucket("test", bucketConfig); + + Bucket bucket = project.openBucket("test", access); + + byte[] expectedData = new byte[1024 * 100]; + Random random = new Random(); + random.nextBytes(expectedData); + { + Writer writer = bucket.newWriter("object/path", new WriterOptions()); + try { + writer.write(expectedData, 0, expectedData.length); + } finally { + writer.close(); + } + } + + { + Reader reader = bucket.newReader("object/path", new ReaderOptions()); + try { + ByteArrayOutputStream writer = new ByteArrayOutputStream(); + byte[] buf = new byte[4096]; + int read = 0; + while ((read = reader.read(buf)) != -1) { + writer.write(buf, 0, read); + } + assertEquals(expectedData.length, writer.size()); + } finally { + reader.close(); + } + } + + bucket.deleteObject("object/path"); + + try { + bucket.deleteObject("object/path"); + } catch (Exception e) { + assertTrue(e.getMessage().startsWith("object not found")); + } + + bucket.close(); + + project.deleteBucket("test"); + } finally { + project.close(); + } + } finally { + uplink.close(); + } + } + + @Test + public void testListObjects() throws Exception { + Config config = new Config(); + + Uplink uplink = new Uplink(config, filesDir); + try { + ProjectOptions options = new ProjectOptions(); + options.setEncryptionKey("TestEncryptionKey".getBytes()); + + Project project = uplink.openProject(VALID_SATELLITE_ADDRESS, VALID_API_KEY, options); + try { + BucketAccess access = new BucketAccess(); + access.setPathEncryptionKey("TestEncryptionKey".getBytes()); + + BucketConfig bucketConfig = new BucketConfig(); + bucketConfig.setRedundancyScheme(new RedundancyScheme()); + + BucketInfo bucketInfo = project.createBucket("testBucket", bucketConfig); + assertEquals("testBucket", bucketInfo.getName()); + + Bucket bucket = project.openBucket("testBucket", access); + + long before = System.currentTimeMillis(); + + for (int i = 0; i < 13; i++) { + Writer writer = bucket.newWriter("path" + i, new WriterOptions()); + try { + byte[] buf = new byte[0]; + writer.write(buf, 0, buf.length); + } finally { + writer.close(); + } + } + + ListOptions listOptions = new ListOptions(); + listOptions.setCursor(""); + listOptions.setDirection(Mobile.DirectionForward); + listOptions.setLimit(20); + + ObjectList list = bucket.listObjects(listOptions); + assertEquals(13, list.length()); + + for (int i = 0; i < list.length(); i++) { + ObjectInfo info = list.item(i); + assertEquals("testBucket", info.getBucket()); + assertTrue(info.getCreated() >= before); + + // cleanup + bucket.deleteObject("path" + i); + } + + bucket.close(); + + project.deleteBucket("testBucket"); + } finally { + project.close(); + } + } finally { + uplink.close(); + } + } +} \ No newline at end of file diff --git a/mobile/libuplink_android/app/src/main/AndroidManifest.xml b/mobile/libuplink_android/app/src/main/AndroidManifest.xml new file mode 100644 index 000000000..11c69ac34 --- /dev/null +++ b/mobile/libuplink_android/app/src/main/AndroidManifest.xml @@ -0,0 +1,14 @@ + + + + + + + diff --git a/mobile/libuplink_android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml b/mobile/libuplink_android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 000000000..1f6bb2906 --- /dev/null +++ b/mobile/libuplink_android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + diff --git a/mobile/libuplink_android/app/src/main/res/drawable/ic_launcher_background.xml b/mobile/libuplink_android/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 000000000..0d025f9bf --- /dev/null +++ b/mobile/libuplink_android/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mobile/libuplink_android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml b/mobile/libuplink_android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml new file mode 100644 index 000000000..eca70cfe5 --- /dev/null +++ b/mobile/libuplink_android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/mobile/libuplink_android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml b/mobile/libuplink_android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml new file mode 100644 index 000000000..eca70cfe5 --- /dev/null +++ b/mobile/libuplink_android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/mobile/libuplink_android/app/src/main/res/mipmap-hdpi/ic_launcher.png b/mobile/libuplink_android/app/src/main/res/mipmap-hdpi/ic_launcher.png new file mode 100644 index 000000000..898f3ed59 Binary files /dev/null and b/mobile/libuplink_android/app/src/main/res/mipmap-hdpi/ic_launcher.png differ diff --git a/mobile/libuplink_android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png b/mobile/libuplink_android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png new file mode 100644 index 000000000..dffca3601 Binary files /dev/null and b/mobile/libuplink_android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png differ diff --git a/mobile/libuplink_android/app/src/main/res/mipmap-mdpi/ic_launcher.png b/mobile/libuplink_android/app/src/main/res/mipmap-mdpi/ic_launcher.png new file mode 100644 index 000000000..64ba76f75 Binary files /dev/null and b/mobile/libuplink_android/app/src/main/res/mipmap-mdpi/ic_launcher.png differ diff --git a/mobile/libuplink_android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png b/mobile/libuplink_android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png new file mode 100644 index 000000000..dae5e0823 Binary files /dev/null and b/mobile/libuplink_android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png differ diff --git a/mobile/libuplink_android/app/src/main/res/mipmap-xhdpi/ic_launcher.png b/mobile/libuplink_android/app/src/main/res/mipmap-xhdpi/ic_launcher.png new file mode 100644 index 000000000..e5ed46597 Binary files /dev/null and b/mobile/libuplink_android/app/src/main/res/mipmap-xhdpi/ic_launcher.png differ diff --git a/mobile/libuplink_android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png b/mobile/libuplink_android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png new file mode 100644 index 000000000..14ed0af35 Binary files /dev/null and b/mobile/libuplink_android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png differ diff --git a/mobile/libuplink_android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png b/mobile/libuplink_android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png new file mode 100644 index 000000000..b0907cac3 Binary files /dev/null and b/mobile/libuplink_android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png differ diff --git a/mobile/libuplink_android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png b/mobile/libuplink_android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png new file mode 100644 index 000000000..d8ae03154 Binary files /dev/null and b/mobile/libuplink_android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png differ diff --git a/mobile/libuplink_android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png b/mobile/libuplink_android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png new file mode 100644 index 000000000..2c18de9e6 Binary files /dev/null and b/mobile/libuplink_android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png differ diff --git a/mobile/libuplink_android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png b/mobile/libuplink_android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png new file mode 100644 index 000000000..beed3cdd2 Binary files /dev/null and b/mobile/libuplink_android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png differ diff --git a/mobile/libuplink_android/app/src/main/res/values/colors.xml b/mobile/libuplink_android/app/src/main/res/values/colors.xml new file mode 100644 index 000000000..69b22338c --- /dev/null +++ b/mobile/libuplink_android/app/src/main/res/values/colors.xml @@ -0,0 +1,6 @@ + + + #008577 + #00574B + #D81B60 + diff --git a/mobile/libuplink_android/app/src/main/res/values/strings.xml b/mobile/libuplink_android/app/src/main/res/values/strings.xml new file mode 100644 index 000000000..826d62999 --- /dev/null +++ b/mobile/libuplink_android/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + libuplink_android + diff --git a/mobile/libuplink_android/app/src/main/res/values/styles.xml b/mobile/libuplink_android/app/src/main/res/values/styles.xml new file mode 100644 index 000000000..5885930df --- /dev/null +++ b/mobile/libuplink_android/app/src/main/res/values/styles.xml @@ -0,0 +1,11 @@ + + + + + + diff --git a/mobile/libuplink_android/build.gradle b/mobile/libuplink_android/build.gradle new file mode 100644 index 000000000..fafc1b970 --- /dev/null +++ b/mobile/libuplink_android/build.gradle @@ -0,0 +1,27 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. + +buildscript { + repositories { + google() + jcenter() + + } + dependencies { + classpath 'com.android.tools.build:gradle:3.4.0' + + // NOTE: Do not place your application dependencies here; they belong + // in the individual module build.gradle files + } +} + +allprojects { + repositories { + google() + jcenter() + + } +} + +task clean(type: Delete) { + delete rootProject.buildDir +} diff --git a/mobile/libuplink_android/gradle.properties b/mobile/libuplink_android/gradle.properties new file mode 100644 index 000000000..82618cecb --- /dev/null +++ b/mobile/libuplink_android/gradle.properties @@ -0,0 +1,15 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx1536m +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true + + diff --git a/mobile/libuplink_android/gradle/wrapper/gradle-wrapper.jar b/mobile/libuplink_android/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..f6b961fd5 Binary files /dev/null and b/mobile/libuplink_android/gradle/wrapper/gradle-wrapper.jar differ diff --git a/mobile/libuplink_android/gradle/wrapper/gradle-wrapper.properties b/mobile/libuplink_android/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..bdbc5d5d5 --- /dev/null +++ b/mobile/libuplink_android/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Tue May 07 14:29:06 CEST 2019 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-all.zip diff --git a/mobile/libuplink_android/gradlew b/mobile/libuplink_android/gradlew new file mode 100755 index 000000000..cccdd3d51 --- /dev/null +++ b/mobile/libuplink_android/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/mobile/libuplink_android/gradlew.bat b/mobile/libuplink_android/gradlew.bat new file mode 100644 index 000000000..e95643d6a --- /dev/null +++ b/mobile/libuplink_android/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/mobile/libuplink_android/settings.gradle b/mobile/libuplink_android/settings.gradle new file mode 100644 index 000000000..e7b4def49 --- /dev/null +++ b/mobile/libuplink_android/settings.gradle @@ -0,0 +1 @@ +include ':app' diff --git a/mobile/object.go b/mobile/object.go new file mode 100644 index 000000000..3e0765f3b --- /dev/null +++ b/mobile/object.go @@ -0,0 +1,96 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package mobile + +import ( + "fmt" + "time" + + libuplink "storj.io/storj/lib/uplink" + "storj.io/storj/pkg/storj" +) + +// ObjectInfo object metadata +type ObjectInfo struct { + Version int32 + Bucket string + Path string + IsPrefix bool + Size int64 + ContentType string + Created int64 + Modified int64 + Expires int64 + + metadata map[string]string +} + +func newObjectInfoFromObject(object storj.Object) *ObjectInfo { + return &ObjectInfo{ + Version: int32(object.Version), + Bucket: object.Bucket.Name, + Path: object.Path, + IsPrefix: object.IsPrefix, + Size: object.Size, + ContentType: object.ContentType, + Created: object.Created.UTC().UnixNano() / int64(time.Millisecond), + Modified: object.Modified.UTC().UnixNano() / int64(time.Millisecond), + Expires: object.Expires.UTC().UnixNano() / int64(time.Millisecond), + metadata: object.Metadata, + } +} + +func newObjectInfoFromObjectMeta(objectMeta libuplink.ObjectMeta) *ObjectInfo { + return &ObjectInfo{ + // TODO ObjectMeta doesn't have Version but storj.Object has + // Version: int32(objectMeta.Version), + Bucket: objectMeta.Bucket, + Path: objectMeta.Path, + IsPrefix: objectMeta.IsPrefix, + Size: objectMeta.Size, + ContentType: objectMeta.ContentType, + Created: objectMeta.Created.UTC().UnixNano() / int64(time.Millisecond), + Modified: objectMeta.Modified.UTC().UnixNano() / int64(time.Millisecond), + Expires: objectMeta.Expires.UTC().UnixNano() / int64(time.Millisecond), + metadata: objectMeta.Metadata, + } +} + +// GetMetadata gets objects custom metadata +func (bl *ObjectInfo) GetMetadata(key string) string { + return bl.metadata[key] +} + +// ObjectList represents list of objects +type ObjectList struct { + list storj.ObjectList +} + +// More returns true if list request was not able to return all results +func (bl *ObjectList) More() bool { + return bl.list.More +} + +// Prefix prefix for objects from list +func (bl *ObjectList) Prefix() string { + return bl.list.Prefix +} + +// Bucket returns bucket name +func (bl *ObjectList) Bucket() string { + return bl.list.Bucket +} + +// Length returns number of returned items +func (bl *ObjectList) Length() int { + return len(bl.list.Items) +} + +// Item gets item from specific index +func (bl *ObjectList) Item(index int) (*ObjectInfo, error) { + if index < 0 && index >= len(bl.list.Items) { + return nil, fmt.Errorf("index out of range") + } + return newObjectInfoFromObject(bl.list.Items[index]), nil +} diff --git a/mobile/scope.go b/mobile/scope.go new file mode 100644 index 000000000..004722bba --- /dev/null +++ b/mobile/scope.go @@ -0,0 +1,31 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package mobile + +import ( + "context" + + "storj.io/storj/internal/fpath" +) + +type scope struct { + ctx context.Context + cancel func() +} + +func rootScope(tempDir string) scope { + ctx := context.Background() + if tempDir == "inmemory" { + ctx = fpath.WithTempData(ctx, "", true) + } else { + ctx = fpath.WithTempData(ctx, tempDir, false) + } + ctx, cancel := context.WithCancel(ctx) + return scope{ctx, cancel} +} + +func (parent *scope) child() scope { + ctx, cancel := context.WithCancel(parent.ctx) + return scope{ctx, cancel} +} diff --git a/mobile/test-libuplink-android.sh b/mobile/test-libuplink-android.sh new file mode 100755 index 000000000..8bc04a75e --- /dev/null +++ b/mobile/test-libuplink-android.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -ueo pipefail + +echo "Executing gomobile bind" + +gomobile bind -target android -o libuplink_android/app/libs/libuplink-android.aar -javapkg io.storj.libuplink storj.io/storj/mobile + +cd libuplink_android + +# Might be easier way than -Pandroid.testInstrumentationRunnerArguments +./gradlew connectedAndroidTest -Pandroid.testInstrumentationRunnerArguments.api.key=$GATEWAY_0_API_KEY -Pandroid.testInstrumentationRunnerArguments.storj.sim.host=$SATELLITE_0_ADDR \ No newline at end of file diff --git a/mobile/test-sim.sh b/mobile/test-sim.sh new file mode 100755 index 000000000..81f53e30a --- /dev/null +++ b/mobile/test-sim.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -ueo pipefail +set +x + +# setup tmpdir for testfiles and cleanup +TMP=$(mktemp -d -t tmp.XXXXXXXXXX) +cleanup(){ + rm -rf "$TMP" +} +trap cleanup EXIT + +export STORJ_NETWORK_DIR=$TMP + +STORJ_NETWORK_HOST4=${STORJ_NETWORK_HOST4:-127.0.0.1} + +# setup the network +storj-sim -x --host $STORJ_NETWORK_HOST4 network setup + +# run tests +storj-sim -x --host $STORJ_NETWORK_HOST4 network test bash test-libuplink-android.sh +storj-sim -x --host $STORJ_NETWORK_HOST4 network destroy \ No newline at end of file diff --git a/mobile/uplink.go b/mobile/uplink.go new file mode 100644 index 000000000..d23fd89a3 --- /dev/null +++ b/mobile/uplink.go @@ -0,0 +1,191 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package mobile + +import ( + "fmt" + + "storj.io/storj/internal/memory" + libuplink "storj.io/storj/lib/uplink" + "storj.io/storj/pkg/storj" +) + +// Config represents configuration options for an Uplink +type Config struct { + + // MaxInlineSize determines whether the uplink will attempt to + // store a new object in the satellite's metainfo. Objects at + // or below this size will be marked for inline storage, and + // objects above this size will not. (The satellite may reject + // the inline storage and require remote storage, still.) + MaxInlineSize int64 + + // MaxMemory is the default maximum amount of memory to be + // allocated for read buffers while performing decodes of + // objects. (This option is overrideable per Bucket if the user + // so desires.) If set to zero, the library default (4 MiB) will + // be used. If set to a negative value, the system will use the + // smallest amount of memory it can. + MaxMemory int64 +} + +// Uplink represents the main entrypoint to Storj V3. An Uplink connects to +// a specific Satellite and caches connections and resources, allowing one to +// create sessions delineated by specific access controls. +type Uplink struct { + scope + lib *libuplink.Uplink +} + +// NewUplink creates a new Uplink. This is the first step to create an uplink +// session with a user specified config or with default config, if nil config. +// Uplink needs also writable temporary directory. +func NewUplink(config *Config, tempDir string) (*Uplink, error) { + scope := rootScope(tempDir) + + cfg := &libuplink.Config{} + if config != nil { + cfg.Volatile.TLS.SkipPeerCAWhitelist = true + cfg.Volatile.MaxInlineSize = memory.Size(config.MaxInlineSize) + cfg.Volatile.MaxMemory = memory.Size(config.MaxMemory) + } + + lib, err := libuplink.NewUplink(scope.ctx, cfg) + if err != nil { + return nil, safeError(err) + } + return &Uplink{scope, lib}, nil +} + +// Close closes the Uplink. This may not do anything at present, but should +// still be called to allow forward compatibility. No Project or Bucket +// objects using this Uplink should be used after calling Close. +func (uplink *Uplink) Close() error { + uplink.cancel() + return safeError(uplink.lib.Close()) +} + +// ProjectOptions allows configuration of various project options during opening +type ProjectOptions struct { + EncryptionKey []byte +} + +// Project represents a specific project access session. +type Project struct { + scope + lib *libuplink.Project +} + +// OpenProject returns a Project handle with the given APIKey +func (uplink *Uplink) OpenProject(satellite string, apikey string, options *ProjectOptions) (*Project, error) { + scope := uplink.scope.child() + + opts := libuplink.ProjectOptions{} + if options != nil { + opts.Volatile.EncryptionKey = &storj.Key{} + copy(opts.Volatile.EncryptionKey[:], options.EncryptionKey) // TODO: error check + } + + key, err := libuplink.ParseAPIKey(apikey) + if err != nil { + return nil, safeError(err) + } + + project, err := uplink.lib.OpenProject(scope.ctx, satellite, key, &opts) + if err != nil { + return nil, safeError(err) + } + + return &Project{scope, project}, nil +} + +// Close closes the Project +func (project *Project) Close() error { + defer project.cancel() + return safeError(project.lib.Close()) +} + +// CreateBucket creates buckets in project +func (project *Project) CreateBucket(bucketName string, opts *BucketConfig) (*BucketInfo, error) { + scope := project.scope.child() + + cfg := libuplink.BucketConfig{} + if opts != nil { + cfg.PathCipher = storj.CipherSuite(opts.PathCipher) + cfg.EncryptionParameters = newStorjEncryptionParameters(opts.EncryptionParameters) + cfg.Volatile.RedundancyScheme = newStorjRedundancyScheme(opts.RedundancyScheme) + cfg.Volatile.SegmentsSize = memory.Size(opts.SegmentsSize) + } + + bucket, err := project.lib.CreateBucket(scope.ctx, bucketName, &cfg) + if err != nil { + return nil, safeError(err) + } + + return newBucketInfo(bucket), nil +} + +// OpenBucket returns a Bucket handle with the given EncryptionAccess +// information. +func (project *Project) OpenBucket(bucketName string, options *BucketAccess) (*Bucket, error) { + scope := project.scope.child() + + opts := libuplink.EncryptionAccess{} + if options != nil { + copy(opts.Key[:], options.PathEncryptionKey) // TODO: error check + opts.EncryptedPathPrefix = options.EncryptedPathPrefix + } + + bucket, err := project.lib.OpenBucket(scope.ctx, bucketName, &opts) + if err != nil { + return nil, safeError(err) + } + + return &Bucket{bucket.Name, scope, bucket}, nil +} + +// GetBucketInfo returns info about the requested bucket if authorized. +func (project *Project) GetBucketInfo(bucketName string) (*BucketInfo, error) { + scope := project.scope.child() + + bucket, _, err := project.lib.GetBucketInfo(scope.ctx, bucketName) + if err != nil { + return nil, safeError(err) + } + + return newBucketInfo(bucket), nil +} + +// ListBuckets will list authorized buckets. +func (project *Project) ListBuckets(cursor string, direction, limit int) (*BucketList, error) { + scope := project.scope.child() + opts := libuplink.BucketListOptions{ + Cursor: cursor, + Direction: storj.ListDirection(direction), + Limit: limit, + } + list, err := project.lib.ListBuckets(scope.ctx, &opts) + if err != nil { + return nil, safeError(err) + } + + return &BucketList{list}, nil +} + +// DeleteBucket deletes a bucket if authorized. If the bucket contains any +// Objects at the time of deletion, they may be lost permanently. +func (project *Project) DeleteBucket(bucketName string) error { + scope := project.scope.child() + + err := project.lib.DeleteBucket(scope.ctx, bucketName) + return safeError(err) +} + +func safeError(err error) error { + // workaround to avoid gomobile panic because of "hash of unhashable type errs.combinedError" + if err == nil { + return nil + } + return fmt.Errorf("%v", err.Error()) +} diff --git a/pkg/eestream/encode.go b/pkg/eestream/encode.go index fa1fb1ac3..d764c2452 100644 --- a/pkg/eestream/encode.go +++ b/pkg/eestream/encode.go @@ -12,6 +12,8 @@ import ( "github.com/vivint/infectious" "go.uber.org/zap" + "storj.io/storj/internal/fpath" + "storj.io/storj/internal/memory" "storj.io/storj/internal/readcloser" "storj.io/storj/internal/sync2" "storj.io/storj/pkg/encryption" @@ -123,13 +125,25 @@ type encodedReader struct { // EncodeReader takes a Reader and a RedundancyStrategy and returns a slice of // io.ReadClosers. -func EncodeReader(ctx context.Context, r io.Reader, rs RedundancyStrategy) ([]io.ReadCloser, error) { +func EncodeReader(ctx context.Context, r io.Reader, rs RedundancyStrategy) (_ []io.ReadCloser, err error) { er := &encodedReader{ rs: rs, pieces: make(map[int]*encodedPiece, rs.TotalCount()), } - pipeReaders, pipeWriter, err := sync2.NewTeeFile(rs.TotalCount(), os.TempDir()) + var pipeReaders []sync2.PipeReader + var pipeWriter sync2.PipeWriter + + tempDir, inmemory, _ := fpath.GetTempData(ctx) + if inmemory { + // TODO what default inmemory size will be enough + pipeReaders, pipeWriter, err = sync2.NewTeeInmemory(rs.TotalCount(), memory.MiB.Int64()) + } else { + if tempDir == "" { + tempDir = os.TempDir() + } + pipeReaders, pipeWriter, err = sync2.NewTeeFile(rs.TotalCount(), tempDir) + } if err != nil { return nil, err }