From a264a4422b3a902f179e330d066c9fc6ad10d6ae Mon Sep 17 00:00:00 2001 From: Kaloyan Raev Date: Tue, 30 Mar 2021 17:06:09 +0300 Subject: [PATCH] cmd: remove metainfo-migrator and metainfo-verifier We already merged the multipart-upload branch to main. These two tools make only sense if we are migrating a satellite from Pointer DB to Metabase. There is one remaining satellite to migrate, but these tools should be used from the respective release branch instead of from main. Removing these tools from main will: 1) Avoid the mistake to use them from the main branch instead of from the respective release branch. 2) Allow to finally remove any code related to the old Pointer DB. Change-Id: Ied66098c5d0b8fefeb5d6e92b5e0ef5c6603df5d --- Makefile | 2 +- cmd/metainfo-migration/fastpb/gogo.proto | 143 - cmd/metainfo-migration/fastpb/pointer_test.go | 57 - cmd/metainfo-migration/fastpb/pointerdb.pb.go | 2403 ----------------- cmd/metainfo-migration/fastpb/pointerdb.proto | 77 - cmd/metainfo-migration/fastpb/types.go | 17 - cmd/metainfo-migration/main.go | 83 - cmd/metainfo-migration/migrator.go | 705 ----- cmd/metainfo-migration/migrator_test.go | 293 -- cmd/metainfo-verifier/main.go | 73 - cmd/metainfo-verifier/verifier.go | 297 -- go.sum | 1 - scripts/test-sim-backwards.sh | 4 - .../test-sim-rolling-upgrade.sh | 4 - 14 files changed, 1 insertion(+), 4158 deletions(-) delete mode 100644 cmd/metainfo-migration/fastpb/gogo.proto delete mode 100644 cmd/metainfo-migration/fastpb/pointer_test.go delete mode 100644 cmd/metainfo-migration/fastpb/pointerdb.pb.go delete mode 100644 cmd/metainfo-migration/fastpb/pointerdb.proto delete mode 100644 cmd/metainfo-migration/fastpb/types.go delete mode 100644 cmd/metainfo-migration/main.go delete mode 100644 cmd/metainfo-migration/migrator.go delete mode 100644 cmd/metainfo-migration/migrator_test.go delete mode 100644 cmd/metainfo-verifier/main.go delete mode 100644 cmd/metainfo-verifier/verifier.go diff --git a/Makefile b/Makefile index 1979ff357..bf68350c6 100644 --- a/Makefile +++ b/Makefile @@ -379,4 +379,4 @@ bump-dependencies: go mod tidy update-proto-lock: - protolock commit --ignore "satellite/internalpb,storagenode/internalpb,cmd/metainfo-migration/fastpb" \ No newline at end of file + protolock commit --ignore "satellite/internalpb,storagenode/internalpb" \ No newline at end of file diff --git a/cmd/metainfo-migration/fastpb/gogo.proto b/cmd/metainfo-migration/fastpb/gogo.proto deleted file mode 100644 index 937487bf8..000000000 --- a/cmd/metainfo-migration/fastpb/gogo.proto +++ /dev/null @@ -1,143 +0,0 @@ -// Protocol Buffers for Go with Gadgets -// -// Copyright (c) 2013, The GoGo Authors. All rights reserved. -// http://github.com/gogo/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto2"; -package gogoproto; - -import "google/protobuf/descriptor.proto"; - -option java_package = "com.google.protobuf"; -option java_outer_classname = "GoGoProtos"; - -extend google.protobuf.EnumOptions { - optional bool goproto_enum_prefix = 62001; - optional bool goproto_enum_stringer = 62021; - optional bool enum_stringer = 62022; - optional string enum_customname = 62023; - optional bool enumdecl = 62024; -} - -extend google.protobuf.EnumValueOptions { - optional string enumvalue_customname = 66001; -} - -extend google.protobuf.FileOptions { - optional bool goproto_getters_all = 63001; - optional bool goproto_enum_prefix_all = 63002; - optional bool goproto_stringer_all = 63003; - optional bool verbose_equal_all = 63004; - optional bool face_all = 63005; - optional bool gostring_all = 63006; - optional bool populate_all = 63007; - optional bool stringer_all = 63008; - optional bool onlyone_all = 63009; - - optional bool equal_all = 63013; - optional bool description_all = 63014; - optional bool testgen_all = 63015; - optional bool benchgen_all = 63016; - optional bool marshaler_all = 63017; - optional bool unmarshaler_all = 63018; - optional bool stable_marshaler_all = 63019; - - optional bool sizer_all = 63020; - - optional bool goproto_enum_stringer_all = 63021; - optional bool enum_stringer_all = 63022; - - optional bool unsafe_marshaler_all = 63023; - optional bool unsafe_unmarshaler_all = 63024; - - optional bool goproto_extensions_map_all = 63025; - optional bool goproto_unrecognized_all = 63026; - optional bool gogoproto_import = 63027; - optional bool protosizer_all = 63028; - optional bool compare_all = 63029; - optional bool typedecl_all = 63030; - optional bool enumdecl_all = 63031; - - optional bool goproto_registration = 63032; - optional bool messagename_all = 63033; - - optional bool goproto_sizecache_all = 63034; - optional bool goproto_unkeyed_all = 63035; -} - -extend google.protobuf.MessageOptions { - optional bool goproto_getters = 64001; - optional bool goproto_stringer = 64003; - optional bool verbose_equal = 64004; - optional bool face = 64005; - optional bool gostring = 64006; - optional bool populate = 64007; - optional bool stringer = 67008; - optional bool onlyone = 64009; - - optional bool equal = 64013; - optional bool description = 64014; - optional bool testgen = 64015; - optional bool benchgen = 64016; - optional bool marshaler = 64017; - optional bool unmarshaler = 64018; - optional bool stable_marshaler = 64019; - - optional bool sizer = 64020; - - optional bool unsafe_marshaler = 64023; - optional bool unsafe_unmarshaler = 64024; - - optional bool goproto_extensions_map = 64025; - optional bool goproto_unrecognized = 64026; - - optional bool protosizer = 64028; - - optional bool typedecl = 64030; - - optional bool messagename = 64033; - - optional bool goproto_sizecache = 64034; - optional bool goproto_unkeyed = 64035; -} - -extend google.protobuf.FieldOptions { - optional bool nullable = 65001; - optional bool embed = 65002; - optional string customtype = 65003; - optional string customname = 65004; - optional string jsontag = 65005; - optional string moretags = 65006; - optional string casttype = 65007; - optional string castkey = 65008; - optional string castvalue = 65009; - - optional bool stdtime = 65010; - optional bool stdduration = 65011; - optional bool wktpointer = 65012; - optional bool compare = 65013; - -} \ No newline at end of file diff --git a/cmd/metainfo-migration/fastpb/pointer_test.go b/cmd/metainfo-migration/fastpb/pointer_test.go deleted file mode 100644 index 643bb9c84..000000000 --- a/cmd/metainfo-migration/fastpb/pointer_test.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (C) 2021 Storj Labs, Inc. -// See LICENSE for copying information. - -package fastpb_test - -import ( - "encoding/hex" - "testing" - - proto "github.com/gogo/protobuf/proto" - - "storj.io/common/pb" - "storj.io/storj/cmd/metainfo-migration/fastpb" -) - -var testRemoteSegment, _ = hex.DecodeString("080122bc1c0a08101e1864203c2850122000000000000000000000000000000000000000000000000000000000000000001a2212204f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64981855ad8681d001a22122086d1e91e00167939cb6694d2c422acd208a0072939487f6999eb9d18a44784001a2212205d87f3c67cf22746e995af5a25367951baa2ff6cd471c483f15fb90badb37c001a22122021b6d95526a41a9504680b4e7c8b763a1b1d49d4955c8486216325253fec73001a221220d7a9e28bf921119c160f0702448615bbda08313f6a8eb668d20bf505987592001a221220668a5bdf2c7fc4844592d2572bcd0668d2d6c52f5054e2d0836bf84c7174cb001a22122076364cc3dbd968b0f7172ed85794bb358b0c3b525da1786f9fff094279db19001a221220ebd7a19d0f7bbacbe0255aa5b7d44bec40f84c892b9bffd43629b0223beea5001a221220f74391f445d15afd4294040374f6924b98cbf8713f8d962d7c8d019192c242001a221220e2cafccae3a61fb586b14323a6bc8f9e7df1d929333ff993933bea6f5b3af6001a2212200374366c4719e43a1b067d89bc7f01f1f573981659a44ff17a4c7215a3b539001a2212201e5849c6077dbb5722f5717a289a266f97647981998ebea89c0b4b37397011001a22122082ed6f4125c8fa7311e4d7defa922daae7786667f7e936cd4f24abf7df866b001a22122056038367ad6145de1ee8f4a8b0993ebdf8883a0ad8be9c3978b04883e56a15001a2212208de563afa467d49dec6a40e9a1d007f033c2823061bdd0eaa59f8e4da64301001a221220220d0b29688b734b8ea0f3ca9936e8461f10d77c96ea80a7a665f606f6a63b001a2212203dfd2567c18979e4d60f26686d9bf2fb26c901ff354cde1607ee294b39f32b001a2212207822ba64f84ab43ca0c6e6b91c1fd3be8990434179d3af4491a369012db92d001a2212204fc39d1734ff5716428953bb6865fcf92b0c3a17c9028be9914eb7649c6c93001a221220800979d1830356f2a54c3deab2a4b4475d63afbe8fb56987c77f5818526f18001a221220be823350eab13935f31d84484517e924aef78ae151c00755925836b7075885001a2212200c30ec29a3703934bf50a28da102975deda77e758579ea3dfe4136abf752b3001a221220271d03e944b3c9db366b75045f8efd69d22ae5411947cb553d7694267aef4e001a221220ea406b32d6108bd68584f57e37caac6e33feaa3263a399437024ba9c9b1467001a221220274f01a910ae295f6efbfe5f5abf44ccde263b5606633e2bf0006f28295d7d001a221220069f01a239c4365854c3af7f6b41d631f92b9a8d12f41257325fff332f7576001a221220620556304a3e3eae14c28d0cea39d2901a52720da85ca1e4b38eaf3f44c6c6001a2212208362f2f54fc00e09d6fc25640854c15dfcacaa8a2cecce5a3aba53ab705b18001a22122094b4d338a5143e63408d8724b0cf3fae17a3f79be1072fb63c35d6042c4160001a2212208ee9e2a9f3fb4ffb0019b454d522b5ffa17604193fb8966710a7960732ca52001a22122053c3f520c889b79bf504cfb57c7601232d589baccea9d6e263e25c27741d3f001a22122062cbbb15d9afbcbf7f7da41ab0408e3969c2e2cdcf233438bf1774ace7709a001a221220091e9a83fdeae0ec55eb233a9b5394cb3c7856b546d313c8a3b4c1c0e05447001a221220ba370eb36dbcfdec90b302dcdc3b9ef522e2a6f1ed0afec1f8e20faabedf6b001a2212202e717d3a748a58677a0c56348f8921a266b11d0f334c62fe52ba53af19779c001a221220948b6570ffa0b773963c130ad797ddeafe4e3ad29b5125210f0ef1c314090f001a221220c79a6f571c246f3e9ac0b7413ef110bd58b00ce73bff706f7ff4b6f44090a3001a22122011f3208e4e4b89cb5165ce64002cbd9c2887aa113df2468928d5a23b9ca740001a2212200c9382d9c6034ad2960c796503e1ce221725f50caf1fbfe831b10b7bf5b15c001a221220a53dbf8e7dcafc9e138647a4b44ed4bce964ed47f74aa594468ced323cb76f001a2212203fac476c9fb03fc9228fbae88fd580663a0454b68312207f0a3b584c623164001a221220b49753b5d5027ce15a4f0a58250d8fb50e77f2bf4f0152e5d49435807f9d4b001a221220be6fb77970466a5626fe33408cf9e88e2c797408a32d29416baf206a329cff001a2212204a75e498320982c85aad70384859c05a4b13a1d5b2f5bfef5a6ed92da482ca001a221220568e5b6fe9d8a9ddd9eb09277b92cef9046efa18500944cbe800a0b1527ea6001a22122029a861d2f6497a3235c37f4192779ec1d96b3b1c5424fce0b727b03072e641001a221220761f03abaa40abc9448fddeb2191d945c04767af847afd0edb5d8857b799ac001a2212208e4affabe3037ffe7fa68aa8af5e39cc416e734d373c5ebebc9cdcc595bcce001a2212207bd3d8df93fab7e125ddebafe65a31bd5d41e2d2ce9c2b17892f0fea1931a2001a221220220777a93143dfdcbfa68406e877073ff08834e197a4034aa48afa3f85b8a6001a22122008caebbac880b5b89b93da53810164402104e648b6226a1b78021851f5d9ac001a221220313a89ddfc454c5f8f72ac89b38b19f53784c19e9beac03c875a27db029de3001a221220e37a42318813487685929359ca8c5eb94e152dc1af42ea3d1676c1bdd19ab8001a221220925c6daee4de5ef9f9dcf08dfcbd02b80809398585928a0f7de50be1a6dc1d001a22122068e8537988fddce562e9b948c918bba3e933e5c400cde5e60c5ead6fc7ae77001a2212201d259b188a4b21c86fbc23d728b45347eada650af24c56d0800a8691332088001a22122005bd55c446e25eb07590bafcccbec6177536401d9a2b7f512b54bfc9d00532001a221220f5aaa7c3a96bc59b489f77d9042c5bce26b163defde5ee6a0fbb3e9346cef8001a2212200ae9515ef30fa47a364e75aea9e111d596e685a591121966e031650d510354001a221220845580ff560760fd36514ca197c875f1d02d9216eba7627e2398322eb5cf43001a2212202bd2e5b887d4630fb8d4747ead6eb82acd1c5b078143ee26a586ad23139d50001a221220723470bf24a865837c9123461c41f5ff99aa99ce24eb4d788576e3336e6549001a22122022558fdf297b9fa007864bafd7cd4ca1b2fb5766ab431a032b72b9a7e937ed001a2212208d0801f29055d3090d2463718254f9442483c7b98b938045da519843854b0e001a221220f7ba951a493f321f0966603022c1dfc579b99ed9d20d573ad53171c8fef7f1001a221220e4613bb365b2ebb44f0ffb6907136385cdc838f0bdd4c812f042577410aca0001a221220c2afbc4c79c62572e20f8ed94ee62b4de7aa1cc84c887e1f7c31e927dfe52a001a2212208f46627eb5d3a4fe16fafce23623e196c9dfff7fbaff4ffe94f4589733e563001a2212209d3045aad3e226488ac02cca4291aed169dce5039d6ab00e40f67aab29332d001a221220448b35507c7c8a09c4db07105dc31003620405da3b2169f5a910c9d0096e5e001a221220f1b570680746acd0cc7760331b663138d6d342b051b5df410637cf7aee9b0c001a22122010a8f9980630f34ce001c0ab7ac65e502d39b216cbc50e73a32eaf936401e2001a2212206bd8b82c30d346bc4b2fa319f245a8657ec122eaf4ad5425c249ee160e17b9001a22122041c2aee5df820ac85de3f8e784870fd87a36cc0d163833df636613a9cc9474001a221220b6592835b9f6f4f8c0e70dbeebae7b14cdb9bc41033aa5baf40d45e24d72ea001a221220a28e3ca030c9937ab8409a7cbf05ae21f97425254543d94d115900b90ae703001a2212207d9856d2441d14ba49a677de8b18cb454b99ddd9daa7ccbb7500dae4e2e5df001a221220f3859ebddada6745fba6a04c5c37c7ca35036f11732ce8bc27b48868611fc7001a22122082a491bfabd7a19df50fdc78a55dbbc2fd37f9296566557fab885b039f30e7001a221220f0cd5961e19b642221db44a69497b8ad99408fe1e037c68bf7c5e5de1d2c68001a2212202348ec1189fb2e36973cef09ff14be23922801f6eaee41409158b45f2dec82001a2212207caaba160cd640ff73495fe4a05ce1202ca7287ed3235b95e69f571fa5e656001a221220a51fae1ebdd7aa6269c2ec7f4057b33593bc84888c970fd528d4a99a1eab9d001a22122020134537cd6d02282e0981e140232a4a87383a21d1845c408ad75704381303001a2212200bd5a30dcca6e3aa2df04715d879279a96879a4f3690ac2025a60c7db15e05001a221220ebc34b734355fe4a059bd3899d920e95f1c46d432f9b08e64d7f9b38965d5a001a221220a7ac183c3833e1a3425ead69d4f975012fd1a49ed832f69e6e9c63b453ec04001a2212209e7a5cf944232d10353f64434abae060f6506ad3fdb1f4415b0af9ce8c208b001a2212200ee526741539fa3203c77ecba410fd6718f227e0b430f9bcb049a3d38540dc001a2212202969120ce80f2007cd42a708a721aa29987b45d4e428811984ecad349cc35d001a2212203515cefe0b002cee5e71c47935e281ebfc4b8b652b69ccb092e55a20f1b9f9001a221220046296124621928739a86671cc180152b953e3bf9d19f825c3dd54ae1688e4001a221220fb5efe65dcdad34bc860010e7c8c997cd5f9e320ca7d39d4ba801a175b1c76001a22122057832f3f36d7d893e216e4c7bbdb548d0ba48449330027368b34f9c69776b4001a2212201532da1c5be68ef4eebe8cb8fa7dc5483fb70c2c896334cb1f9cb5dfe044fa001a2212206197ff5dfd02f2ba3884c53dd718c8560da743a8e9d4aeae20ccef002d82ca001a2212202592b8d8f2a8df3b0c35f15b9b370dca80d4ca8e9a133eb52094f2dd5c0873001a22122052315d828846e37df68fd10658b480f2ac84233633957e688e924ffe3713b5001a22122076fd8a56da8bb07daa8eb4eb8f7334f99256e2766a4109150eed424f0f7435001a221220cdea66e5baaa03edc918e8305bb19fc0c6b4ddb4aa3886cb5090940fc6d4ca00288080c002320c08ccc1c5800610e8c7c7db023a0c08ccf9af8f0610e8c7c7db0242800266c691e4d52de51ccd3c064e174eb4b95d1df0585b8834ddf59e360f4b09d99616ae60c0d0708a1649b9872c30bc404e8723a0f1af775a92cb5bafcc6c301fac702c8e8f196b8f6d0c1e58aa2e46b8cce73075c9962c1152fe0f8f67da8199ffc35e63d2c58d641610e63ee297b3d6cae7ebedc0ae2c0ce5bfc9550891af441d123add3bd9c4ae982a7e93afb309c743384e9fc5492d598685893f5cfcce9acf2e389160182a614f787211b97f9e1715e685d22c950959c0fbd1d4732437342d1e3106c899f72ba41f106fe79bcd25992c705c5bb98c9cddfe98f7a434e83fe795a6a550e4f9e6405f7319304bcadca9dd9feb8dcef5409e47c65d8a3a08ed244a0c08ccc1c5800610e8c7c7db02500a") -var testInlineSegment, _ = hex.DecodeString("1a8020518f1b1f62d06f36b31bdfc56d8351a980a8a9568e8b0a1e21263f77c62aa3fc8746080a2b831a52d538c32b35927fc844304a46513d152d10feeb0bab49c63002451524ba37db776241e03404fa1a9fec370ad50822ebc6d4c12eb42356a9302b8e2b482e4388c1cdcf510c345d219e52bd8eacec0998557e595717f554cbd087c715e8ba9d50cfa4343fcd0d69f390cc58473d8a39dbd17f4ee5276575fd521de9db8ca595374595ac32b98b8c4da7899a554384934ef69f1fbb53773a02335deafe30222ecb47c2ee45659c207a72fb1e817b647a49a86f84f3a6c21d33e7c58f4529fd3243538ecac78866191bf357b770adfe802374d0bd0c42792a96cfc30c34183ba20a3e2ae949839fc29261f611a922d8dded9f2cc3817218e476264d8b7ff980277e62a1c8c1681afb7588a799ae6d52133835a34b74dfcae6b5a1e977abb6b2678912134c17cada44d1aca9500ca428097e17e383c098c4065186b6b1561310c9517192667cb01ad973c86183d3dd07257b771c1044b1d59168e146a8c1164bd3ccad80260dfbceb74533425b2b0de961a5a39271ede2386e0e6c44d97b1adc6d9f46b052072041ba0672706f1d3fa7e1402b68e1c10f5bae10295564d20fe5b46096cdfa0483de9f1f5574c4c91451888dbe1f5c6f0bdac2275ffe01e34ac7a5cea2d03d1b20c069def374d2cd967c8f22621d7e233619c0568c1ed5d3c3fbcad1d5a11df5691b9bd56156d4d039725de88e31dc10a4e80a6a47185f6d9ba66731fdff9f358beb904792dbc41fe16dde4be3cb459b1bcd5b5afbd2f23a0e906ead745c9f752dc60029b826a739e8c05ed4152f33b3c736374f8d8b660fdde82a9358301f0de2fa74e5e1a62f24bd7ec324b6eb0d4aac8b5fddc2f02346786ecb3e76b0c39a5c2f1c7b0ef33cdfba6cdd1e1a32c7cebe63376e0983174f3fb620d8b1304fda5b0292fd13aa05b3026a4f0e5ee15909ffdce7c26d85bc07d0d8a4a770f135d26faa0aeef818fbd1192dd49397cc3b5ae74bc615f4faa76233c6a71c91eed1944f8025316d231521b660e21fb78c56e85d2e54071140c2c2d237d0ce7f247ee67ab0e3500ac60ad02a6511ddf458bcbe19bcbb4718534fd2d9e57eaf2992812bd6b0db16bdee7f8cfd0d4fb8e6a014e3ab0075e5920dc94b347eed7e058f552a3039adb0cdc8d0b1cbc70d233c0717bd10c83ecf9dfbf9c7d8a0769641a9b64b9d01e2e951ac1e50bcf192d11d627a84cf2c1d43b58c8ff23aecd9639c9d4a9bedaac8880b6be1656def487adc8d5d806e8c1d7c8a55a5a72225a5004fcb5ba6a9bfe9dc63677921d73ebd16d3ad0fe51b55291d9347daf7e3ebbdb05bfd721013bb341791d89d3fb4ca5227ad2609a34c095275d248cc039e44613b6b7c2c7bd5506596e827314df994f2cbac5c15512cd345dbdcdbb1d804a6a7819f96b93886257a9c7a09304ca507149c19835216fdeff5e1c3abda3e28dca43f69c5aeec5750d18d76ea381ef0f9d783f8c0ee3d53514f85913cd504b4b06ee34e072aac43cd83efc56941acdc98fd478f141c65381025f143745e5981b856e97441ee706b2ce4b5b39ffee7ba5a152786d8265077b20f8f9b7175f7f3a141de3275c279c152354f9a4570e378f05b92b41b2a9c477db49062aefddf118279368a09b7accc6e660658c9edd04e9e0b87af4442593e5e6c5afeeafa53c7a60ad4fa64d4ac8161198c96429b02cdbfeef0375200d91e1cd8e9dfae7b685b2915566258c21bbafc658d7dc5df7484c514608167fe2478dc80b8849080d574e3abfc68c7f6d034e0f5a7e31a23cd16362c08d1bbedd2a7e90cedc8b6a2a808988ddef3448e8fb2dd9364be15ca44f21ed5fea3646462238d5dc19f30402411c477c16da3bd626844189ed6a393efe5466c16e1a0fb2f028ab6bed7399827bb76769c0c89cdc0a1aa80da1eb4e76a2a565299a9d289141a0f3b871ad0f587dd97f523ec1b3825c2867c46d00b4cf7a3053a8f518a2b25bd23d1906998f798b09f1f813ad357cf956d7e8023405fd53545b54f27330e1d91b2e8398ebd42ab12a471e51c7dcbf3051aa1c19cb13a9ea02a5e2ee9bb8735823a55ad481b5e7d77228fb15ffafcf2c908aaa87599a68f1bb27a9bda4884a99730f0406ebd56d13d80ea561d4447b441809ac5b84b5aded9770d6cc0dcf81e068337d1e8bda7fdaa3fe24ad927ef3fd16adf586928c6986ef2892a6dba7b1e066c23e1737dbc26faeb469eaabb6ccac290f364a9877ac0d166494a0cb56b28307d6596536a58d38ba0379e9e4c20c9ac5389f9732f427a346f821ebf4e8444f2d3dd25a59b108ad45341b5fd7b759b03ab399c9c2d63790e711ace2f874c4843c40d63dc8b722f3280f3db6fa2b16050ef1d1164a46f031940cb2f8d626a5aecf5c4d60ef04ae3e84da5b988f87e240ce75aea5d72ce46488ea91ef209353fe5b661cc93ce95c002435cb76b96efe98694a524417fa853f35e7311a67e303e4d4643c7c8bf9298f178fe574bd342cd2d9bc5c11b70950db4a699a0b16ae826a084bc9579f768098977f4e897029ccdc617768fcec887de42cf555b32de5861fa26ffe0d6175d3f80841eb4f897b81557ba1b325ed6c4abb4ede51be89d9ceec0f4b1f74c8e9bdbaf3056560f43372350c67e2cf5a8965cf3604e214486cbdd47909d37b721667cf26da399b4cbadfa0f0521ebeb72e11d06116d51e6a2793924a9cf4024b921c8818d918b7a376992c8ac0ca37f21cff97ccebcca500743fa3fcfb62afffe466afaae5bb73d661bd1ab79f2d5ffe772b28f776f6406ad6342bc05f6764d58dfb4abe43ca75f6c54e0bf96de6c2f2393aa19e5d46a3f3f952a4fa753da5d9418a5734c12d800c53a35ec927d031a66bb84473115424cd58febdd1c07b6f31a0903f552127788df9be7eee2adaccbe2844b96e1d03144edc7fa5e8fe35159c6a4beb45bc830368c5f23979fc6c2e54a57e754c1468792420079efd9f5d3a1d7b612031f9a45443c706f0d9f02446bc0d67bad5c6ead170c60a5597e4543f2887e978f4ff4edf5729c85313d9cab23dca4d5fc7783c44f1784481bb9923bb30218126d084753c23e67f168bda1a3ed304e8091d24e96e059c8db2c86a3191a02df39bde474b43e963ab465e4a27ce43e254d1450a283dd3cf5657c5c1d9efa0356dd2ec2edac8b29a561a1b016355be0411b1006756dcac7f01a7c1a270e6ee3dfdaa0038224df5a98e9bf6b366e60da8ba3bd6cf767c6da528f5596661f13c1f4ec2f492a728627e85b9227ffbde81a25a2daf1f4987ed735c26972173646236b9be64a378c9ee391a089aba5abb88bf86a30096c6fa2e97b53faa91f8e6c372c95c78908e8665c3b2cacfe07156e062d8d2c6dc36abac501ae2d60e572f6677d9645248e34507df5a256f970940742939f0c248ee5127eebe61a2d5c89f6805ed4f29d704585b2f76a169d73eb8f75b9d2b0099ce0941049df3e20a14347eaff38d8a6e4d37b45b3fa9c94312d7fb96da5bb1a8e65ee483645c54412dec21dcdeff6141e6333cab7c77a1b79845b2713557ffdd44760496b2271fd16ca08c13c0f25a54ef356abb8ea5734f23ca15deb70ba32b3b601fb42dab4d2704ac71e011d43d8eb30c1610df799882b0a453f0d550ce258b6e016f4bd1aadbe395eb42e0e0fa140d87bf3ba8485001da6a80bbda9f54156428c09c9989a3acd52db1ebfe1b0efc469d6c137e6bd5b56eae1e7d3ed0191fa8a509c17e0896b8c98077bcb72dad96e0761bb714277060498df91c71e2c67bb30aa0b45d1ab4fb933d7713108d7335ee90196c34d8745f8dba1285045c7b512344ef05bf42bc8554c0b41cbe54f2bd25cd4847ec46ddbd1c5cae737e503685df85dcac80618f3c6789bfb0992279ae5bdd5e88b6a86aa21c10cc10e591e2124eda0f893d9e0c811f20aaa931942b8ac8cc47768b8b7bc767a6951929cb971ee92411a843bc0b5164c45b54da04409a52de86e79d13d8e735216191e757912d43940c16f7acf88eb169a0c72f26a3e1350e3cd872b17ba873b373e1c7d712d04aff6b192904c1880f95553b746e71b3f889b1748bc228a9812cbbade2a36ba76cc7167d79bfa0f075d256c568cccbcd759a2aa45ebbf3dcd9539fc47953d82dcf1ae3aecc09d96c4fcd567344d04b6ee25c569c421b0ad79917e8b0730fb5ffe45dc5ffa867ab332ae208bc30dbf1b5b63db5f688fd71e67d4357d979b6352b348a851516c795cf29d5a762b5bc498ef82cf1c3ef5bf1a535a80772d65d7c6b98c9bf95e794f99f1020579dbef3d6a8ddd47cf3012e09ddce22d175c6cb4d4ce85e285702938ee9389be4513f4cb6beb6ab3cebf3c87d71993f46c543bf41715404bea94b0a4570be06be8d5b655c006181cd0251a0f03a613445a5f611cfa90e4b4fea1e1cc4d24a338efbc6b3fb23ad17e686fed6f400e560ff9665d05d0ad0f772bed2f878e5b283a103a6f832f8a1639ac6bb50e338e7bb8867ab1c1016050402aab4a4ad1bbb61a1a2a54fdca83136c9e1ed31dc9e3b9e78ffa23ab769145299af9a80169aa8c351f1fe388df7921ef7e69949f96806c42a937e88dbc102603767a0539524ff806f6e12be69decf9ebace768c1eb0abbcef0e3b12dcd0121a36501887839139cb12f7fa33f6c37e16b4b0e5b34c01068597d8459759fa857548243c228c91a11f424f982831e2494fe3b81b93a345c364e30dfa76d38b273e2ada2a120554bf4e15207e28a921cd2742333242b19f2830fba90684ecc1108980bd558e05a5f2be4025bcab88e71a933a4c112ddb3fa70090fda9474f5f10debaf0f643c43eb8fc72cb247d912dc0ba91f2785128e29dc4ae26ca6539f3a4d7866cdfab5a5fda85e30205990096bc6c212db8e048ff12ef1a316a05650fb8be3c4846a2774aed4a798063d7e32a3ab419df9c8603b087631d196d581e71843d6d37af3ca7ddd28b0e75d4e2bb1baa401c88015fce9bd6cc33b703a1b50f0f33dcf2cdc6d8bd5c7ea168de9482b806ba00213be68418b475ddf4bcff757c5fef748bd30646b0dcb7c124a6fc89f7128c8cb9eb691d7fd5f3d1cb777cbefcba8398a9f6c5492eb49133108f35c14f14b2c66e8557d2c7128fdda18c0bc321715abed2bdd9039bac87395592579251ee5d221f204f4b5206b2bede765b7716745696194e4fbacac0e1381e8753548a7d49e8ec6fca062dcf57f2a55e03bde79eeda02b143c290eef695f255da99d1f1e416353cd100b08475996cc9220c8633a4bc6b23669f03095db245b03fd0e087c06faeddbc330f34dfd2ed2f68901dac2fbedf3669cb1432c37b89d189a8b527d6527a7ac6e58a40ecb753af267476d4e6c6eae91028c800a28f2a50c5bba5d448809bc70a1f5b7d8d5b8cd333e752e46e289f28f634c19c4f8dc70d6b1a6d1318ea57c2155055a8f7ec3433f44d26301608f6a1519e84af7f7f84a55285b5736d25b948281c41581bc312bb4d146833908765ea89fe74a09dc5d4629a659b66402f682e1382f116c94d870f29b76e114c4dd139f914ceec7a3d1a826892a9161e0294fda808dfd7097cfb9c332868e73cdadd7600269537efdc38355504fac87eff99e9d587de004ea0dcbae095d8fcf634e363b0819be47e45d776fc27b72bb89b8874e25200b29b88c1d41a2a5101fe513b2e1b301b71e93fad9ef362d7cc22e78716edea0efc38c48ae8caa78ca66d3c45c5e44c8250cc93288080c002320c08ccc1c5800610a483e8db023a0c08ccf9af8f0610a483e8db02428002d67ae2df803d3655c62d4238c2e8ae4f9f824eb0aa7ced0ed37278441fbb30a0d5c76ae63d249f480dae98bce27e7dd15f25b2780b7a07f06d05dd209b4aefa464af5e8fe48fd5dce031fd8e0acc6816aa0642be6d6f4c7a0e736ab311c1e1848f51abeafecdba3ed72607b6d35a4e0281b42bdd23197a9e1a8a94159492476a835f8dcf9d3690ca1d31ef5d949ed8cd876afc130a597db951a67c4a5237447a20aa5e298615280ea66602e2cf01f4acac4e136ad93b33bdb11cceb1c7f69a1f7f15d003e6634af4682246b18a1a12a015196c97a066de1dff195ce89a15b0b66555e1a36655415ccd3067a3bd8cccb49a36ee136745f91904570493845c556a4a0b088092b8c398feffffff01") - -func BenchmarkCommon_RemoteSegment(b *testing.B) { - for i := 0; i < b.N; i++ { - p := pb.Pointer{} - err := proto.Unmarshal(testRemoteSegment, &p) - if err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkCommon_InlineSegment(b *testing.B) { - for i := 0; i < b.N; i++ { - p := pb.Pointer{} - err := proto.Unmarshal(testInlineSegment, &p) - if err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkFast_RemoteSegment(b *testing.B) { - for i := 0; i < b.N; i++ { - p := fastpb.Pointer{} - err := proto.Unmarshal(testRemoteSegment, &p) - if err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkFast_InlineSegment(b *testing.B) { - for i := 0; i < b.N; i++ { - p := fastpb.Pointer{} - err := proto.Unmarshal(testInlineSegment, &p) - if err != nil { - b.Fatal(err) - } - } -} diff --git a/cmd/metainfo-migration/fastpb/pointerdb.pb.go b/cmd/metainfo-migration/fastpb/pointerdb.pb.go deleted file mode 100644 index 1f71df355..000000000 --- a/cmd/metainfo-migration/fastpb/pointerdb.pb.go +++ /dev/null @@ -1,2403 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: pointerdb.proto - -package fastpb - -import ( - fmt "fmt" - proto "github.com/gogo/protobuf/proto" - _ "github.com/gogo/protobuf/types" - github_com_gogo_protobuf_types "github.com/gogo/protobuf/types" - io "io" - math "math" - math_bits "math/bits" - time "time" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf -var _ = time.Kitchen - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package - -type RedundancyScheme_SchemeType int32 - -const ( - RedundancyScheme_INVALID RedundancyScheme_SchemeType = 0 - RedundancyScheme_RS RedundancyScheme_SchemeType = 1 -) - -var RedundancyScheme_SchemeType_name = map[int32]string{ - 0: "INVALID", - 1: "RS", -} - -var RedundancyScheme_SchemeType_value = map[string]int32{ - "INVALID": 0, - "RS": 1, -} - -func (x RedundancyScheme_SchemeType) String() string { - return proto.EnumName(RedundancyScheme_SchemeType_name, int32(x)) -} - -func (RedundancyScheme_SchemeType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_75fef806d28fc810, []int{0, 0} -} - -type Pointer_DataType int32 - -const ( - Pointer_INLINE Pointer_DataType = 0 - Pointer_REMOTE Pointer_DataType = 1 -) - -var Pointer_DataType_name = map[int32]string{ - 0: "INLINE", - 1: "REMOTE", -} - -var Pointer_DataType_value = map[string]int32{ - "INLINE": 0, - "REMOTE": 1, -} - -func (x Pointer_DataType) String() string { - return proto.EnumName(Pointer_DataType_name, int32(x)) -} - -func (Pointer_DataType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_75fef806d28fc810, []int{3, 0} -} - -type RedundancyScheme struct { - Type RedundancyScheme_SchemeType `protobuf:"varint,1,opt,name=type,proto3,enum=fastpb.RedundancyScheme_SchemeType" json:"type,omitempty"` - // these values apply to RS encoding - MinReq int32 `protobuf:"varint,2,opt,name=min_req,json=minReq,proto3" json:"min_req,omitempty"` - Total int32 `protobuf:"varint,3,opt,name=total,proto3" json:"total,omitempty"` - RepairThreshold int32 `protobuf:"varint,4,opt,name=repair_threshold,json=repairThreshold,proto3" json:"repair_threshold,omitempty"` - SuccessThreshold int32 `protobuf:"varint,5,opt,name=success_threshold,json=successThreshold,proto3" json:"success_threshold,omitempty"` - ErasureShareSize int32 `protobuf:"varint,6,opt,name=erasure_share_size,json=erasureShareSize,proto3" json:"erasure_share_size,omitempty"` -} - -func (m *RedundancyScheme) Reset() { *m = RedundancyScheme{} } -func (m *RedundancyScheme) String() string { return proto.CompactTextString(m) } -func (*RedundancyScheme) ProtoMessage() {} -func (*RedundancyScheme) Descriptor() ([]byte, []int) { - return fileDescriptor_75fef806d28fc810, []int{0} -} -func (m *RedundancyScheme) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *RedundancyScheme) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_RedundancyScheme.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *RedundancyScheme) XXX_Merge(src proto.Message) { - xxx_messageInfo_RedundancyScheme.Merge(m, src) -} -func (m *RedundancyScheme) XXX_Size() int { - return m.Size() -} -func (m *RedundancyScheme) XXX_DiscardUnknown() { - xxx_messageInfo_RedundancyScheme.DiscardUnknown(m) -} - -var xxx_messageInfo_RedundancyScheme proto.InternalMessageInfo - -func (m *RedundancyScheme) GetType() RedundancyScheme_SchemeType { - if m != nil { - return m.Type - } - return RedundancyScheme_INVALID -} - -func (m *RedundancyScheme) GetMinReq() int32 { - if m != nil { - return m.MinReq - } - return 0 -} - -func (m *RedundancyScheme) GetTotal() int32 { - if m != nil { - return m.Total - } - return 0 -} - -func (m *RedundancyScheme) GetRepairThreshold() int32 { - if m != nil { - return m.RepairThreshold - } - return 0 -} - -func (m *RedundancyScheme) GetSuccessThreshold() int32 { - if m != nil { - return m.SuccessThreshold - } - return 0 -} - -func (m *RedundancyScheme) GetErasureShareSize() int32 { - if m != nil { - return m.ErasureShareSize - } - return 0 -} - -type RemotePiece struct { - PieceNum int32 `protobuf:"varint,1,opt,name=piece_num,json=pieceNum,proto3" json:"piece_num,omitempty"` - NodeId NodeID `protobuf:"bytes,2,opt,name=node_id,json=nodeId,proto3,customtype=NodeID" json:"node_id"` -} - -func (m *RemotePiece) Reset() { *m = RemotePiece{} } -func (m *RemotePiece) String() string { return proto.CompactTextString(m) } -func (*RemotePiece) ProtoMessage() {} -func (*RemotePiece) Descriptor() ([]byte, []int) { - return fileDescriptor_75fef806d28fc810, []int{1} -} -func (m *RemotePiece) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *RemotePiece) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_RemotePiece.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *RemotePiece) XXX_Merge(src proto.Message) { - xxx_messageInfo_RemotePiece.Merge(m, src) -} -func (m *RemotePiece) XXX_Size() int { - return m.Size() -} -func (m *RemotePiece) XXX_DiscardUnknown() { - xxx_messageInfo_RemotePiece.DiscardUnknown(m) -} - -var xxx_messageInfo_RemotePiece proto.InternalMessageInfo - -func (m *RemotePiece) GetPieceNum() int32 { - if m != nil { - return m.PieceNum - } - return 0 -} - -type RemoteSegment struct { - Redundancy *RedundancyScheme `protobuf:"bytes,1,opt,name=redundancy,proto3" json:"redundancy,omitempty"` - RootPieceId PieceID `protobuf:"bytes,2,opt,name=root_piece_id,json=rootPieceId,proto3,customtype=PieceID" json:"root_piece_id"` - RemotePieces []RemotePiece `protobuf:"bytes,3,rep,name=remote_pieces,json=remotePieces,proto3" json:"remote_pieces"` -} - -func (m *RemoteSegment) Reset() { *m = RemoteSegment{} } -func (m *RemoteSegment) String() string { return proto.CompactTextString(m) } -func (*RemoteSegment) ProtoMessage() {} -func (*RemoteSegment) Descriptor() ([]byte, []int) { - return fileDescriptor_75fef806d28fc810, []int{2} -} -func (m *RemoteSegment) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *RemoteSegment) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_RemoteSegment.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *RemoteSegment) XXX_Merge(src proto.Message) { - xxx_messageInfo_RemoteSegment.Merge(m, src) -} -func (m *RemoteSegment) XXX_Size() int { - return m.Size() -} -func (m *RemoteSegment) XXX_DiscardUnknown() { - xxx_messageInfo_RemoteSegment.DiscardUnknown(m) -} - -var xxx_messageInfo_RemoteSegment proto.InternalMessageInfo - -func (m *RemoteSegment) GetRedundancy() *RedundancyScheme { - if m != nil { - return m.Redundancy - } - return nil -} - -func (m *RemoteSegment) GetRemotePieces() []RemotePiece { - if m != nil { - return m.RemotePieces - } - return nil -} - -type Pointer struct { - Type Pointer_DataType `protobuf:"varint,1,opt,name=type,proto3,enum=fastpb.Pointer_DataType" json:"type,omitempty"` - InlineSegment []byte `protobuf:"bytes,3,opt,name=inline_segment,json=inlineSegment,proto3" json:"inline_segment,omitempty"` - Remote *RemoteSegment `protobuf:"bytes,4,opt,name=remote,proto3" json:"remote,omitempty"` - SegmentSize int64 `protobuf:"varint,5,opt,name=segment_size,json=segmentSize,proto3" json:"segment_size,omitempty"` - CreationDate time.Time `protobuf:"bytes,6,opt,name=creation_date,json=creationDate,proto3,stdtime" json:"creation_date"` - ExpirationDate time.Time `protobuf:"bytes,7,opt,name=expiration_date,json=expirationDate,proto3,stdtime" json:"expiration_date"` - Metadata []byte `protobuf:"bytes,8,opt,name=metadata,proto3" json:"metadata,omitempty"` -} - -func (m *Pointer) Reset() { *m = Pointer{} } -func (m *Pointer) String() string { return proto.CompactTextString(m) } -func (*Pointer) ProtoMessage() {} -func (*Pointer) Descriptor() ([]byte, []int) { - return fileDescriptor_75fef806d28fc810, []int{3} -} -func (m *Pointer) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Pointer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Pointer.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *Pointer) XXX_Merge(src proto.Message) { - xxx_messageInfo_Pointer.Merge(m, src) -} -func (m *Pointer) XXX_Size() int { - return m.Size() -} -func (m *Pointer) XXX_DiscardUnknown() { - xxx_messageInfo_Pointer.DiscardUnknown(m) -} - -var xxx_messageInfo_Pointer proto.InternalMessageInfo - -func (m *Pointer) GetType() Pointer_DataType { - if m != nil { - return m.Type - } - return Pointer_INLINE -} - -func (m *Pointer) GetInlineSegment() []byte { - if m != nil { - return m.InlineSegment - } - return nil -} - -func (m *Pointer) GetRemote() *RemoteSegment { - if m != nil { - return m.Remote - } - return nil -} - -func (m *Pointer) GetSegmentSize() int64 { - if m != nil { - return m.SegmentSize - } - return 0 -} - -func (m *Pointer) GetCreationDate() time.Time { - if m != nil { - return m.CreationDate - } - return time.Time{} -} - -func (m *Pointer) GetExpirationDate() time.Time { - if m != nil { - return m.ExpirationDate - } - return time.Time{} -} - -func (m *Pointer) GetMetadata() []byte { - if m != nil { - return m.Metadata - } - return nil -} - -type SegmentMeta struct { - EncryptedKey []byte `protobuf:"bytes,1,opt,name=encrypted_key,json=encryptedKey,proto3" json:"encrypted_key,omitempty"` - KeyNonce []byte `protobuf:"bytes,2,opt,name=key_nonce,json=keyNonce,proto3" json:"key_nonce,omitempty"` -} - -func (m *SegmentMeta) Reset() { *m = SegmentMeta{} } -func (m *SegmentMeta) String() string { return proto.CompactTextString(m) } -func (*SegmentMeta) ProtoMessage() {} -func (*SegmentMeta) Descriptor() ([]byte, []int) { - return fileDescriptor_75fef806d28fc810, []int{4} -} -func (m *SegmentMeta) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *SegmentMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_SegmentMeta.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *SegmentMeta) XXX_Merge(src proto.Message) { - xxx_messageInfo_SegmentMeta.Merge(m, src) -} -func (m *SegmentMeta) XXX_Size() int { - return m.Size() -} -func (m *SegmentMeta) XXX_DiscardUnknown() { - xxx_messageInfo_SegmentMeta.DiscardUnknown(m) -} - -var xxx_messageInfo_SegmentMeta proto.InternalMessageInfo - -func (m *SegmentMeta) GetEncryptedKey() []byte { - if m != nil { - return m.EncryptedKey - } - return nil -} - -func (m *SegmentMeta) GetKeyNonce() []byte { - if m != nil { - return m.KeyNonce - } - return nil -} - -type StreamInfo struct { - DeprecatedNumberOfSegments int64 `protobuf:"varint,1,opt,name=deprecated_number_of_segments,json=deprecatedNumberOfSegments,proto3" json:"deprecated_number_of_segments,omitempty"` - SegmentsSize int64 `protobuf:"varint,2,opt,name=segments_size,json=segmentsSize,proto3" json:"segments_size,omitempty"` - LastSegmentSize int64 `protobuf:"varint,3,opt,name=last_segment_size,json=lastSegmentSize,proto3" json:"last_segment_size,omitempty"` - Metadata []byte `protobuf:"bytes,4,opt,name=metadata,proto3" json:"metadata,omitempty"` -} - -func (m *StreamInfo) Reset() { *m = StreamInfo{} } -func (m *StreamInfo) String() string { return proto.CompactTextString(m) } -func (*StreamInfo) ProtoMessage() {} -func (*StreamInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_75fef806d28fc810, []int{5} -} -func (m *StreamInfo) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *StreamInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_StreamInfo.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *StreamInfo) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamInfo.Merge(m, src) -} -func (m *StreamInfo) XXX_Size() int { - return m.Size() -} -func (m *StreamInfo) XXX_DiscardUnknown() { - xxx_messageInfo_StreamInfo.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamInfo proto.InternalMessageInfo - -func (m *StreamInfo) GetDeprecatedNumberOfSegments() int64 { - if m != nil { - return m.DeprecatedNumberOfSegments - } - return 0 -} - -func (m *StreamInfo) GetSegmentsSize() int64 { - if m != nil { - return m.SegmentsSize - } - return 0 -} - -func (m *StreamInfo) GetLastSegmentSize() int64 { - if m != nil { - return m.LastSegmentSize - } - return 0 -} - -func (m *StreamInfo) GetMetadata() []byte { - if m != nil { - return m.Metadata - } - return nil -} - -type StreamMeta struct { - EncryptedStreamInfo []byte `protobuf:"bytes,1,opt,name=encrypted_stream_info,json=encryptedStreamInfo,proto3" json:"encrypted_stream_info,omitempty"` - EncryptionType int32 `protobuf:"varint,2,opt,name=encryption_type,json=encryptionType,proto3" json:"encryption_type,omitempty"` - EncryptionBlockSize int32 `protobuf:"varint,3,opt,name=encryption_block_size,json=encryptionBlockSize,proto3" json:"encryption_block_size,omitempty"` - LastSegmentMeta *SegmentMeta `protobuf:"bytes,4,opt,name=last_segment_meta,json=lastSegmentMeta,proto3" json:"last_segment_meta,omitempty"` - NumberOfSegments int64 `protobuf:"varint,5,opt,name=number_of_segments,json=numberOfSegments,proto3" json:"number_of_segments,omitempty"` -} - -func (m *StreamMeta) Reset() { *m = StreamMeta{} } -func (m *StreamMeta) String() string { return proto.CompactTextString(m) } -func (*StreamMeta) ProtoMessage() {} -func (*StreamMeta) Descriptor() ([]byte, []int) { - return fileDescriptor_75fef806d28fc810, []int{6} -} -func (m *StreamMeta) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *StreamMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_StreamMeta.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *StreamMeta) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamMeta.Merge(m, src) -} -func (m *StreamMeta) XXX_Size() int { - return m.Size() -} -func (m *StreamMeta) XXX_DiscardUnknown() { - xxx_messageInfo_StreamMeta.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamMeta proto.InternalMessageInfo - -func (m *StreamMeta) GetEncryptedStreamInfo() []byte { - if m != nil { - return m.EncryptedStreamInfo - } - return nil -} - -func (m *StreamMeta) GetEncryptionType() int32 { - if m != nil { - return m.EncryptionType - } - return 0 -} - -func (m *StreamMeta) GetEncryptionBlockSize() int32 { - if m != nil { - return m.EncryptionBlockSize - } - return 0 -} - -func (m *StreamMeta) GetLastSegmentMeta() *SegmentMeta { - if m != nil { - return m.LastSegmentMeta - } - return nil -} - -func (m *StreamMeta) GetNumberOfSegments() int64 { - if m != nil { - return m.NumberOfSegments - } - return 0 -} - -func init() { - proto.RegisterEnum("fastpb.RedundancyScheme_SchemeType", RedundancyScheme_SchemeType_name, RedundancyScheme_SchemeType_value) - proto.RegisterEnum("fastpb.Pointer_DataType", Pointer_DataType_name, Pointer_DataType_value) - proto.RegisterType((*RedundancyScheme)(nil), "fastpb.RedundancyScheme") - proto.RegisterType((*RemotePiece)(nil), "fastpb.RemotePiece") - proto.RegisterType((*RemoteSegment)(nil), "fastpb.RemoteSegment") - proto.RegisterType((*Pointer)(nil), "fastpb.Pointer") - proto.RegisterType((*SegmentMeta)(nil), "fastpb.SegmentMeta") - proto.RegisterType((*StreamInfo)(nil), "fastpb.StreamInfo") - proto.RegisterType((*StreamMeta)(nil), "fastpb.StreamMeta") -} - -func init() { proto.RegisterFile("pointerdb.proto", fileDescriptor_75fef806d28fc810) } - -var fileDescriptor_75fef806d28fc810 = []byte{ - // 843 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0x4d, 0x6f, 0x1b, 0x45, - 0x18, 0xf6, 0xc6, 0xf1, 0x07, 0xaf, 0x3f, 0x3b, 0xa5, 0xc2, 0x32, 0xc2, 0x49, 0xb7, 0x42, 0x0d, - 0x10, 0x5c, 0xc9, 0x3d, 0xc0, 0x09, 0x54, 0xcb, 0x39, 0xac, 0x68, 0x9c, 0x6a, 0x1c, 0x71, 0x5d, - 0x8d, 0x77, 0x5f, 0x3b, 0xab, 0x78, 0x67, 0xb6, 0xb3, 0x63, 0x09, 0xf7, 0x27, 0x20, 0x0e, 0xfd, - 0x0f, 0xfc, 0x0d, 0xc4, 0xb9, 0xc7, 0x1e, 0x11, 0x87, 0x82, 0x92, 0x3f, 0x82, 0x66, 0x66, 0x3f, - 0x9c, 0x08, 0x0e, 0x5c, 0x92, 0x9d, 0xf7, 0x7d, 0xe6, 0xfd, 0x78, 0x9e, 0x67, 0x0c, 0xbd, 0x44, - 0x44, 0x5c, 0xa1, 0x0c, 0x97, 0xe3, 0x44, 0x0a, 0x25, 0x48, 0x7d, 0xc5, 0x52, 0x95, 0x2c, 0x87, - 0x47, 0x6b, 0x21, 0xd6, 0x1b, 0x7c, 0x66, 0xa2, 0xcb, 0xed, 0xea, 0x99, 0x8a, 0x62, 0x4c, 0x15, - 0x8b, 0x13, 0x0b, 0x1c, 0xc2, 0x5a, 0xac, 0x85, 0xfd, 0x76, 0x7f, 0x3d, 0x80, 0x3e, 0xc5, 0x70, - 0xcb, 0x43, 0xc6, 0x83, 0xdd, 0x22, 0xb8, 0xc2, 0x18, 0xc9, 0x37, 0x70, 0xa8, 0x76, 0x09, 0x0e, - 0x9c, 0x63, 0xe7, 0xa4, 0x3b, 0x79, 0x32, 0xb6, 0x85, 0xc7, 0xf7, 0x71, 0x63, 0xfb, 0xef, 0x72, - 0x97, 0x20, 0x35, 0x17, 0xc8, 0x27, 0xd0, 0x88, 0x23, 0xee, 0x4b, 0x7c, 0x3d, 0x38, 0x38, 0x76, - 0x4e, 0x6a, 0xb4, 0x1e, 0x47, 0x9c, 0xe2, 0x6b, 0xf2, 0x31, 0xd4, 0x94, 0x50, 0x6c, 0x33, 0xa8, - 0x9a, 0xb0, 0x3d, 0x90, 0x2f, 0xa0, 0x2f, 0x31, 0x61, 0x91, 0xf4, 0xd5, 0x95, 0xc4, 0xf4, 0x4a, - 0x6c, 0xc2, 0xc1, 0xa1, 0x01, 0xf4, 0x6c, 0xfc, 0x32, 0x0f, 0x93, 0xaf, 0xe0, 0x41, 0xba, 0x0d, - 0x02, 0x4c, 0xd3, 0x3d, 0x6c, 0xcd, 0x60, 0xfb, 0x59, 0xa2, 0x04, 0x9f, 0x02, 0x41, 0xc9, 0xd2, - 0xad, 0x44, 0x3f, 0xbd, 0x62, 0xfa, 0x6f, 0xf4, 0x06, 0x07, 0x75, 0x8b, 0xce, 0x32, 0x0b, 0x9d, - 0x58, 0x44, 0x6f, 0xd0, 0x7d, 0x0c, 0x50, 0x2e, 0x42, 0x5a, 0xd0, 0xf0, 0xe6, 0x3f, 0xbe, 0x78, - 0xe9, 0xcd, 0xfa, 0x15, 0x52, 0x87, 0x03, 0xba, 0xe8, 0x3b, 0xee, 0x02, 0x5a, 0x14, 0x63, 0xa1, - 0xf0, 0x55, 0x84, 0x01, 0x92, 0x4f, 0xe1, 0xa3, 0x44, 0x7f, 0xf8, 0x7c, 0x1b, 0x1b, 0x92, 0x6a, - 0xb4, 0x69, 0x02, 0xf3, 0x6d, 0x4c, 0x9e, 0x42, 0x83, 0x8b, 0x10, 0xfd, 0x28, 0x34, 0x1c, 0xb4, - 0xa7, 0xdd, 0x77, 0x1f, 0x8e, 0x2a, 0x7f, 0x7e, 0x38, 0xaa, 0xcf, 0x45, 0x88, 0xde, 0x8c, 0xd6, - 0x75, 0xda, 0x0b, 0xdd, 0xdf, 0x1d, 0xe8, 0xd8, 0xaa, 0x0b, 0x5c, 0xc7, 0xc8, 0x15, 0xf9, 0x16, - 0x40, 0x16, 0x1c, 0x9b, 0xc2, 0xad, 0xc9, 0xe0, 0xbf, 0xd8, 0xa7, 0x7b, 0x58, 0xf2, 0x1c, 0x3a, - 0x52, 0x08, 0xe5, 0xdb, 0xb1, 0x8a, 0xd6, 0xbd, 0xac, 0x75, 0xc3, 0xcc, 0xed, 0xcd, 0x68, 0x4b, - 0xa3, 0xec, 0x21, 0x24, 0xdf, 0x41, 0x47, 0x9a, 0xfe, 0xf6, 0x5a, 0x3a, 0xa8, 0x1e, 0x57, 0x4f, - 0x5a, 0x93, 0x87, 0x65, 0xc7, 0x62, 0xe5, 0xe9, 0xa1, 0xae, 0x44, 0xdb, 0xb2, 0x0c, 0xa5, 0xee, - 0xcf, 0x55, 0x68, 0xbc, 0xb2, 0x26, 0x24, 0xa7, 0x77, 0x2c, 0x53, 0x0c, 0x9d, 0xa5, 0xc7, 0x33, - 0xa6, 0xd8, 0x9e, 0x4f, 0x3e, 0x87, 0x6e, 0xc4, 0x37, 0x11, 0x47, 0x3f, 0xb5, 0xab, 0x1b, 0x5f, - 0xb4, 0x69, 0xc7, 0x46, 0x73, 0x3e, 0xbe, 0x86, 0xba, 0x6d, 0x68, 0x5c, 0xd1, 0x9a, 0x3c, 0xba, - 0x3b, 0x59, 0x06, 0xa3, 0x19, 0x88, 0x3c, 0x86, 0x76, 0x56, 0xce, 0x0a, 0xae, 0xed, 0x51, 0xa5, - 0xad, 0x2c, 0xa6, 0xb5, 0x26, 0x1e, 0x74, 0x02, 0x89, 0x4c, 0x45, 0x82, 0xfb, 0x21, 0x53, 0xd6, - 0x14, 0xad, 0xc9, 0x70, 0x6c, 0xdf, 0xcc, 0x38, 0x7f, 0x33, 0xe3, 0xcb, 0xfc, 0xcd, 0x4c, 0x9b, - 0x7a, 0xf3, 0xb7, 0x7f, 0x1d, 0x39, 0xb4, 0x9d, 0x5f, 0x9d, 0x31, 0x85, 0xe4, 0x1c, 0x7a, 0xf8, - 0x53, 0x12, 0xc9, 0xbd, 0x62, 0x8d, 0xff, 0x51, 0xac, 0x5b, 0x5e, 0x36, 0xe5, 0x86, 0xd0, 0x8c, - 0x51, 0xb1, 0x90, 0x29, 0x36, 0x68, 0x1a, 0x32, 0x8a, 0xb3, 0xeb, 0x42, 0x33, 0x27, 0x90, 0x00, - 0xd4, 0xbd, 0xf9, 0x4b, 0x6f, 0x7e, 0xd6, 0xaf, 0xe8, 0x6f, 0x7a, 0x76, 0x7e, 0x71, 0x79, 0xd6, - 0x77, 0xdc, 0x0b, 0x68, 0x65, 0x7c, 0x9c, 0xa3, 0x62, 0xe4, 0x09, 0x74, 0x90, 0x07, 0x72, 0x97, - 0x28, 0x0c, 0xfd, 0x6b, 0xb4, 0x6e, 0x6a, 0xd3, 0x76, 0x11, 0xfc, 0x01, 0x77, 0xda, 0xc7, 0xd7, - 0xb8, 0xf3, 0xb9, 0xe0, 0x01, 0x5a, 0xc7, 0xd0, 0xe6, 0x35, 0xee, 0xe6, 0xfa, 0xec, 0xfe, 0xe6, - 0x00, 0x2c, 0x94, 0x44, 0x16, 0x7b, 0x7c, 0x25, 0xc8, 0x0b, 0xf8, 0x2c, 0xc4, 0x44, 0x62, 0xc0, - 0x74, 0x45, 0xbe, 0x8d, 0x97, 0x28, 0x7d, 0xb1, 0xca, 0x05, 0x4c, 0x4d, 0x83, 0x2a, 0x1d, 0x96, - 0xa0, 0xb9, 0xc1, 0x5c, 0xac, 0xb2, 0xb1, 0x52, 0x3d, 0x53, 0x8e, 0xb6, 0x02, 0x1d, 0x98, 0x2b, - 0xb9, 0x68, 0xa9, 0x51, 0xe8, 0x4b, 0x78, 0xb0, 0x61, 0xa9, 0xf2, 0xef, 0x28, 0x59, 0x35, 0xc0, - 0x9e, 0x4e, 0x2c, 0xf6, 0xd4, 0xdc, 0xe7, 0xec, 0xf0, 0x1e, 0x67, 0xbf, 0x1c, 0xe4, 0xe3, 0x1b, - 0x3e, 0x26, 0xf0, 0xa8, 0xe4, 0x23, 0x35, 0x71, 0x3f, 0xe2, 0x2b, 0x91, 0xf1, 0xf2, 0xb0, 0x48, - 0xee, 0xad, 0xfc, 0x14, 0x7a, 0x59, 0x58, 0x2b, 0x6c, 0xec, 0x6d, 0x7f, 0xd5, 0xba, 0x65, 0xd8, - 0x68, 0x52, 0x16, 0xd7, 0xc0, 0xe5, 0x46, 0x04, 0xd7, 0xe5, 0xdc, 0xb5, 0xa2, 0x78, 0x24, 0xf8, - 0x54, 0xe7, 0xcc, 0xec, 0xdf, 0xdf, 0xdb, 0x53, 0x0f, 0x9e, 0xd9, 0xbc, 0x78, 0x80, 0x7b, 0x82, - 0xde, 0x59, 0xde, 0x6c, 0x74, 0x0a, 0xe4, 0x5f, 0x54, 0xb0, 0x9e, 0xef, 0xf3, 0x7b, 0xdc, 0x4f, - 0x07, 0xef, 0x6e, 0x46, 0xce, 0xfb, 0x9b, 0x91, 0xf3, 0xf7, 0xcd, 0xc8, 0x79, 0x7b, 0x3b, 0xaa, - 0xbc, 0xbf, 0x1d, 0x55, 0xfe, 0xb8, 0x1d, 0x55, 0x96, 0x75, 0x63, 0xd3, 0xe7, 0xff, 0x04, 0x00, - 0x00, 0xff, 0xff, 0xd3, 0xcd, 0x4c, 0x1c, 0x50, 0x06, 0x00, 0x00, -} - -func (m *RedundancyScheme) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *RedundancyScheme) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *RedundancyScheme) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.ErasureShareSize != 0 { - i = encodeVarintPointerdb(dAtA, i, uint64(m.ErasureShareSize)) - i-- - dAtA[i] = 0x30 - } - if m.SuccessThreshold != 0 { - i = encodeVarintPointerdb(dAtA, i, uint64(m.SuccessThreshold)) - i-- - dAtA[i] = 0x28 - } - if m.RepairThreshold != 0 { - i = encodeVarintPointerdb(dAtA, i, uint64(m.RepairThreshold)) - i-- - dAtA[i] = 0x20 - } - if m.Total != 0 { - i = encodeVarintPointerdb(dAtA, i, uint64(m.Total)) - i-- - dAtA[i] = 0x18 - } - if m.MinReq != 0 { - i = encodeVarintPointerdb(dAtA, i, uint64(m.MinReq)) - i-- - dAtA[i] = 0x10 - } - if m.Type != 0 { - i = encodeVarintPointerdb(dAtA, i, uint64(m.Type)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *RemotePiece) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *RemotePiece) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *RemotePiece) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - { - size := m.NodeId.Size() - i -= size - if _, err := m.NodeId.MarshalTo(dAtA[i:]); err != nil { - return 0, err - } - i = encodeVarintPointerdb(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - if m.PieceNum != 0 { - i = encodeVarintPointerdb(dAtA, i, uint64(m.PieceNum)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *RemoteSegment) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *RemoteSegment) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *RemoteSegment) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.RemotePieces) > 0 { - for iNdEx := len(m.RemotePieces) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.RemotePieces[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPointerdb(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x1a - } - } - { - size := m.RootPieceId.Size() - i -= size - if _, err := m.RootPieceId.MarshalTo(dAtA[i:]); err != nil { - return 0, err - } - i = encodeVarintPointerdb(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - if m.Redundancy != nil { - { - size, err := m.Redundancy.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPointerdb(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *Pointer) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Pointer) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Pointer) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Metadata) > 0 { - i -= len(m.Metadata) - copy(dAtA[i:], m.Metadata) - i = encodeVarintPointerdb(dAtA, i, uint64(len(m.Metadata))) - i-- - dAtA[i] = 0x42 - } - n2, err2 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.ExpirationDate, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.ExpirationDate):]) - if err2 != nil { - return 0, err2 - } - i -= n2 - i = encodeVarintPointerdb(dAtA, i, uint64(n2)) - i-- - dAtA[i] = 0x3a - n3, err3 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.CreationDate, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.CreationDate):]) - if err3 != nil { - return 0, err3 - } - i -= n3 - i = encodeVarintPointerdb(dAtA, i, uint64(n3)) - i-- - dAtA[i] = 0x32 - if m.SegmentSize != 0 { - i = encodeVarintPointerdb(dAtA, i, uint64(m.SegmentSize)) - i-- - dAtA[i] = 0x28 - } - if m.Remote != nil { - { - size, err := m.Remote.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPointerdb(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x22 - } - if len(m.InlineSegment) > 0 { - i -= len(m.InlineSegment) - copy(dAtA[i:], m.InlineSegment) - i = encodeVarintPointerdb(dAtA, i, uint64(len(m.InlineSegment))) - i-- - dAtA[i] = 0x1a - } - if m.Type != 0 { - i = encodeVarintPointerdb(dAtA, i, uint64(m.Type)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *SegmentMeta) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *SegmentMeta) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *SegmentMeta) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.KeyNonce) > 0 { - i -= len(m.KeyNonce) - copy(dAtA[i:], m.KeyNonce) - i = encodeVarintPointerdb(dAtA, i, uint64(len(m.KeyNonce))) - i-- - dAtA[i] = 0x12 - } - if len(m.EncryptedKey) > 0 { - i -= len(m.EncryptedKey) - copy(dAtA[i:], m.EncryptedKey) - i = encodeVarintPointerdb(dAtA, i, uint64(len(m.EncryptedKey))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *StreamInfo) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *StreamInfo) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *StreamInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Metadata) > 0 { - i -= len(m.Metadata) - copy(dAtA[i:], m.Metadata) - i = encodeVarintPointerdb(dAtA, i, uint64(len(m.Metadata))) - i-- - dAtA[i] = 0x22 - } - if m.LastSegmentSize != 0 { - i = encodeVarintPointerdb(dAtA, i, uint64(m.LastSegmentSize)) - i-- - dAtA[i] = 0x18 - } - if m.SegmentsSize != 0 { - i = encodeVarintPointerdb(dAtA, i, uint64(m.SegmentsSize)) - i-- - dAtA[i] = 0x10 - } - if m.DeprecatedNumberOfSegments != 0 { - i = encodeVarintPointerdb(dAtA, i, uint64(m.DeprecatedNumberOfSegments)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *StreamMeta) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *StreamMeta) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *StreamMeta) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.NumberOfSegments != 0 { - i = encodeVarintPointerdb(dAtA, i, uint64(m.NumberOfSegments)) - i-- - dAtA[i] = 0x28 - } - if m.LastSegmentMeta != nil { - { - size, err := m.LastSegmentMeta.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPointerdb(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x22 - } - if m.EncryptionBlockSize != 0 { - i = encodeVarintPointerdb(dAtA, i, uint64(m.EncryptionBlockSize)) - i-- - dAtA[i] = 0x18 - } - if m.EncryptionType != 0 { - i = encodeVarintPointerdb(dAtA, i, uint64(m.EncryptionType)) - i-- - dAtA[i] = 0x10 - } - if len(m.EncryptedStreamInfo) > 0 { - i -= len(m.EncryptedStreamInfo) - copy(dAtA[i:], m.EncryptedStreamInfo) - i = encodeVarintPointerdb(dAtA, i, uint64(len(m.EncryptedStreamInfo))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func encodeVarintPointerdb(dAtA []byte, offset int, v uint64) int { - offset -= sovPointerdb(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *RedundancyScheme) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Type != 0 { - n += 1 + sovPointerdb(uint64(m.Type)) - } - if m.MinReq != 0 { - n += 1 + sovPointerdb(uint64(m.MinReq)) - } - if m.Total != 0 { - n += 1 + sovPointerdb(uint64(m.Total)) - } - if m.RepairThreshold != 0 { - n += 1 + sovPointerdb(uint64(m.RepairThreshold)) - } - if m.SuccessThreshold != 0 { - n += 1 + sovPointerdb(uint64(m.SuccessThreshold)) - } - if m.ErasureShareSize != 0 { - n += 1 + sovPointerdb(uint64(m.ErasureShareSize)) - } - return n -} - -func (m *RemotePiece) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.PieceNum != 0 { - n += 1 + sovPointerdb(uint64(m.PieceNum)) - } - l = m.NodeId.Size() - n += 1 + l + sovPointerdb(uint64(l)) - return n -} - -func (m *RemoteSegment) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Redundancy != nil { - l = m.Redundancy.Size() - n += 1 + l + sovPointerdb(uint64(l)) - } - l = m.RootPieceId.Size() - n += 1 + l + sovPointerdb(uint64(l)) - if len(m.RemotePieces) > 0 { - for _, e := range m.RemotePieces { - l = e.Size() - n += 1 + l + sovPointerdb(uint64(l)) - } - } - return n -} - -func (m *Pointer) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Type != 0 { - n += 1 + sovPointerdb(uint64(m.Type)) - } - l = len(m.InlineSegment) - if l > 0 { - n += 1 + l + sovPointerdb(uint64(l)) - } - if m.Remote != nil { - l = m.Remote.Size() - n += 1 + l + sovPointerdb(uint64(l)) - } - if m.SegmentSize != 0 { - n += 1 + sovPointerdb(uint64(m.SegmentSize)) - } - l = github_com_gogo_protobuf_types.SizeOfStdTime(m.CreationDate) - n += 1 + l + sovPointerdb(uint64(l)) - l = github_com_gogo_protobuf_types.SizeOfStdTime(m.ExpirationDate) - n += 1 + l + sovPointerdb(uint64(l)) - l = len(m.Metadata) - if l > 0 { - n += 1 + l + sovPointerdb(uint64(l)) - } - return n -} - -func (m *SegmentMeta) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.EncryptedKey) - if l > 0 { - n += 1 + l + sovPointerdb(uint64(l)) - } - l = len(m.KeyNonce) - if l > 0 { - n += 1 + l + sovPointerdb(uint64(l)) - } - return n -} - -func (m *StreamInfo) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.DeprecatedNumberOfSegments != 0 { - n += 1 + sovPointerdb(uint64(m.DeprecatedNumberOfSegments)) - } - if m.SegmentsSize != 0 { - n += 1 + sovPointerdb(uint64(m.SegmentsSize)) - } - if m.LastSegmentSize != 0 { - n += 1 + sovPointerdb(uint64(m.LastSegmentSize)) - } - l = len(m.Metadata) - if l > 0 { - n += 1 + l + sovPointerdb(uint64(l)) - } - return n -} - -func (m *StreamMeta) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.EncryptedStreamInfo) - if l > 0 { - n += 1 + l + sovPointerdb(uint64(l)) - } - if m.EncryptionType != 0 { - n += 1 + sovPointerdb(uint64(m.EncryptionType)) - } - if m.EncryptionBlockSize != 0 { - n += 1 + sovPointerdb(uint64(m.EncryptionBlockSize)) - } - if m.LastSegmentMeta != nil { - l = m.LastSegmentMeta.Size() - n += 1 + l + sovPointerdb(uint64(l)) - } - if m.NumberOfSegments != 0 { - n += 1 + sovPointerdb(uint64(m.NumberOfSegments)) - } - return n -} - -func sovPointerdb(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozPointerdb(x uint64) (n int) { - return sovPointerdb(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (m *RedundancyScheme) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: RedundancyScheme: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: RedundancyScheme: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) - } - m.Type = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Type |= RedundancyScheme_SchemeType(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field MinReq", wireType) - } - m.MinReq = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.MinReq |= int32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Total", wireType) - } - m.Total = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Total |= int32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 4: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field RepairThreshold", wireType) - } - m.RepairThreshold = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.RepairThreshold |= int32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 5: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field SuccessThreshold", wireType) - } - m.SuccessThreshold = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.SuccessThreshold |= int32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 6: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ErasureShareSize", wireType) - } - m.ErasureShareSize = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.ErasureShareSize |= int32(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipPointerdb(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPointerdb - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPointerdb - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *RemotePiece) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: RemotePiece: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: RemotePiece: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field PieceNum", wireType) - } - m.PieceNum = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.PieceNum |= int32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field NodeId", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthPointerdb - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthPointerdb - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if err := m.NodeId.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPointerdb(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPointerdb - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPointerdb - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *RemoteSegment) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: RemoteSegment: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: RemoteSegment: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Redundancy", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPointerdb - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPointerdb - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Redundancy == nil { - m.Redundancy = &RedundancyScheme{} - } - if err := m.Redundancy.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field RootPieceId", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthPointerdb - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthPointerdb - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if err := m.RootPieceId.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field RemotePieces", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPointerdb - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPointerdb - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.RemotePieces = append(m.RemotePieces, RemotePiece{}) - if err := m.RemotePieces[len(m.RemotePieces)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPointerdb(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPointerdb - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPointerdb - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *Pointer) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Pointer: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Pointer: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) - } - m.Type = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Type |= Pointer_DataType(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field InlineSegment", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthPointerdb - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthPointerdb - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.InlineSegment = append(m.InlineSegment[:0], dAtA[iNdEx:postIndex]...) - if m.InlineSegment == nil { - m.InlineSegment = []byte{} - } - iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Remote", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPointerdb - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPointerdb - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Remote == nil { - m.Remote = &RemoteSegment{} - } - if err := m.Remote.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 5: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field SegmentSize", wireType) - } - m.SegmentSize = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.SegmentSize |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 6: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field CreationDate", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPointerdb - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPointerdb - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(&m.CreationDate, dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 7: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ExpirationDate", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPointerdb - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPointerdb - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(&m.ExpirationDate, dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 8: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Metadata", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthPointerdb - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthPointerdb - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Metadata = append(m.Metadata[:0], dAtA[iNdEx:postIndex]...) - if m.Metadata == nil { - m.Metadata = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPointerdb(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPointerdb - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPointerdb - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *SegmentMeta) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: SegmentMeta: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: SegmentMeta: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field EncryptedKey", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthPointerdb - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthPointerdb - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.EncryptedKey = append(m.EncryptedKey[:0], dAtA[iNdEx:postIndex]...) - if m.EncryptedKey == nil { - m.EncryptedKey = []byte{} - } - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field KeyNonce", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthPointerdb - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthPointerdb - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.KeyNonce = append(m.KeyNonce[:0], dAtA[iNdEx:postIndex]...) - if m.KeyNonce == nil { - m.KeyNonce = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPointerdb(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPointerdb - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPointerdb - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *StreamInfo) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: StreamInfo: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: StreamInfo: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field DeprecatedNumberOfSegments", wireType) - } - m.DeprecatedNumberOfSegments = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.DeprecatedNumberOfSegments |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field SegmentsSize", wireType) - } - m.SegmentsSize = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.SegmentsSize |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field LastSegmentSize", wireType) - } - m.LastSegmentSize = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.LastSegmentSize |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Metadata", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthPointerdb - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthPointerdb - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Metadata = append(m.Metadata[:0], dAtA[iNdEx:postIndex]...) - if m.Metadata == nil { - m.Metadata = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPointerdb(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPointerdb - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPointerdb - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *StreamMeta) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: StreamMeta: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: StreamMeta: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field EncryptedStreamInfo", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthPointerdb - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthPointerdb - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.EncryptedStreamInfo = append(m.EncryptedStreamInfo[:0], dAtA[iNdEx:postIndex]...) - if m.EncryptedStreamInfo == nil { - m.EncryptedStreamInfo = []byte{} - } - iNdEx = postIndex - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field EncryptionType", wireType) - } - m.EncryptionType = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.EncryptionType |= int32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field EncryptionBlockSize", wireType) - } - m.EncryptionBlockSize = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.EncryptionBlockSize |= int32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field LastSegmentMeta", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPointerdb - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPointerdb - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.LastSegmentMeta == nil { - m.LastSegmentMeta = &SegmentMeta{} - } - if err := m.LastSegmentMeta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 5: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field NumberOfSegments", wireType) - } - m.NumberOfSegments = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPointerdb - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.NumberOfSegments |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipPointerdb(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPointerdb - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthPointerdb - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipPointerdb(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowPointerdb - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowPointerdb - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - return iNdEx, nil - case 1: - iNdEx += 8 - return iNdEx, nil - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowPointerdb - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthPointerdb - } - iNdEx += length - if iNdEx < 0 { - return 0, ErrInvalidLengthPointerdb - } - return iNdEx, nil - case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowPointerdb - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipPointerdb(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthPointerdb - } - } - return iNdEx, nil - case 4: - return iNdEx, nil - case 5: - iNdEx += 4 - return iNdEx, nil - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - } - panic("unreachable") -} - -var ( - ErrInvalidLengthPointerdb = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowPointerdb = fmt.Errorf("proto: integer overflow") -) diff --git a/cmd/metainfo-migration/fastpb/pointerdb.proto b/cmd/metainfo-migration/fastpb/pointerdb.proto deleted file mode 100644 index 0259415e7..000000000 --- a/cmd/metainfo-migration/fastpb/pointerdb.proto +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (C) 2019 Storj Labs, Inc. -// See LICENSE for copying information. - -syntax = "proto3"; - -package fastpb; - -import "google/protobuf/timestamp.proto"; -import "gogo.proto"; - -// This is a reduced version of pointerdb.proto for things that are -// relevant to migration. - -message RedundancyScheme { - enum SchemeType { - INVALID = 0; - RS = 1; - } - SchemeType type = 1; - - // these values apply to RS encoding - int32 min_req = 2; // minimum required for reconstruction - int32 total = 3; // total amount of pieces we generated - int32 repair_threshold = 4; // amount of pieces we need to drop to before triggering repair - int32 success_threshold = 5; // amount of pieces we need to store to call it a success - - int32 erasure_share_size = 6; -} - -message RemotePiece { - int32 piece_num = 1; - bytes node_id = 2 [(gogoproto.customtype) = "NodeID", (gogoproto.nullable) = false]; -} - -message RemoteSegment { - RedundancyScheme redundancy = 1; - bytes root_piece_id = 2 [(gogoproto.customtype) = "PieceID", (gogoproto.nullable) = false]; - repeated RemotePiece remote_pieces = 3 [(gogoproto.nullable) = false]; -} - -message Pointer { - enum DataType { - INLINE = 0; - REMOTE = 1; - } - - DataType type = 1; - - bytes inline_segment = 3; - RemoteSegment remote = 4; - int64 segment_size = 5; - - google.protobuf.Timestamp creation_date = 6 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false]; - google.protobuf.Timestamp expiration_date = 7 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false]; - - bytes metadata = 8; -} - -message SegmentMeta { - bytes encrypted_key = 1; - bytes key_nonce = 2; -} - -message StreamInfo { - int64 deprecated_number_of_segments = 1; - int64 segments_size = 2; - int64 last_segment_size = 3; - bytes metadata = 4; -} - -message StreamMeta { - bytes encrypted_stream_info = 1; - int32 encryption_type = 2; - int32 encryption_block_size = 3; - SegmentMeta last_segment_meta = 4; - int64 number_of_segments = 5; -} diff --git a/cmd/metainfo-migration/fastpb/types.go b/cmd/metainfo-migration/fastpb/types.go deleted file mode 100644 index 3a31caf1a..000000000 --- a/cmd/metainfo-migration/fastpb/types.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (C) 2021 Storj Labs, Inc. -// See LICENSE for copying information. - -// Package fastpb contains generated marshaling for Pointers. -package fastpb - -import ( - "storj.io/common/storj" -) - -//go:generate protoc -I=. --gogofaster_out=Mgoogle/protobuf/any.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/duration.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/struct.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/wrappers.proto=github.com/gogo/protobuf/types,paths=source_relative:. pointerdb.proto - -// NodeID is a unique node identifier. -type NodeID = storj.NodeID - -// PieceID is a unique identifier for pieces. -type PieceID = storj.PieceID diff --git a/cmd/metainfo-migration/main.go b/cmd/metainfo-migration/main.go deleted file mode 100644 index 24049ab22..000000000 --- a/cmd/metainfo-migration/main.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (C) 2021 Storj Labs, Inc. -// See LICENSE for copying information. - -package main - -import ( - "context" - "flag" - "log" - - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -const ( - defaultReadBatchSize = 3000000 - defaultWriteBatchSize = 100 - defaultWriteParallelLimit = 6 - defaultPreGeneratedStreamIDs = 100 -) - -var ( - cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") - - readBatchSize = flag.Int("readBatchSize", defaultReadBatchSize, "batch size for selecting pointers from DB") - writeBatchSize = flag.Int("writeBatchSize", defaultWriteBatchSize, "batch size for inserting objects and segments") - writeParallelLimit = flag.Int("writeParallelLimit", defaultWriteParallelLimit, "limit of parallel batch writes") - preGeneratedStreamIDs = flag.Int("preGeneratedStreamIDs", defaultPreGeneratedStreamIDs, "number of pre generated stream ids for segment") - nodes = flag.String("nodes", "", "file with nodes ids") - invalidObjects = flag.String("invalidObjects", "", "file for storing invalid objects") - - pointerdb = flag.String("pointerdb", "", "connection URL for PointerDB") - metabasedb = flag.String("metabasedb", "", "connection URL for MetabaseDB") -) - -func main() { - flag.Parse() - - if *pointerdb == "" { - log.Fatalln("Flag '--pointerdb' is not set") - } - if *metabasedb == "" { - log.Fatalln("Flag '--metabasedb' is not set") - } - - ctx := context.Background() - log, err := zap.Config{ - Encoding: "console", - Level: zap.NewAtomicLevelAt(zapcore.DebugLevel), - OutputPaths: []string{"stdout"}, - ErrorOutputPaths: []string{"stdout"}, - EncoderConfig: zapcore.EncoderConfig{ - LevelKey: "L", - NameKey: "N", - CallerKey: "C", - MessageKey: "M", - StacktraceKey: "S", - LineEnding: zapcore.DefaultLineEnding, - EncodeLevel: zapcore.CapitalLevelEncoder, - EncodeTime: zapcore.ISO8601TimeEncoder, - EncodeDuration: zapcore.StringDurationEncoder, - EncodeCaller: zapcore.ShortCallerEncoder, - }, - }.Build() - if err != nil { - panic(err) - } - defer func() { _ = log.Sync() }() - - config := Config{ - PreGeneratedStreamIDs: *preGeneratedStreamIDs, - ReadBatchSize: *readBatchSize, - WriteBatchSize: *writeBatchSize, - WriteParallelLimit: *writeParallelLimit, - Nodes: *nodes, - InvalidObjectsFile: *invalidObjects, - } - migrator := NewMigrator(log, *pointerdb, *metabasedb, config) - err = migrator.MigrateProjects(ctx) - if err != nil { - panic(err) - } -} diff --git a/cmd/metainfo-migration/migrator.go b/cmd/metainfo-migration/migrator.go deleted file mode 100644 index a4c2a2926..000000000 --- a/cmd/metainfo-migration/migrator.go +++ /dev/null @@ -1,705 +0,0 @@ -// Copyright (C) 2021 Storj Labs, Inc. -// See LICENSE for copying information. - -package main - -import ( - "bufio" - "bytes" - "context" - "encoding/binary" - "encoding/csv" - "encoding/hex" - "errors" - "os" - "runtime/pprof" - "sort" - "strconv" - "strings" - "time" - - proto "github.com/gogo/protobuf/proto" - "github.com/jackc/pgx/v4" - "github.com/jackc/pgx/v4/pgxpool" - "github.com/zeebo/errs" - "go.uber.org/zap" - - "storj.io/common/pb" - "storj.io/common/storj" - "storj.io/common/sync2" - "storj.io/common/uuid" - "storj.io/storj/cmd/metainfo-migration/fastpb" - "storj.io/storj/satellite/metainfo" - "storj.io/storj/satellite/metainfo/metabase" -) - -const objectArgs = 14 -const segmentArgs = 11 - -// EntryKey map key for object. -type EntryKey struct { - Bucket string - Key metabase.ObjectKey -} - -// Object represents object metadata. -type Object struct { - StreamID uuid.UUID - CreationDate time.Time - ExpireAt *time.Time - EncryptedMetadata []byte - EncryptedMetadataKey []byte - EncryptedMetadataNonce []byte - Encryption int64 - TotalEncryptedSize int64 - SegmentsRead int64 - SegmentsExpected int64 -} - -// Config initial settings for migrator. -type Config struct { - PreGeneratedStreamIDs int - ReadBatchSize int - WriteBatchSize int - WriteParallelLimit int - Nodes string - InvalidObjectsFile string - NumberOfRetries int -} - -// Migrator defines metainfo migrator. -type Migrator struct { - log *zap.Logger - pointerDBStr string - metabaseDBStr string - config Config - - objects [][]interface{} - segments [][]interface{} - - objectsSQL string - segmentsSQL string - - metabaseLimiter *sync2.Limiter -} - -// NewMigrator creates new metainfo migrator. -func NewMigrator(log *zap.Logger, pointerDBStr, metabaseDBStr string, config Config) *Migrator { - if config.ReadBatchSize == 0 { - config.ReadBatchSize = defaultReadBatchSize - } - if config.WriteBatchSize == 0 { - config.WriteBatchSize = defaultWriteBatchSize - } - if config.WriteParallelLimit == 0 { - config.WriteParallelLimit = defaultWriteParallelLimit - } - if config.PreGeneratedStreamIDs == 0 { - config.PreGeneratedStreamIDs = defaultPreGeneratedStreamIDs - } - if config.NumberOfRetries == 0 { - config.NumberOfRetries = 5 - } - return &Migrator{ - log: log, - pointerDBStr: pointerDBStr, - metabaseDBStr: metabaseDBStr, - config: config, - - objects: make([][]interface{}, 0, config.WriteBatchSize), - segments: make([][]interface{}, 0, config.WriteBatchSize), - objectsSQL: prepareObjectsSQL(config.WriteBatchSize), - segmentsSQL: prepareSegmentsSQL(config.WriteBatchSize), - - metabaseLimiter: sync2.NewLimiter(config.WriteParallelLimit), - } -} - -// MigrateProjects migrates all projects in pointerDB database. -func (m *Migrator) MigrateProjects(ctx context.Context) (err error) { - m.log.Debug("Databases", zap.String("PointerDB", m.pointerDBStr), zap.String("MetabaseDB", m.metabaseDBStr)) - - pointerDBConn, err := pgx.Connect(ctx, m.pointerDBStr) - if err != nil { - return errs.New("unable to connect %q: %w", m.pointerDBStr, err) - } - defer func() { err = errs.Combine(err, pointerDBConn.Close(ctx)) }() - - mb, err := metainfo.OpenMetabase(ctx, m.log.Named("metabase"), m.metabaseDBStr) - if err != nil { - return err - } - if err := mb.MigrateToLatest(ctx); err != nil { - return err - } - defer func() { err = errs.Combine(err, mb.Close()) }() - - aliasCache := metabase.NewNodeAliasCache(mb) - - config, err := pgxpool.ParseConfig(m.metabaseDBStr) - if err != nil { - return err - } - config.MaxConns = 10 - - metabaseConn, err := pgxpool.ConnectConfig(ctx, config) - if err != nil { - return errs.New("unable to connect %q: %w", m.metabaseDBStr, err) - } - defer func() { metabaseConn.Close() }() - - if *cpuprofile != "" { - f, err := os.Create(*cpuprofile) - if err != nil { - return err - } - err = pprof.StartCPUProfile(f) - if err != nil { - return err - } - defer pprof.StopCPUProfile() - } - - pointer := &fastpb.Pointer{} - streamMeta := &fastpb.StreamMeta{} - segmentMeta := &fastpb.SegmentMeta{} - segmentPosition := metabase.SegmentPosition{} - object := Object{} - location := metabase.ObjectLocation{} - - objects := make(map[EntryKey]Object) - var currentProject uuid.UUID - var fullpath, lastFullPath, metadata []byte - var allObjects, allSegments, zombieSegments int64 - - var invalidObjectsWriter *csv.Writer - if m.config.InvalidObjectsFile != "" { - objFile, err := os.Create(m.config.InvalidObjectsFile) - if err != nil { - return err - } - defer func() { err = errs.Combine(err, objFile.Close()) }() - - invalidObjectsWriter = csv.NewWriter(objFile) - } else { - invalidObjectsWriter = csv.NewWriter(os.Stdout) - } - - err = invalidObjectsWriter.Write([]string{"project_id", "bucket_name", "object_key", "stream_id", "expected_segments", "read_segments"}) - if err != nil { - return err - } - - start := time.Now() - if m.config.Nodes != "" { - err = m.aliasNodes(ctx, mb) - if err != nil { - return err - } - } - - lastCheck := time.Now() - m.log.Info("Start generating StreamIDs", zap.Int("total", m.config.PreGeneratedStreamIDs)) - ids, err := generateStreamIDs(m.config.PreGeneratedStreamIDs) - if err != nil { - return err - } - m.log.Info("Finished generating StreamIDs", zap.Duration("took", time.Since(lastCheck))) - - m.log.Info("Start", zap.Time("time", start), - zap.Int("readBatchSize", m.config.ReadBatchSize), - zap.Int("writeBatchSize", m.config.WriteBatchSize), - zap.Int("writeParallelLimit", m.config.WriteParallelLimit), - ) - - lastCheck = time.Now() - for { - hasResults := false - err = func() error { - var rows pgx.Rows - if len(lastFullPath) == 0 { - m.withRetry(ctx, func() (err error) { - rows, err = pointerDBConn.Query(ctx, `SELECT fullpath, metadata FROM pathdata ORDER BY fullpath ASC LIMIT $1`, m.config.ReadBatchSize) - return err - }) - } else { - m.withRetry(ctx, func() (err error) { - rows, err = pointerDBConn.Query(ctx, `SELECT fullpath, metadata FROM pathdata WHERE fullpath > $1 ORDER BY fullpath ASC LIMIT $2`, lastFullPath, m.config.ReadBatchSize) - return err - }) - } - - defer func() { rows.Close() }() - - for rows.Next() { - hasResults = true - err = rows.Scan(&fullpath, &metadata) - if err != nil { - return err - } - - lastFullPath = fullpath - - segmentKey, err := metabase.ParseSegmentKey(metabase.SegmentKey(fullpath)) - if err != nil { - // we should skip such errors as it looks we can have outdated entries - // in pointerdb like `project_id/l/bucket_name` without object key - m.log.Warn("unable to parse segment key", zap.Error(err)) - continue - } - - if !bytes.Equal(currentProject[:], segmentKey.ProjectID[:]) { - if len(objects) != 0 { - // TODO should we add such incomplete object into metabase? - for key, object := range objects { - err = invalidObjectsWriter.Write([]string{ - currentProject.String(), - key.Bucket, - hex.EncodeToString([]byte(key.Key)), - object.StreamID.String(), - strconv.FormatInt(object.SegmentsExpected, 10), - strconv.FormatInt(object.SegmentsRead, 10), - }) - if err != nil { - return err - } - } - invalidObjectsWriter.Flush() - - if err := invalidObjectsWriter.Error(); err != nil { - return err - } - - m.log.Warn("Object map should be empty after processing whole project", zap.String("ProjectID", currentProject.String()), zap.Int("Number of objects", len(objects))) - } - - currentProject = segmentKey.ProjectID - - for b := range objects { - delete(objects, b) - } - } - err = proto.Unmarshal(metadata, pointer) - if err != nil { - return err - } - - if allSegments != 0 && allSegments%1000000 == 0 { - m.log.Info("Processed segments", zap.Int64("segments", allSegments), zap.Duration("took", time.Since(lastCheck))) - lastCheck = time.Now() - } - - key := EntryKey{ - Bucket: segmentKey.BucketName, - Key: segmentKey.ObjectKey, - } - - // TODO: - // * detect empty objects and insert only object - if segmentKey.Position.Index == metabase.LastSegmentIndex { - // process last segment, it contains information about object and segment metadata - if len(ids) == 0 { - return errs.New("not enough generated stream ids") - } - streamID := ids[0] - err = proto.Unmarshal(pointer.Metadata, streamMeta) - if err != nil { - return err - } - // remove used ID - ids = ids[1:] - - var expireAt *time.Time - if !pointer.ExpirationDate.IsZero() { - // because we are reusing Pointer struct using it directly can cause race - copy := pointer.ExpirationDate - expireAt = © - } - - encryption, err := encodeEncryption(storj.EncryptionParameters{ - CipherSuite: storj.CipherSuite(streamMeta.EncryptionType), - BlockSize: streamMeta.EncryptionBlockSize, - }) - if err != nil { - return err - } - - object.StreamID = streamID - object.CreationDate = pointer.CreationDate - object.ExpireAt = expireAt - object.Encryption = encryption - object.EncryptedMetadataKey = streamMeta.LastSegmentMeta.EncryptedKey - object.EncryptedMetadataNonce = streamMeta.LastSegmentMeta.KeyNonce - object.EncryptedMetadata = pointer.Metadata // TODO this needs to be striped to EncryptedStreamInfo - - object.SegmentsRead = 1 - object.TotalEncryptedSize = pointer.SegmentSize - object.SegmentsExpected = streamMeta.NumberOfSegments - - // if object has only one segment then just insert it and don't put into map - if streamMeta.NumberOfSegments == 1 { - location.ProjectID = currentProject - location.BucketName = key.Bucket - location.ObjectKey = key.Key - err = m.insertObject(ctx, metabaseConn, location, object) - if err != nil { - return err - } - allObjects++ - } else { - objects[key] = object - } - - segmentPosition.Index = uint32(streamMeta.NumberOfSegments - 1) - err = m.insertSegment(ctx, metabaseConn, aliasCache, streamID, segmentPosition.Encode(), pointer, streamMeta.LastSegmentMeta) - if err != nil { - return err - } - } else { - object, ok := objects[key] - if !ok { - // TODO verify if its possible that DB has zombie segments - zombieSegments++ - } else { - err = pb.Unmarshal(pointer.Metadata, segmentMeta) - if err != nil { - return err - } - - segmentPosition.Index = segmentKey.Position.Index - err = m.insertSegment(ctx, metabaseConn, aliasCache, object.StreamID, segmentPosition.Encode(), pointer, segmentMeta) - if err != nil { - return err - } - - object.SegmentsRead++ - object.TotalEncryptedSize += pointer.SegmentSize - if object.SegmentsRead == object.SegmentsExpected { - location.ProjectID = currentProject - location.BucketName = key.Bucket - location.ObjectKey = key.Key - err = m.insertObject(ctx, metabaseConn, location, object) - if err != nil { - return err - } - allObjects++ - - delete(objects, key) - } else { - objects[key] = object - } - } - } - - allSegments++ - } - - return rows.Err() - }() - if err != nil { - return err - } - if !hasResults { - break - } - } - - err = m.flushObjects(ctx, metabaseConn) - if err != nil { - return err - } - err = m.flushSegments(ctx, metabaseConn) - if err != nil { - return err - } - - m.metabaseLimiter.Wait() - - m.log.Info("Finished", zap.Int64("objects", allObjects), zap.Int64("segments", allSegments), zap.Int64("invalid", zombieSegments), zap.Duration("total", time.Since(start))) - - return nil -} - -func (m *Migrator) insertObject(ctx context.Context, conn *pgxpool.Pool, location metabase.ObjectLocation, object Object) error { - m.objects = append(m.objects, []interface{}{ - location.ProjectID, location.BucketName, []byte(location.ObjectKey), 1, object.StreamID, - object.CreationDate, object.ExpireAt, - metabase.Committed, object.SegmentsRead, - object.EncryptedMetadata, object.EncryptedMetadataKey, object.EncryptedMetadataNonce, - object.TotalEncryptedSize, - object.Encryption, - }) - - if len(m.objects) >= m.config.WriteBatchSize { - err := m.flushObjects(ctx, conn) - if err != nil { - return err - } - } - return nil -} - -func (m *Migrator) insertSegment(ctx context.Context, conn *pgxpool.Pool, aliasCache *metabase.NodeAliasCache, streamID uuid.UUID, position uint64, pointer *fastpb.Pointer, segmentMeta *fastpb.SegmentMeta) (err error) { - var rootPieceID storj.PieceID - var remotePieces metabase.Pieces - var redundancy int64 - if pointer.Type == fastpb.Pointer_REMOTE && pointer.Remote != nil { - rootPieceID = pointer.Remote.RootPieceId - redundancy, err = encodeRedundancy(pointer.Remote.Redundancy) - if err != nil { - return err - } - - for _, remotePiece := range pointer.Remote.RemotePieces { - remotePieces = append(remotePieces, metabase.Piece{ - Number: uint16(remotePiece.PieceNum), - StorageNode: remotePiece.NodeId, - }) - } - } - - pieces, err := aliasCache.ConvertPiecesToAliases(ctx, remotePieces) - if err != nil { - return err - } - - sort.Slice(pieces, func(i, j int) bool { - return pieces[i].Number < pieces[j].Number - }) - - m.segments = append(m.segments, []interface{}{ - streamID, position, - rootPieceID, - segmentMeta.EncryptedKey, segmentMeta.KeyNonce, - pointer.SegmentSize, 0, 0, - redundancy, - pointer.InlineSegment, pieces, - }) - - if len(m.segments) >= m.config.WriteBatchSize { - err = m.flushSegments(ctx, conn) - if err != nil { - return err - } - } - return nil -} - -func encodeEncryption(params storj.EncryptionParameters) (int64, error) { - var bytes [8]byte - bytes[0] = byte(params.CipherSuite) - binary.LittleEndian.PutUint32(bytes[1:], uint32(params.BlockSize)) - return int64(binary.LittleEndian.Uint64(bytes[:])), nil -} - -func encodeRedundancy(redundancy *fastpb.RedundancyScheme) (int64, error) { - params := storj.RedundancyScheme{} - if redundancy != nil { - params.Algorithm = storj.RedundancyAlgorithm(redundancy.Type) - params.ShareSize = redundancy.ErasureShareSize - params.RequiredShares = int16(redundancy.MinReq) - params.RepairShares = int16(redundancy.RepairThreshold) - params.OptimalShares = int16(redundancy.SuccessThreshold) - params.TotalShares = int16(redundancy.Total) - } - - var bytes [8]byte - bytes[0] = byte(params.Algorithm) - - if params.ShareSize >= (1 << 24) { - return 0, errors.New("redundancy ShareSize is too big to encode") - } - - bytes[1] = byte(params.ShareSize >> 0) - bytes[2] = byte(params.ShareSize >> 8) - bytes[3] = byte(params.ShareSize >> 16) - - bytes[4] = byte(params.RequiredShares) - bytes[5] = byte(params.RepairShares) - bytes[6] = byte(params.OptimalShares) - bytes[7] = byte(params.TotalShares) - - return int64(binary.LittleEndian.Uint64(bytes[:])), nil -} - -func (m *Migrator) flushObjects(ctx context.Context, conn *pgxpool.Pool) error { - if len(m.objects) == 0 { - return nil - } - - objectsSQL := m.objectsSQL - if len(m.objects) < m.config.WriteBatchSize { - objectsSQL = prepareObjectsSQL(len(m.objects)) - } - - // TODO make predefined instance for that - params := []interface{}{} - for _, object := range m.objects { - params = append(params, object...) - } - - m.metabaseLimiter.Go(ctx, func() { - params := params - m.withRetry(ctx, func() error { - _, err := conn.Exec(ctx, objectsSQL, params...) - return err - }) - }) - - m.objects = m.objects[:0] - return nil -} - -func (m *Migrator) flushSegments(ctx context.Context, conn *pgxpool.Pool) error { - if len(m.segments) == 0 { - return nil - } - - segmentsSQL := m.segmentsSQL - if len(m.segments) < m.config.WriteBatchSize { - segmentsSQL = prepareSegmentsSQL(len(m.segments)) - } - - // TODO make predefined instance for that - params := make([]interface{}, 0, len(m.segments)*segmentArgs) - for _, segment := range m.segments { - params = append(params, segment...) - } - - m.metabaseLimiter.Go(ctx, func() { - params := params - m.withRetry(ctx, func() error { - _, err := conn.Exec(ctx, segmentsSQL, params...) - return err - }) - }) - - m.segments = m.segments[:0] - return nil -} - -func prepareObjectsSQL(batchSize int) string { - sql := ` - INSERT INTO objects ( - project_id, bucket_name, object_key, version, stream_id, - created_at, expires_at, - status, segment_count, - encrypted_metadata, encrypted_metadata_encrypted_key, encrypted_metadata_nonce, - total_encrypted_size, - encryption - ) VALUES - ` - i := 1 - for i < batchSize*objectArgs { - sql += parameters(objectArgs, i) + "," - i += objectArgs - } - return strings.TrimSuffix(sql, ",") -} - -func prepareSegmentsSQL(batchSize int) string { - sql := `INSERT INTO segments ( - stream_id, position, - root_piece_id, encrypted_key, encrypted_key_nonce, - encrypted_size, plain_offset, plain_size, - redundancy, - inline_data, remote_alias_pieces - ) VALUES - ` - i := 1 - for i < batchSize*segmentArgs { - sql += parameters(segmentArgs, i) + "," - i += segmentArgs - } - - return strings.TrimSuffix(sql, ",") -} - -func parameters(args, index int) string { - values := make([]string, args) - for i := index; i < args+index; i++ { - values[i-index] = "$" + strconv.Itoa(i) - } - return "(" + strings.Join(values, ",") + ")" -} - -func generateStreamIDs(numberOfIDs int) ([]uuid.UUID, error) { - ids := make([]uuid.UUID, numberOfIDs) - var err error - for i := 0; i < len(ids); i++ { - ids[i], err = uuid.New() - if err != nil { - return []uuid.UUID{}, err - } - } - - sort.Slice(ids, func(i, j int) bool { - return bytes.Compare(ids[i][:], ids[j][:]) == -1 - }) - return ids, nil -} - -func (m *Migrator) aliasNodes(ctx context.Context, mb metainfo.MetabaseDB) error { - start := time.Now() - m.log.Info("Start aliasing nodes") - file, err := os.Open(m.config.Nodes) - if err != nil { - return err - } - defer func() { err = errs.Combine(err, file.Close()) }() - - scanner := bufio.NewScanner(file) - nodes := make([]storj.NodeID, 0, 30000) - for scanner.Scan() { - line := scanner.Text() - decoded, err := hex.DecodeString(line) - if err != nil { - m.log.Error("unable decode node id", zap.String("value", line), zap.Error(err)) - continue - } - node, err := storj.NodeIDFromBytes(decoded) - if err != nil { - m.log.Error("unable create node id", zap.String("value", line), zap.Error(err)) - continue - } - nodes = append(nodes, node) - } - - // batch is used because we had issue with CRDB to put all nodes in one insert - batch := 1000 - for len(nodes) > 0 { - if len(nodes) < batch { - batch = len(nodes) - } - - err = mb.EnsureNodeAliases(ctx, metabase.EnsureNodeAliases{ - Nodes: nodes[:batch], - }) - if err != nil { - return err - } - nodes = nodes[batch:] - m.log.Info("Left to insert", zap.Int("nodes", len(nodes))) - } - m.log.Info("Finished aliasing nodes", zap.Duration("took", time.Since(start))) - return nil -} - -func (m *Migrator) withRetry(ctx context.Context, fn func() error) { - var err error - for i := 0; i < m.config.NumberOfRetries; i++ { - err = fn() - if err != nil { - m.log.Error("error occur", zap.Int("retry", i), zap.Error(err)) - - if !sync2.Sleep(ctx, 3*time.Second) { - m.log.Fatal("context error", zap.Error(ctx.Err())) - } - continue - } - return - } - - // make no sense to continue if even single query to DB fails - m.log.Fatal("query failed after retries", zap.Error(err)) -} diff --git a/cmd/metainfo-migration/migrator_test.go b/cmd/metainfo-migration/migrator_test.go deleted file mode 100644 index 0fa870cfe..000000000 --- a/cmd/metainfo-migration/migrator_test.go +++ /dev/null @@ -1,293 +0,0 @@ -// Copyright (C) 2021 Storj Labs, Inc. -// See LICENSE for copying information. - -package main_test - -import ( - "context" - "strconv" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" - - "storj.io/common/pb" - "storj.io/common/testcontext" - "storj.io/common/testrand" - "storj.io/common/uuid" - migration "storj.io/storj/cmd/metainfo-migration" - "storj.io/storj/private/dbutil/tempdb" - "storj.io/storj/satellite/metainfo" - "storj.io/storj/satellite/metainfo/metabase" - "storj.io/storj/satellite/satellitedb/satellitedbtest" - "storj.io/storj/storage" -) - -func TestMigrator_SingleSegmentObj(t *testing.T) { - expectedEntries := 1 - expectedProjectID := testrand.UUID() - expectedBucket := []byte("bucket-name") - expectedObjectKey := []byte("encrypted-key") - - var pointer *pb.Pointer - createPointers := func(t *testing.T, ctx context.Context, pointerDB metainfo.PointerDB) { - var err error - pointer, err = createLastSegment(ctx, pointerDB, expectedProjectID, expectedBucket, expectedObjectKey, 1) - require.NoError(t, err) - - // create invalid segment key which should be ignored during migration - err = pointerDB.Put(ctx, storage.Key("ff5b056b-5763-41f8-a928-286723cfefc9/l/test_bucket"), storage.Value([]byte{})) - require.NoError(t, err) - } - - checkMigration := func(t *testing.T, ctx context.Context, metabaseDB metainfo.MetabaseDB) { - objects, err := metabaseDB.TestingAllObjects(ctx) - require.NoError(t, err) - require.Len(t, objects, expectedEntries) - - { // verify object - require.EqualValues(t, expectedProjectID, objects[0].ProjectID) - require.EqualValues(t, expectedBucket, objects[0].BucketName) - require.EqualValues(t, expectedObjectKey, objects[0].ObjectKey) - require.EqualValues(t, pointer.SegmentSize, objects[0].TotalEncryptedSize) - require.EqualValues(t, 1, objects[0].SegmentCount) - require.Equal(t, metabase.Committed, objects[0].Status) - require.Zero(t, objects[0].TotalPlainSize) - require.WithinDuration(t, pointer.CreationDate, objects[0].CreatedAt, 5*time.Second) - require.WithinDuration(t, pointer.ExpirationDate, *objects[0].ExpiresAt, 5*time.Second) - - streamMeta := &pb.StreamMeta{} - err = pb.Unmarshal(pointer.Metadata, streamMeta) - require.NoError(t, err) - - require.Equal(t, pointer.Metadata, objects[0].EncryptedMetadata) - require.EqualValues(t, streamMeta.LastSegmentMeta.EncryptedKey, objects[0].EncryptedMetadataEncryptedKey) - require.EqualValues(t, streamMeta.LastSegmentMeta.KeyNonce, objects[0].EncryptedMetadataNonce) - - require.EqualValues(t, streamMeta.EncryptionType, objects[0].Encryption.CipherSuite) - require.EqualValues(t, streamMeta.EncryptionBlockSize, objects[0].Encryption.BlockSize) - - } - - { // verify segment - segments, err := metabaseDB.TestingAllSegments(ctx) - require.NoError(t, err) - require.Equal(t, len(segments), expectedEntries) - - require.Zero(t, segments[0].Position.Part) - require.Zero(t, segments[0].Position.Index) - require.Zero(t, segments[0].PlainOffset) - require.Zero(t, segments[0].PlainSize) - - require.EqualValues(t, pointer.Remote.RootPieceId, segments[0].RootPieceID) - - redundancy := pointer.Remote.Redundancy - require.EqualValues(t, redundancy.ErasureShareSize, segments[0].Redundancy.ShareSize) - require.EqualValues(t, redundancy.Type, segments[0].Redundancy.Algorithm) - require.EqualValues(t, redundancy.MinReq, segments[0].Redundancy.RequiredShares) - require.EqualValues(t, redundancy.RepairThreshold, segments[0].Redundancy.RepairShares) - require.EqualValues(t, redundancy.SuccessThreshold, segments[0].Redundancy.OptimalShares) - require.EqualValues(t, redundancy.Total, segments[0].Redundancy.TotalShares) - require.Empty(t, segments[0].InlineData) - - require.Equal(t, len(pointer.Remote.RemotePieces), len(segments[0].Pieces)) - for i, piece := range pointer.Remote.RemotePieces { - require.EqualValues(t, piece.PieceNum, segments[0].Pieces[i].Number) - require.Equal(t, piece.NodeId, segments[0].Pieces[i].StorageNode) - } - } - } - test(t, createPointers, checkMigration) -} - -func TestMigrator_ManyOneSegObj(t *testing.T) { - expectedEntries := 300 - createPointers := func(t *testing.T, ctx context.Context, pointerDB metainfo.PointerDB) { - projectID := testrand.UUID() - for i := 0; i < expectedEntries; i++ { - _, err := createLastSegment(ctx, pointerDB, projectID, []byte("bucket-name"), []byte("encrypted-key"+strconv.Itoa(i)), 1) - require.NoError(t, err) - } - - // create invalid segment key which should be ignored during migration - err := pointerDB.Put(ctx, storage.Key("005b056b-5763-41f8-a928-286723cfefc9/l/test_bucket"), storage.Value([]byte{})) - require.NoError(t, err) - } - - checkMigration := func(t *testing.T, ctx context.Context, metabaseDB metainfo.MetabaseDB) { - objects, err := metabaseDB.TestingAllObjects(ctx) - require.NoError(t, err) - require.Len(t, objects, expectedEntries) - - segments, err := metabaseDB.TestingAllSegments(ctx) - require.NoError(t, err) - require.Equal(t, len(segments), expectedEntries) - } - test(t, createPointers, checkMigration) -} - -func TestMigrator_MultiSegmentObj(t *testing.T) { - expectedEntries := 1000 - createPointers := func(t *testing.T, ctx context.Context, pointerDB metainfo.PointerDB) { - projectID := testrand.UUID() - - _, err := createLastSegment(ctx, pointerDB, projectID, []byte("bucket-name"), []byte("encrypted-key"), expectedEntries+1) - require.NoError(t, err) - for i := 0; i < expectedEntries; i++ { - err = createSegment(ctx, pointerDB, projectID, uint32(i), []byte("bucket-name"), []byte("encrypted-key")) - require.NoError(t, err) - } - } - - checkMigration := func(t *testing.T, ctx context.Context, metabaseDB metainfo.MetabaseDB) { - objects, err := metabaseDB.TestingAllObjects(ctx) - require.NoError(t, err) - require.Len(t, objects, 1) - - segments, err := metabaseDB.TestingAllSegments(ctx) - require.NoError(t, err) - require.Equal(t, len(segments), expectedEntries+1) - } - test(t, createPointers, checkMigration) -} - -func test(t *testing.T, createPointers func(t *testing.T, ctx context.Context, pointerDB metainfo.PointerDB), checkMigration func(t *testing.T, ctx context.Context, metabaseDB metainfo.MetabaseDB)) { - for _, satelliteDB := range satellitedbtest.Databases() { - satelliteDB := satelliteDB - if strings.EqualFold(satelliteDB.MasterDB.URL, "omit") { - continue - } - t.Run(satelliteDB.Name, func(t *testing.T) { - ctx := testcontext.New(t) - defer ctx.Cleanup() - - log := zaptest.NewLogger(t) - - schemaSuffix := satellitedbtest.SchemaSuffix() - schema := satellitedbtest.SchemaName(t.Name(), "category", 0, schemaSuffix) - - pointerTempDB, err := tempdb.OpenUnique(ctx, satelliteDB.PointerDB.URL, schema) - require.NoError(t, err) - - pointerDB, err := satellitedbtest.CreatePointerDBOnTopOf(ctx, log, pointerTempDB) - require.NoError(t, err) - defer ctx.Check(pointerDB.Close) - - schema = satellitedbtest.SchemaName(t.Name(), "category", 1, schemaSuffix) - metabaseTempDB, err := tempdb.OpenUnique(ctx, satelliteDB.MetabaseDB.URL, schema) - require.NoError(t, err) - metabaseDB, err := satellitedbtest.CreateMetabaseDBOnTopOf(ctx, log, metabaseTempDB) - require.NoError(t, err) - defer ctx.Check(metabaseDB.Close) - - createPointers(t, ctx, pointerDB) - - // TODO workaround for pgx - pConnStr := strings.Replace(pointerTempDB.ConnStr, "cockroach", "postgres", 1) - mConnStr := strings.Replace(metabaseTempDB.ConnStr, "cockroach", "postgres", 1) - migrator := migration.NewMigrator(log, pConnStr, mConnStr, migration.Config{ - PreGeneratedStreamIDs: 1000, - WriteBatchSize: 3, - WriteParallelLimit: 6, - InvalidObjectsFile: ctx.File(satelliteDB.Name + "_invalid_objects.csv"), - }) - err = migrator.MigrateProjects(ctx) - require.NoError(t, err) - - checkMigration(t, ctx, metabaseDB) - }) - } -} - -func createLastSegment(ctx context.Context, pointerDB metainfo.PointerDB, projectID uuid.UUID, bucket, encryptedKey []byte, numberOfSegments int) (*pb.Pointer, error) { - pointer := &pb.Pointer{} - pointer.Type = pb.Pointer_REMOTE - pointer.SegmentSize = 10 - pointer.CreationDate = time.Now() - pointer.ExpirationDate = time.Now() - pointer.Remote = &pb.RemoteSegment{ - RootPieceId: testrand.PieceID(), - Redundancy: &pb.RedundancyScheme{ - ErasureShareSize: 256, - Type: pb.RedundancyScheme_RS, - - MinReq: 1, - RepairThreshold: 2, - SuccessThreshold: 3, - Total: 4, - }, - RemotePieces: []*pb.RemotePiece{}, - } - - for i := 0; i < 10; i++ { - pointer.Remote.RemotePieces = append(pointer.Remote.RemotePieces, &pb.RemotePiece{ - PieceNum: int32(i), - NodeId: testrand.NodeID(), - }) - } - - streamMeta := &pb.StreamMeta{} - streamMeta.NumberOfSegments = int64(numberOfSegments) - streamMeta.EncryptedStreamInfo = testrand.Bytes(1024) - streamMeta.EncryptionBlockSize = 256 - streamMeta.EncryptionType = int32(pb.CipherSuite_ENC_AESGCM) - streamMeta.LastSegmentMeta = &pb.SegmentMeta{ - EncryptedKey: testrand.Bytes(256), - KeyNonce: testrand.Bytes(32), - } - metaBytes, err := pb.Marshal(streamMeta) - if err != nil { - return nil, err - } - pointer.Metadata = metaBytes - - path := strings.Join([]string{projectID.String(), "l", string(bucket), string(encryptedKey)}, "/") - - pointerBytes, err := pb.Marshal(pointer) - if err != nil { - return nil, err - } - err = pointerDB.Put(ctx, storage.Key(path), storage.Value(pointerBytes)) - if err != nil { - return nil, err - } - return pointer, nil -} - -func createSegment(ctx context.Context, pointerDB metainfo.PointerDB, projectID uuid.UUID, segmentIndex uint32, bucket, encryptedKey []byte) error { - pointer := &pb.Pointer{} - pointer.Type = pb.Pointer_REMOTE - pointer.SegmentSize = 10 - pointer.CreationDate = time.Now() - pointer.ExpirationDate = time.Now() - pointer.Remote = &pb.RemoteSegment{ - RootPieceId: testrand.PieceID(), - Redundancy: &pb.RedundancyScheme{}, - RemotePieces: []*pb.RemotePiece{}, - } - - segmentMeta := &pb.SegmentMeta{ - EncryptedKey: testrand.Bytes(256), - KeyNonce: testrand.Bytes(32), - } - - metaBytes, err := pb.Marshal(segmentMeta) - if err != nil { - return err - } - pointer.Metadata = metaBytes - - path := strings.Join([]string{projectID.String(), "s" + strconv.Itoa(int(segmentIndex)), string(bucket), string(encryptedKey)}, "/") - - pointerBytes, err := pb.Marshal(pointer) - if err != nil { - return err - } - err = pointerDB.Put(ctx, storage.Key(path), storage.Value(pointerBytes)) - if err != nil { - return err - } - return nil -} diff --git a/cmd/metainfo-verifier/main.go b/cmd/metainfo-verifier/main.go deleted file mode 100644 index d79b52475..000000000 --- a/cmd/metainfo-verifier/main.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (C) 2021 Storj Labs, Inc. -// See LICENSE for copying information. - -package main - -import ( - "context" - "flag" - "log" - - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -const ( - defaultSamplePercent = 1.0 -) - -var ( - cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") - - samplePercent = flag.Float64("samplePercent", defaultSamplePercent, "sample size to verify in percents") - - pointerdb = flag.String("pointerdb", "", "connection URL for PointerDB") - metabasedb = flag.String("metabasedb", "", "connection URL for MetabaseDB") -) - -func main() { - flag.Parse() - - if *pointerdb == "" { - log.Fatalln("Flag '--pointerdb' is not set") - } - if *metabasedb == "" { - log.Fatalln("Flag '--metabasedb' is not set") - } - if *samplePercent < 0 || *samplePercent > 100 { - log.Fatalln("Flag '--samplePercent' can take values between 0 and 100") - } - - ctx := context.Background() - log, err := zap.Config{ - Encoding: "console", - Level: zap.NewAtomicLevelAt(zapcore.DebugLevel), - OutputPaths: []string{"stdout"}, - ErrorOutputPaths: []string{"stdout"}, - EncoderConfig: zapcore.EncoderConfig{ - LevelKey: "L", - NameKey: "N", - CallerKey: "C", - MessageKey: "M", - StacktraceKey: "S", - LineEnding: zapcore.DefaultLineEnding, - EncodeLevel: zapcore.CapitalLevelEncoder, - EncodeTime: zapcore.ISO8601TimeEncoder, - EncodeDuration: zapcore.StringDurationEncoder, - EncodeCaller: zapcore.ShortCallerEncoder, - }, - }.Build() - if err != nil { - panic(err) - } - defer func() { _ = log.Sync() }() - - config := Config{ - SamplePercent: *samplePercent, - } - verifier := NewVerifier(log, *pointerdb, *metabasedb, config) - err = verifier.VerifyPointers(ctx) - if err != nil { - panic(err) - } -} diff --git a/cmd/metainfo-verifier/verifier.go b/cmd/metainfo-verifier/verifier.go deleted file mode 100644 index 065f33459..000000000 --- a/cmd/metainfo-verifier/verifier.go +++ /dev/null @@ -1,297 +0,0 @@ -// Copyright (C) 2021 Storj Labs, Inc. -// See LICENSE for copying information. - -package main - -import ( - "bytes" - "context" - "os" - "runtime/pprof" - "sort" - "time" - - proto "github.com/gogo/protobuf/proto" - "github.com/jackc/pgx/v4" - "github.com/zeebo/errs" - "go.uber.org/zap" - - "storj.io/common/pb" - "storj.io/storj/cmd/metainfo-migration/fastpb" - "storj.io/storj/satellite/metainfo" - "storj.io/storj/satellite/metainfo/metabase" -) - -// Config initial settings for verifier. -type Config struct { - SamplePercent float64 -} - -// Verifier defines metainfo migration verifier. -type Verifier struct { - log *zap.Logger - pointerDBStr string - metabaseDBStr string - config Config -} - -// NewVerifier creates new metainfo migration verifier. -func NewVerifier(log *zap.Logger, pointerDBStr, metabaseDBStr string, config Config) *Verifier { - if config.SamplePercent == 0 { - config.SamplePercent = defaultSamplePercent - } - return &Verifier{ - log: log, - pointerDBStr: pointerDBStr, - metabaseDBStr: metabaseDBStr, - config: config, - } -} - -// VerifyPointers verifies a sample of all migrated pointers. -func (v *Verifier) VerifyPointers(ctx context.Context) (err error) { - v.log.Debug("Databases", zap.String("PointerDB", v.pointerDBStr), zap.String("MetabaseDB", v.metabaseDBStr)) - - pointerDBConn, err := pgx.Connect(ctx, v.pointerDBStr) - if err != nil { - return errs.New("unable to connect %q: %w", v.pointerDBStr, err) - } - defer func() { err = errs.Combine(err, pointerDBConn.Close(ctx)) }() - - mb, err := metainfo.OpenMetabase(ctx, v.log.Named("metabase"), v.metabaseDBStr) - if err != nil { - return err - } - defer func() { err = errs.Combine(err, mb.Close()) }() - - if *cpuprofile != "" { - f, err := os.Create(*cpuprofile) - if err != nil { - return err - } - err = pprof.StartCPUProfile(f) - if err != nil { - return err - } - defer pprof.StopCPUProfile() - } - - pointer := &fastpb.Pointer{} - streamMeta := &fastpb.StreamMeta{} - segmentMeta := &fastpb.SegmentMeta{} - var fullpath, metadata []byte - var allSegments int64 - var lastSegment bool - - start := time.Now() - - v.log.Info("Start", zap.Time("time", start), - zap.Float64("samplePercent", v.config.SamplePercent), - ) - - rows, err := pointerDBConn.Query(ctx, `SELECT fullpath, metadata FROM pathdata WHERE RANDOM() < $1`, v.config.SamplePercent/100) - if err != nil { - return err - } - defer func() { rows.Close() }() - - lastCheck := time.Now() - for rows.Next() { - err = rows.Scan(&fullpath, &metadata) - if err != nil { - return err - } - - segmentLocation, err := metabase.ParseSegmentKey(metabase.SegmentKey(fullpath)) - if err != nil { - return errs.New("%v; pointer: %s", err, fullpath) - } - - err = proto.Unmarshal(metadata, pointer) - if err != nil { - return wrap(err, segmentLocation) - } - - lastSegment = segmentLocation.Position.Index == metabase.LastSegmentIndex - if lastSegment { - err = proto.Unmarshal(pointer.Metadata, streamMeta) - if err != nil { - return wrap(err, segmentLocation) - } - // calculate the metabase segment position, so we can query the segment from the metabase - segmentLocation.Position.Index = uint32(streamMeta.NumberOfSegments) - 1 - } - - segment, err := mb.GetSegmentByLocation(ctx, metabase.GetSegmentByLocation{SegmentLocation: segmentLocation}) - if err != nil { - return wrap(err, segmentLocation) - } - - if pointer.Type == fastpb.Pointer_DataType(pb.Pointer_REMOTE) { - if len(segment.InlineData) > 0 { - return wrap(errs.New("unexpected inline data for remote segment"), segmentLocation) - } - if pointer.Remote.RootPieceId != segment.RootPieceID { - return wrap(errs.New("root piece id does not match: want %s, got %s", pointer.Remote.RootPieceId, segment.RootPieceID), segmentLocation) - } - if pointer.Remote.Redundancy.Type != fastpb.RedundancyScheme_SchemeType(segment.Redundancy.Algorithm) { - return wrap(errs.New("redundancy scheme type does not match: want %d, got %d", pointer.Remote.Redundancy.Type, segment.Redundancy.Algorithm), segmentLocation) - } - if pointer.Remote.Redundancy.MinReq != int32(segment.Redundancy.RequiredShares) { - return wrap(errs.New("redundancy scheme required shares does not match: want %d, got %d", pointer.Remote.Redundancy.MinReq, segment.Redundancy.RequiredShares), segmentLocation) - } - if pointer.Remote.Redundancy.RepairThreshold != int32(segment.Redundancy.RepairShares) { - return wrap(errs.New("redundancy scheme repair shares does not match: want %d, got %d", pointer.Remote.Redundancy.RepairThreshold, segment.Redundancy.RepairShares), segmentLocation) - } - if pointer.Remote.Redundancy.SuccessThreshold != int32(segment.Redundancy.OptimalShares) { - return wrap(errs.New("redundancy scheme optimal shares does not match: want %d, got %d", pointer.Remote.Redundancy.SuccessThreshold, segment.Redundancy.OptimalShares), segmentLocation) - } - if pointer.Remote.Redundancy.Total != int32(segment.Redundancy.TotalShares) { - return wrap(errs.New("redundancy scheme total shares does not match: want %d, got %d", pointer.Remote.Redundancy.Total, segment.Redundancy.TotalShares), segmentLocation) - } - if pointer.Remote.Redundancy.ErasureShareSize != segment.Redundancy.ShareSize { - return wrap(errs.New("redundancy scheme erasure share size does not match: want %d, got %d", pointer.Remote.Redundancy.ErasureShareSize, segment.Redundancy.ShareSize), segmentLocation) - } - if len(pointer.Remote.RemotePieces) != segment.Pieces.Len() { - return wrap(errs.New("number of remote pieces does not match: want %d, got %d", len(pointer.Remote.RemotePieces), segment.Pieces.Len()), segmentLocation) - } - sort.Slice(pointer.Remote.RemotePieces, func(i, k int) bool { - return pointer.Remote.RemotePieces[i].PieceNum < pointer.Remote.RemotePieces[k].PieceNum - }) - sort.Slice(segment.Pieces, func(i, k int) bool { - return segment.Pieces[i].Number < segment.Pieces[k].Number - }) - for i, piece := range pointer.Remote.RemotePieces { - if piece.PieceNum != int32(segment.Pieces[i].Number) { - return wrap(errs.New("piece number does not match for remote piece %d: want %d, got %d", i, piece.PieceNum, segment.Pieces[i].Number), segmentLocation) - } - if piece.NodeId != segment.Pieces[i].StorageNode { - return wrap(errs.New("storage node id does not match for remote piece %d: want %s, got %s", i, piece.NodeId, segment.Pieces[i].StorageNode), segmentLocation) - } - } - } else { - if !bytes.Equal(pointer.InlineSegment, segment.InlineData) { - return wrap(errs.New("inline data does not match: want %x, got %x", pointer.InlineSegment, segment.InlineData), segmentLocation) - } - if !segment.RootPieceID.IsZero() { - return wrap(errs.New("unexpected root piece id for inline segment"), segmentLocation) - } - if !segment.Redundancy.IsZero() { - return wrap(errs.New("unexpected redundancy scheme for inline segment"), segmentLocation) - } - if segment.Pieces.Len() > 0 { - return wrap(errs.New("unexpected remote pieces for inline segment"), segmentLocation) - } - } - - if segment.StreamID.IsZero() { - return wrap(errs.New("missing stream id in segment"), segmentLocation) - } - if pointer.SegmentSize != int64(segment.EncryptedSize) { - return wrap(errs.New("segment size does not match: want %d, got %d", pointer.SegmentSize, segment.EncryptedSize), segmentLocation) - } - if segment.PlainOffset != 0 { - return wrap(errs.New("unexpected plain offset: %d", segment.PlainOffset), segmentLocation) - } - if segment.PlainSize != 0 { - return wrap(errs.New("unexpected plain size: %d", segment.PlainSize), segmentLocation) - } - - if lastSegment { - object, err := mb.GetObjectLatestVersion(ctx, metabase.GetObjectLatestVersion{ObjectLocation: segmentLocation.Object()}) - if err != nil { - return wrap(err, segmentLocation) - } - if object.StreamID.IsZero() { - return wrap(errs.New("missing stream id in object"), segmentLocation) - } - if object.StreamID != segment.StreamID { - return wrap(errs.New("stream id does no match: object %s, segment %s", object.StreamID, segment.StreamID), segmentLocation) - } - if object.Version != 1 { - return wrap(errs.New("unexpected version: want %d, got %d", 1, object.Version), segmentLocation) - } - if object.Status != metabase.Committed { - return wrap(errs.New("unexpected status: want %d, got %d", metabase.Committed, object.Status), segmentLocation) - } - if !withinDuration(pointer.CreationDate, object.CreatedAt, 1*time.Microsecond) { - return wrap(errs.New("creation date does not match: want %s, got %s", pointer.CreationDate, object.CreatedAt), segmentLocation) - } - if object.ExpiresAt == nil { - if !pointer.ExpirationDate.IsZero() { - return wrap(errs.New("missing expiration date"), segmentLocation) - } - } else if !withinDuration(pointer.ExpirationDate, *object.ExpiresAt, 1*time.Microsecond) { - return wrap(errs.New("expiration date does not match: want %s, got %s", pointer.ExpirationDate, object.ExpiresAt), segmentLocation) - } - if int32(streamMeta.NumberOfSegments) != object.SegmentCount { - return wrap(errs.New("number of segments does not match: want %d, got %d", streamMeta.NumberOfSegments, object.SegmentCount), segmentLocation) - } - if object.FixedSegmentSize != 0 { - return wrap(errs.New("unexpected fixed segment size: %d", object.FixedSegmentSize), segmentLocation) - } - if object.SegmentCount == 1 { - if pointer.SegmentSize != object.TotalEncryptedSize { - return wrap(errs.New("total encrypted size does not match: want %d, got %d", pointer.SegmentSize, object.TotalEncryptedSize), segmentLocation) - } - } else { - if pointer.SegmentSize >= object.TotalEncryptedSize { - return wrap(errs.New("total encrypted size does not match: want >%d, got %d", pointer.SegmentSize, object.TotalEncryptedSize), segmentLocation) - } - } - if object.TotalPlainSize != 0 { - return wrap(errs.New("unexpected total plain size: %d", object.TotalPlainSize), segmentLocation) - } - if streamMeta.EncryptionType != int32(object.Encryption.CipherSuite) { - return wrap(errs.New("encryption type does not match: want %d, got %d", streamMeta.EncryptionType, object.Encryption.CipherSuite), segmentLocation) - } - if streamMeta.EncryptionBlockSize != object.Encryption.BlockSize { - return wrap(errs.New("encryption block size does not match: want %d, got %d", streamMeta.EncryptionBlockSize, object.Encryption.BlockSize), segmentLocation) - } - if !bytes.Equal(streamMeta.LastSegmentMeta.EncryptedKey, object.EncryptedMetadataEncryptedKey) { - return wrap(errs.New("encrypted metadata encrypted key does not match: want %x, got %x", streamMeta.LastSegmentMeta.EncryptedKey, object.EncryptedMetadataEncryptedKey), segmentLocation) - } - if !bytes.Equal(streamMeta.LastSegmentMeta.KeyNonce, object.EncryptedMetadataNonce) { - return wrap(errs.New("encrypted metadata key nonce does not match: want %x, got %x", streamMeta.LastSegmentMeta.KeyNonce, object.EncryptedMetadataNonce), segmentLocation) - } - if !bytes.Equal(pointer.Metadata, object.EncryptedMetadata) { - return wrap(errs.New("encrypted metadata does not match: want %x, got %x", pointer.Metadata, object.EncryptedMetadata), segmentLocation) - } - if object.ZombieDeletionDeadline != nil { - return wrap(errs.New("unexpected zombie deletion deadline: %s", object.ZombieDeletionDeadline), segmentLocation) - } - } else { - err = pb.Unmarshal(pointer.Metadata, segmentMeta) - if err != nil { - return wrap(err, segmentLocation) - } - if !bytes.Equal(segmentMeta.EncryptedKey, segment.EncryptedKey) { - return wrap(errs.New("segment metadata encrypted key does not match: want %x, got %x", segmentMeta.EncryptedKey, segment.EncryptedKey), segmentLocation) - } - if !bytes.Equal(segmentMeta.KeyNonce, segment.EncryptedKeyNonce) { - return wrap(errs.New("segment metadata key nonce does not match: want %x, got %x", segmentMeta.KeyNonce, segment.EncryptedKeyNonce), segmentLocation) - } - } - - if allSegments != 0 && allSegments%100 == 0 { - v.log.Info("Processed segments", zap.Int64("segments", allSegments), zap.Duration("took", time.Since(lastCheck))) - lastCheck = time.Now() - } - - allSegments++ - } - - v.log.Info("Finished", zap.Int64("segments", allSegments), zap.Duration("Total", time.Since(start))) - - return rows.Err() -} - -func withinDuration(expected, actual time.Time, delta time.Duration) bool { - dt := expected.Sub(actual) - return -delta < dt && dt < delta -} - -func wrap(err error, segment metabase.SegmentLocation) error { - return errs.New("%v; project: %x, bucket: %s, object: %x, index: %d", - err, segment.ProjectID, segment.BucketName, segment.ObjectKey, segment.Position.Index) -} diff --git a/go.sum b/go.sum index db569ab2e..0f464127a 100644 --- a/go.sum +++ b/go.sum @@ -334,7 +334,6 @@ github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= diff --git a/scripts/test-sim-backwards.sh b/scripts/test-sim-backwards.sh index 2866bdde4..3403914f8 100755 --- a/scripts/test-sim-backwards.sh +++ b/scripts/test-sim-backwards.sh @@ -146,10 +146,6 @@ STORJ_NUM_NODES=9 ## Run tests on the branch under test. ## -go install storj.io/storj/cmd/metainfo-migration -STORJ_MIGRATION_DB=${STORJ_MIGRATION_DB:-$STORJ_SIM_POSTGRES} -metainfo-migration --pointerdb "${STORJ_MIGRATION_DB}" --metabasedb "${STORJ_MIGRATION_DB}" - # check that branch uplink + branch network can read fully release data test_branch -b release-network-release-uplink download diff --git a/scripts/tests/rollingupgrade/test-sim-rolling-upgrade.sh b/scripts/tests/rollingupgrade/test-sim-rolling-upgrade.sh index 53cba2b9c..764e9ef2c 100755 --- a/scripts/tests/rollingupgrade/test-sim-rolling-upgrade.sh +++ b/scripts/tests/rollingupgrade/test-sim-rolling-upgrade.sh @@ -142,10 +142,6 @@ setup_stage(){ if [[ $stage == "2" ]] then mv $dest_sat_cfg_dir/satellite $dest_sat_cfg_dir/old_satellite - - go build -v -o $test_dir/bin storj.io/storj/cmd/metainfo-migration >/dev/null 2>&1 - STORJ_MIGRATION_DB=${STORJ_MIGRATION_DB:-$STORJ_SIM_POSTGRES} - $test_dir/bin/metainfo-migration --pointerdb "${STORJ_MIGRATION_DB}" --metabasedb "${STORJ_MIGRATION_DB}" fi # ln binary and copy config.yaml for desired version